You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by br...@apache.org on 2017/06/12 13:58:32 UTC
[2/2] nifi git commit: NIFI-3696 - ConfigMigration and FileManager
tools
NIFI-3696 - ConfigMigration and FileManager tools
This closes #1889.
Signed-off-by: Bryan Rosander <br...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/8ef4fddd
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/8ef4fddd
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/8ef4fddd
Branch: refs/heads/master
Commit: 8ef4fddddd815e373523ac6a3e10012d34fe4a78
Parents: 7761903
Author: Yolanda M. Davis <yo...@gmail.com>
Authored: Wed May 31 10:17:23 2017 -0400
Committer: Bryan Rosander <br...@apache.org>
Committed: Mon Jun 12 09:50:02 2017 -0400
----------------------------------------------------------------------
.../src/main/asciidoc/administration-guide.adoc | 79 +++
.../nifi/toolkit/admin/AbstractAdminTool.groovy | 48 +-
.../admin/configmigrator/ConfigMigrator.groovy | 223 +++++++
.../rules/ConfigMigrationRule.groovy | 25 +
.../rules/GenericMigrationRule.groovy | 32 +
.../rules/PropertyMigrationRule.groovy | 98 +++
.../rules/XmlMigrationRule.groovy | 45 ++
.../admin/filemanager/FileManagerTool.groovy | 599 +++++++++++++++++++
.../admin/nodemanager/NodeManagerTool.groovy | 58 +-
.../admin/notify/NotificationTool.groovy | 17 +-
.../nifi/toolkit/admin/util/AdminUtil.groovy | 43 +-
.../configmigrator/ConfigMigratorSpec.groovy | 224 +++++++
.../filemanager/FileManagerToolSpec.groovy | 511 ++++++++++++++++
.../toolkit/admin/util/AdminUtilSpec.groovy | 31 +
.../conf/bootstrap-notification-services.xml | 46 ++
.../src/test/resources/conf/logback.xml | 169 ++++++
.../filemanager/nifi-test-archive.tar.gz | Bin 27167 -> 29306 bytes
.../rules/v1_1_0/authorizations-xml.groovy | 35 ++
.../rules/v1_2_0/authorizations-xml.groovy | 35 ++
.../rules/v1_2_0/authorizers-xml.groovy | 35 ++
.../rules/v1_2_0/bootstrap-conf.groovy | 36 ++
.../bootstrap-notification-services-xml.groovy | 46 ++
.../resources/rules/v1_2_0/logback-xml.groovy | 70 +++
.../v1_2_0/login-identity-providers-xml.groovy | 36 ++
.../rules/v1_2_0/nifi-properties.groovy | 33 +
.../rules/v1_2_0/state-management-xml.groovy | 37 ++
.../resources/rules/v1_2_0/users-xml.groovy | 35 ++
.../rules/v1_2_0/zookeeper-properties.groovy | 35 ++
.../rules/v1_3_0/authorizations-xml.groovy | 35 ++
.../conf/bootstrap-notification-services.xml | 53 ++
.../src/test/resources/upgrade/conf/logback.xml | 170 ++++++
.../src/main/assembly/dependencies.xml | 5 +
.../src/main/resources/bin/file-manager.bat | 39 ++
.../src/main/resources/bin/file-manager.sh | 119 ++++
.../resources/rules/v1_2_0/logback-xml.groovy | 70 +++
.../rules/v1_2_0/nifi-properties.groovy | 35 ++
36 files changed, 3131 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/8ef4fddd/nifi-docs/src/main/asciidoc/administration-guide.adoc
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/administration-guide.adoc b/nifi-docs/src/main/asciidoc/administration-guide.adoc
index cc1197c..37283bd 100644
--- a/nifi-docs/src/main/asciidoc/administration-guide.adoc
+++ b/nifi-docs/src/main/asciidoc/administration-guide.adoc
@@ -1269,6 +1269,7 @@ and clustered environments. These utilities include:
* Notify -- The notification tool allows administrators to send bulletins to the NiFi UI using the command line.
* Node Manager -- The node manager tool allows administrators to perform a status check on a node as well as to connect, disconnect, or remove nodes that are part of a cluster.
+* File Manager -- The file manager tool allows administrators to backup, install or restore a NiFi installation from backup.
The admin toolkit is bundled with the nifi-toolkit and can be executed with scripts found in the _bin_ folder.
@@ -1402,6 +1403,84 @@ cluster if restarted and the flow for the cluster has not changed. If the flow w
the removed node should be deleted before restarting the node to allow it to obtain the cluster flow (otherwise
an uninheritable flow file exception may occur).
+=== File Manager
+
+The File Manager utility allows system administrators to take a backup of an existing NiFi installation, install a new version of NiFi
+in a designated location (while migrating any previous configuration settings) or restore an installation from a previous backup.
+File Manager supports NiFi version 1.0.0 and higher and is available in 'file-manager.bat' file for use on Windows machines.
+
+To show help:
+
+ file-manager.sh -h
+
+The following are available options:
+
+* `-o,--operation <arg>` File operation (install | backup | restore)
+* `-b,--backupDir <arg>` Backup NiFi Directory (used with backup or restore operation)
+* `-c,--nifiCurrentDir <arg>` Current NiFi Installation Directory (used optionally with install or restore operation)
+* `-d,--nifiInstallDir <arg>` NiFi Installation Directory (used with install or restore operation)
+* `-i,--installFile <arg>` NiFi Install File (used with install operation)
+* `-r,--nifiRollbackDir <arg>` NiFi Installation Directory (used with install or restore operation)
+* `-t,--bootstrapConf <arg>` Current NiFi Bootstrap Configuration File (used optionally)
+* `-m,--moveRepositories` Allow repositories to be moved to new/restored nifi directory from existing installation, if available (used optionally with install or restore operation)
+* `-x,--overwriteConfigs` Overwrite existing configuration directory with upgrade changes (used optionally with install or restore operation)
+* `-v,--verbose` Verbose messaging (optional)
+* `-h,--help` Print help info (optional)
+
+Example usage on Linux:
+
+ # backup NiFi installation
+ # option -t may be provided to ensure backup of external boostrap.conf file
+ ./file-manager.sh
+ -o backup
+ –b /tmp/nifi_bak
+ –c /usr/nifi_old
+ -v
+
+ # install NiFi using compressed tar file into /usr/nifi directory (should install as /usr/nifi/nifi-1.3.0).
+ # migrate existing configurations with location determined by external bootstrap.conf and move over repositories from nifi_old
+ # options -t and -c should both be provided if migration of configurations, state and repositories are required
+ ./file-manager.sh
+ -o install
+ –i nifi-1.3.0.tar.gz
+ –d /usr/nifi
+ –c /usr/nifi/nifi_old
+ -t /usr/nifi/old_conf/bootstrap.conf
+ -v
+ -m
+
+ # restore NiFi installation from backup directory and move back repositories
+ # option -t may be provided to ensure bootstrap.conf is restored to the file path provided, otherwise it is placed in the
+ # default directory under the rollback path (e.g. /usr/nifi_old/conf)
+ ./file-manager.sh
+ -o restore
+ –b /tmp/nifi_bak
+ –r /usr/nifi_old
+ –c /usr/nifi
+ -m
+ -v
+
+=== Expected Behavior
+
+Backup:
+
+During the backup operation a backup directory is created in a designated location for an existing NiFi installation. Backups will capture all critical files
+(including any internal or external configurations, libraries, scripts and documents) however it excludes backing up repositories and logs due to potential size.
+If configuration/library files are external from the existing installation folder the backup operation will capture those as well.
+
+Install:
+
+During the install operation File Manager will perform installation using the designated NiFi binary file (either tar.gz or zip file)
+to create a new installation or migrate an existing nifi installation to a new one. Installation can optionally move repositories (if located within the configuration
+folder of the current installation) to the new installation as well as migrate configuration files to the newer installation.
+
+Restore:
+
+The restore operation allows an existing installation to revert back to a previous installation. Using an existing backup directory (created from the backup operation)
+the FileManager utility will restore libraries, scripts and documents as well as revert to previous configurations. NOTE: If repositories were changed due to the installation
+of a newer version of NiFi these may no longer be compatible during restore. In that scenario exclude the -m option to ensure new repositories will be created or, if repositories
+live outside of the NiFi directory, remove them so they can be recreated on startup after restore.
+
[[clustering]]
Clustering Configuration
http://git-wip-us.apache.org/repos/asf/nifi/blob/8ef4fddd/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/AbstractAdminTool.groovy
----------------------------------------------------------------------
diff --git a/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/AbstractAdminTool.groovy b/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/AbstractAdminTool.groovy
index 79c5ef1..9d1eeae 100644
--- a/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/AbstractAdminTool.groovy
+++ b/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/AbstractAdminTool.groovy
@@ -19,11 +19,7 @@ package org.apache.nifi.toolkit.admin
import org.apache.nifi.toolkit.admin.util.AdminUtil
import org.apache.commons.cli.HelpFormatter
import org.apache.commons.cli.Options
-import org.apache.commons.lang3.SystemUtils
-import org.apache.nifi.toolkit.admin.util.Version
-import org.apache.nifi.util.StringUtils
import org.slf4j.Logger
-import java.nio.file.Path
import java.nio.file.Paths
public abstract class AbstractAdminTool {
@@ -65,45 +61,11 @@ public abstract class AbstractAdminTool {
protected abstract Logger getLogger()
- Properties getBootstrapConf(Path bootstrapConfFileName) {
- Properties bootstrapProperties = new Properties()
- File bootstrapConf = bootstrapConfFileName.toFile()
- bootstrapProperties.load(new FileInputStream(bootstrapConf))
- return bootstrapProperties
- }
-
- String getRelativeDirectory(String directory, String rootDirectory) {
- if (directory.startsWith("./")) {
- final String directoryUpdated = SystemUtils.IS_OS_WINDOWS ? File.separator + directory.substring(2,directory.length()) : directory.substring(1,directory.length())
- rootDirectory + directoryUpdated
- } else {
- directory
- }
- }
-
- Boolean supportedNiFiMinimumVersion(final String nifiConfDirName, final String nifiLibDirName, final String supportedMinimumVersion){
- final File nifiConfDir = new File(nifiConfDirName)
- final File nifiLibDir = new File (nifiLibDirName)
- final String versionStr = AdminUtil.getNiFiVersion(nifiConfDir,nifiLibDir)
-
- if(!StringUtils.isEmpty(versionStr)){
- Version version = new Version(versionStr.replace("-","."),".")
- Version minVersion = new Version(supportedMinimumVersion,".")
- Version.VERSION_COMPARATOR.compare(version,minVersion) >= 0
- }else{
- return false
- }
-
- }
-
- Boolean supportedNiFiMinimumVersion(final String nifiCurrentDirName, final String supportedMinimumVersion){
- final String bootstrapConfFileName = Paths.get(nifiCurrentDirName,"conf","bootstrap.conf").toString()
- final File bootstrapConf = new File(bootstrapConfFileName)
- final Properties bootstrapProperties = getBootstrapConf(Paths.get(bootstrapConfFileName))
- final String parentPathName = bootstrapConf.getCanonicalFile().getParentFile().getParentFile().getCanonicalPath()
- final String nifiConfDir = getRelativeDirectory(bootstrapProperties.getProperty("conf.dir"),parentPathName)
- final String nifiLibDir = getRelativeDirectory(bootstrapProperties.getProperty("lib.dir"),parentPathName)
- return supportedNiFiMinimumVersion(nifiConfDir,nifiLibDir,supportedMinimumVersion)
+ Boolean supportedNiFiMinimumVersion(final String nifiCurrentDirName, final String bootstrapConfFileName, final String supportedMinimumVersion){
+ final Properties bootstrapProperties = AdminUtil.getBootstrapConf(Paths.get(bootstrapConfFileName))
+ final String nifiConfDir = AdminUtil.getRelativeDirectory(bootstrapProperties.getProperty("conf.dir"),nifiCurrentDirName)
+ final String nifiLibDir = AdminUtil.getRelativeDirectory(bootstrapProperties.getProperty("lib.dir"),nifiCurrentDirName)
+ return AdminUtil.supportedNiFiMinimumVersion(nifiConfDir,nifiLibDir,supportedMinimumVersion)
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/8ef4fddd/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/configmigrator/ConfigMigrator.groovy
----------------------------------------------------------------------
diff --git a/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/configmigrator/ConfigMigrator.groovy b/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/configmigrator/ConfigMigrator.groovy
new file mode 100644
index 0000000..5a1deee
--- /dev/null
+++ b/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/configmigrator/ConfigMigrator.groovy
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.toolkit.admin.configmigrator
+
+import com.google.common.collect.Lists
+import com.google.common.io.Files
+import org.apache.nifi.toolkit.admin.util.AdminUtil
+import org.apache.commons.cli.ParseException
+import org.apache.commons.io.FileUtils
+import org.apache.nifi.toolkit.admin.util.Version
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
+
+import java.nio.file.Path
+import java.nio.file.Paths
+
+public class ConfigMigrator {
+
+ private final static String SUPPORTED_MINIMUM_VERSION = '1.0.0'
+ private final String RULES_DIR = getRulesDirectory()
+ private final Boolean overwrite
+ protected Logger logger = LoggerFactory.getLogger(ConfigMigrator)
+ protected final Boolean isVerbose
+
+ public ConfigMigrator(Boolean verbose, Boolean overwrite) {
+ this.overwrite = overwrite
+ this.isVerbose = verbose
+ }
+
+ String getRulesDirectory() {
+ final ClassLoader cl = this.getClass().getClassLoader()
+ cl.getResource('rules').path.replaceAll('%20',' ')
+ }
+
+ List<String> getRulesDirectoryName(final String currentVersion, final String upgradeVersion) {
+ Version current = new Version(currentVersion.take(5).toString(),'.')
+ Version upgrade = new Version(upgradeVersion.take(5).toString(),'.')
+ File rulesDir = new File(rulesDirectory)
+ List<File> rules = Lists.newArrayList(rulesDir.listFiles())
+ List<Version> versions = rules.collect { new Version(it.name[1..-1],'_')}
+ versions.sort(Version.VERSION_COMPARATOR)
+ List<Version> matches = versions.findAll { Version.VERSION_COMPARATOR.compare(it,upgrade) <= 0 && Version.VERSION_COMPARATOR.compare(it,current) == 1}
+
+ if(matches.isEmpty()){
+ null
+ }else{
+ matches.sort(Version.VERSION_COMPARATOR)
+ List<String> directoryNames = []
+ matches.each { directoryNames.add(RULES_DIR + File.separator + 'v' + it.toString()) }
+ directoryNames
+ }
+ }
+
+ Boolean supportedVersion(final File script, final String currentVersion) {
+ final Class ruleClass = new GroovyClassLoader(getClass().getClassLoader()).parseClass(script)
+ final GroovyObject ruleObject = (GroovyObject) ruleClass.newInstance()
+ ruleObject.invokeMethod('supportedVersion', [currentVersion])
+ }
+
+ byte[] migrateContent(final File script, final byte[] content, final byte[] upgradeContent) {
+ final Class ruleClass = new GroovyClassLoader(getClass().getClassLoader()).parseClass(script)
+ final GroovyObject ruleObject = (GroovyObject) ruleClass.newInstance()
+ ruleObject.invokeMethod('migrate', [content, upgradeContent])
+ }
+
+ String getScriptRuleName(final String fileName) {
+ fileName.replace('.', '-') + '.groovy'
+ }
+
+ File getUpgradeFile(final File upgradeDir, final String fileName){
+
+ final File[] upgradeFiles = upgradeDir.listFiles({dir, name -> name == fileName }as FilenameFilter)
+ upgradeFiles.size() == 1 ? upgradeFiles[0] : new File(upgradeDir.path + File.separator + fileName)
+ }
+
+ void migrate(final File nifiConfDir, final File nifiLibDir, final File nifiUpgradeConfigDir, final File nifiUpgradeLibDir, final File boostrapConf, final String nifiCurrentDir) {
+
+ final String nifiCurrentVersion = AdminUtil.getNiFiVersion(nifiConfDir,nifiLibDir)
+ final String nifiUpgradeVersion = AdminUtil.getNiFiVersion(nifiUpgradeConfigDir,nifiUpgradeLibDir)
+
+ if (nifiCurrentVersion == null) {
+ throw new IllegalArgumentException('Could not determine current nifi version')
+ }
+
+ if (nifiUpgradeVersion == null) {
+ throw new IllegalArgumentException('Could not determine upgrade nifi version')
+ }
+
+ final List<File> nifiConfigFiles = Lists.newArrayList(nifiConfDir.listFiles())
+ nifiConfigFiles.add(boostrapConf)
+
+ //obtain the rule directories sorted for each version in between current version and upgrade
+ final List<String> ruleDirs = getRulesDirectoryName(nifiCurrentVersion,nifiUpgradeVersion)
+
+ //iterate through all rule scripts in each directory and apply to file
+ if(ruleDirs != null) {
+
+ nifiConfigFiles.each { file ->
+
+ if (!file.isDirectory()) {
+
+ final String scriptName = getScriptRuleName(file.getName())
+ def byte[] content = file.bytes
+ def upgradeFile = getUpgradeFile(nifiUpgradeConfigDir, file.name)
+
+ ruleDirs.each { ruleDir ->
+
+ final File script = new File(ruleDir + File.separator + scriptName)
+
+ if (script.exists() && supportedVersion(script, nifiCurrentVersion)) {
+
+ if (isVerbose) {
+ logger.info('Applying rules to {} from directory {} ', file.name, ruleDir)
+ }
+
+ content = migrateContent(script, content, upgradeFile.exists() ? upgradeFile.bytes : new byte[0])
+
+ } else {
+ if (isVerbose) {
+ logger.info('No migration rule exists in {} for file {}. ',ruleDir,file.getName())
+ }
+ }
+
+ }
+
+ //if file is external from current installation and overwrite is allowed then write to external location
+ //otherwise write to new/upgraded location
+ if (file.parentFile.parentFile!= null && file.parentFile.parentFile.toString() != nifiCurrentDir && this.overwrite) {
+ Files.write(content, file)
+ } else {
+ Files.write(content, upgradeFile)
+ }
+
+ }else{
+ if(!this.overwrite){
+ FileUtils.copyDirectoryToDirectory(file, nifiUpgradeConfigDir)
+ }
+ }
+ }
+
+ }else{
+ if(isVerbose) {
+ logger.info('No upgrade rules are required for these configurations.')
+ }
+ if(!this.overwrite){
+
+ if(isVerbose) {
+ logger.info('Copying configurations over to upgrade directory')
+ }
+
+ nifiConfigFiles.each { file ->
+ if(file.isDirectory()){
+ FileUtils.copyDirectoryToDirectory(file, nifiUpgradeConfigDir)
+ }else {
+ FileUtils.copyFileToDirectory(file, nifiUpgradeConfigDir)
+ }
+ }
+ }
+ }
+
+ }
+
+ public void run(final String nifiCurrentDir, final String bootstrapConfFile, final String nifiUpgDirString) throws ParseException, IllegalArgumentException {
+
+ Path bootstrapConfPath = Paths.get(bootstrapConfFile)
+ File bootstrapConf = Paths.get(bootstrapConfFile).toFile()
+
+ if (!bootstrapConf.exists()) {
+ throw new IllegalArgumentException('NiFi Bootstrap File provided does not exist: ' + bootstrapConfFile)
+ }
+
+ AdminUtil.with{
+
+ Properties bootstrapProperties = getBootstrapConf(bootstrapConfPath)
+ File nifiConfDir = new File(getRelativeDirectory(bootstrapProperties.getProperty('conf.dir'), nifiCurrentDir))
+ File nifiLibDir = new File(getRelativeDirectory(bootstrapProperties.getProperty('lib.dir'), nifiCurrentDir))
+ final File nifiUpgradeConfDir = Paths.get(nifiUpgDirString,'conf').toFile()
+ final File nifiUpgradeLibDir = Paths.get(nifiUpgDirString,'lib').toFile()
+
+ if(supportedNiFiMinimumVersion(nifiConfDir.canonicalPath, nifiLibDir.canonicalPath, SUPPORTED_MINIMUM_VERSION) &&
+ supportedNiFiMinimumVersion(nifiUpgradeConfDir.canonicalPath, nifiUpgradeLibDir.canonicalPath, SUPPORTED_MINIMUM_VERSION)) {
+
+ if (!nifiConfDir.exists() || !nifiConfDir.isDirectory()) {
+ throw new IllegalArgumentException('NiFi Configuration Directory provided is not valid: ' + nifiConfDir.absolutePath)
+ }
+
+ if (!nifiUpgradeConfDir.exists() || !nifiUpgradeConfDir.isDirectory()) {
+ throw new IllegalArgumentException('Upgrade Configuration Directory provided is not valid: ' + nifiUpgradeConfDir)
+ }
+
+ if (isVerbose) {
+ logger.info('Migrating configurations from {} to {}', nifiConfDir.absolutePath, nifiUpgradeConfDir.absolutePath)
+ }
+
+ migrate(nifiConfDir,nifiLibDir,nifiUpgradeConfDir,nifiUpgradeLibDir,bootstrapConf,nifiCurrentDir)
+
+ if (isVerbose) {
+ logger.info('Migration completed.')
+ }
+
+ }else{
+ throw new UnsupportedOperationException('Config Migration Tool only supports NiFi version 1.0.0 and above')
+ }
+ }
+ }
+
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/8ef4fddd/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/configmigrator/rules/ConfigMigrationRule.groovy
----------------------------------------------------------------------
diff --git a/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/configmigrator/rules/ConfigMigrationRule.groovy b/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/configmigrator/rules/ConfigMigrationRule.groovy
new file mode 100644
index 0000000..cad7baf
--- /dev/null
+++ b/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/configmigrator/rules/ConfigMigrationRule.groovy
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.toolkit.admin.configmigrator.rules
+
+interface ConfigMigrationRule {
+
+ Boolean supportedVersion(String version)
+ byte[] migrate(byte[] oldContent, byte[] upgradeContent)
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/8ef4fddd/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/configmigrator/rules/GenericMigrationRule.groovy
----------------------------------------------------------------------
diff --git a/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/configmigrator/rules/GenericMigrationRule.groovy b/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/configmigrator/rules/GenericMigrationRule.groovy
new file mode 100644
index 0000000..1089467
--- /dev/null
+++ b/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/configmigrator/rules/GenericMigrationRule.groovy
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.toolkit.admin.configmigrator.rules
+
+abstract class GenericMigrationRule implements ConfigMigrationRule {
+
+ @Override
+ Boolean supportedVersion(String version) {
+ return true
+ }
+
+ static String getContent(String[] args){
+ File file = new File(args[0])
+ return file.text
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/8ef4fddd/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/configmigrator/rules/PropertyMigrationRule.groovy
----------------------------------------------------------------------
diff --git a/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/configmigrator/rules/PropertyMigrationRule.groovy b/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/configmigrator/rules/PropertyMigrationRule.groovy
new file mode 100644
index 0000000..6936657
--- /dev/null
+++ b/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/configmigrator/rules/PropertyMigrationRule.groovy
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.toolkit.admin.configmigrator.rules
+
+/**
+ * PropertyMigrationRule supports the migration of older key/value (property) based files to new versions
+ * by adding any new variables to the older file while maintaining existing configurations. Classes that extend this rule can also filter
+ * any properties that are no longer needed in new installations
+ */
+abstract class PropertyMigrationRule extends GenericMigrationRule{
+
+ private final static String LICENSE_COMMENTS = "# Licensed to the Apache Software Foundation (ASF) under one or more\n" +
+ "# contributor license agreements. See the NOTICE file distributed with\n" +
+ "# this work for additional information regarding copyright ownership.\n" +
+ "# The ASF licenses this file to You under the Apache License, Version 2.0\n" +
+ "# (the \"License\"); you may not use this file except in compliance with\n" +
+ "# the License. You may obtain a copy of the License at\n" +
+ "#\n" +
+ "# http://www.apache.org/licenses/LICENSE-2.0\n" +
+ "#\n" +
+ "# Unless required by applicable law or agreed to in writing, software\n" +
+ "# distributed under the License is distributed on an \"AS IS\" BASIS,\n" +
+ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n" +
+ "# See the License for the specific language governing permissions and\n" +
+ "# limitations under the License."
+
+ /**
+ *
+ * @param oldContent
+ * @param upgradeContent
+ * @return
+ */
+ @Override
+ byte[] migrate(byte[] oldContent, byte[] upgradeContent) {
+ def properties = new SortedProperties()
+ def upgradeProperties = new SortedProperties()
+
+ properties.load(new ByteArrayInputStream(oldContent))
+ upgradeProperties.load(new ByteArrayInputStream(upgradeContent))
+
+ Enumeration keys = (Enumeration<Object>) properties.keys()
+
+ keys.each { key ->
+ if(keyAllowed(key)) {
+ upgradeProperties.put(key, properties.get(key))
+ }
+ }
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream()
+ upgradeProperties.store(baos,LICENSE_COMMENTS)
+ baos.toByteArray()
+ }
+
+ /**
+ * Return if a key should be included in the set of properties being migrated
+ * @param key
+ * @return boolean
+ */
+ abstract boolean keyAllowed(Object key);
+
+ class SortedProperties extends Properties{
+
+ @Override
+ public Enumeration<Object> keys() {
+
+ Enumeration<Object> keysEnum = super.keys();
+ Vector<Object> keyList = new Vector<Object>();
+ keysEnum.each { e -> keyList.add(e)}
+
+ Collections.sort(keyList,new Comparator<Object>() {
+ @Override
+ int compare(Object o1, Object o2) {
+ o1.toString().compareTo(o2.toString())
+ }
+ })
+
+ keyList.elements()
+ }
+
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/8ef4fddd/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/configmigrator/rules/XmlMigrationRule.groovy
----------------------------------------------------------------------
diff --git a/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/configmigrator/rules/XmlMigrationRule.groovy b/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/configmigrator/rules/XmlMigrationRule.groovy
new file mode 100644
index 0000000..de36000
--- /dev/null
+++ b/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/configmigrator/rules/XmlMigrationRule.groovy
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.toolkit.admin.configmigrator.rules
+
+
+/**
+ * XmlMigrationRule assists classes in migrating existing xml configurations to newer versions
+ * by converting incoming content to Xml nodes which can then be navigated and edited as required.
+ */
+
+abstract class XmlMigrationRule extends GenericMigrationRule{
+
+ @Override
+ byte[] migrate(byte[] oldContent, byte[] upgradeContent) {
+ def oldBais = new ByteArrayInputStream(oldContent)
+ def newBais = new ByteArrayInputStream(upgradeContent)
+ migrateXml(new XmlParser().parse(oldBais), new XmlParser().parse(newBais))
+ }
+
+ abstract byte[] migrateXml(Node oldXmlContent, Node newXmlContent)
+
+ protected byte[] convertToByteArray(Node xml){
+ def writer = new StringWriter()
+ def nodePrinter = new XmlNodePrinter(new PrintWriter(writer))
+ nodePrinter.setPreserveWhitespace(true)
+ nodePrinter.print(xml)
+ return writer.toString().bytes
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/8ef4fddd/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/filemanager/FileManagerTool.groovy
----------------------------------------------------------------------
diff --git a/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/filemanager/FileManagerTool.groovy b/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/filemanager/FileManagerTool.groovy
new file mode 100644
index 0000000..ecead1a
--- /dev/null
+++ b/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/filemanager/FileManagerTool.groovy
@@ -0,0 +1,599 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.toolkit.admin.filemanager
+
+import com.google.common.collect.Sets
+import org.apache.commons.cli.CommandLine
+import org.apache.commons.cli.DefaultParser
+import org.apache.commons.cli.Option
+import org.apache.commons.cli.Options
+import org.apache.commons.cli.ParseException
+import org.apache.commons.compress.archivers.ArchiveEntry
+import org.apache.commons.compress.archivers.ArchiveInputStream
+import org.apache.commons.compress.archivers.ArchiveStreamFactory
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry
+import org.apache.commons.compress.archivers.zip.ZipFile
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream
+import org.apache.commons.compress.utils.IOUtils
+import org.apache.commons.io.FileUtils
+import org.apache.commons.io.FilenameUtils
+import org.apache.commons.lang3.SystemUtils
+import org.apache.nifi.toolkit.admin.AbstractAdminTool
+import org.apache.nifi.toolkit.admin.configmigrator.ConfigMigrator
+import org.apache.nifi.toolkit.admin.util.AdminUtil
+import org.apache.nifi.util.StringUtils
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
+
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.Paths
+import java.nio.file.attribute.PosixFilePermission
+
+public class FileManagerTool extends AbstractAdminTool{
+
+ private static final String DEFAULT_DESCRIPTION = 'This tool is used to perform backup, install and restore activities for a NiFi node. '
+ private static final String HELP_ARG = 'help'
+ private static final String VERBOSE_ARG = 'verbose'
+ private static final String OPERATION = 'operation'
+ private static final String NIFI_CURRENT_DIR = 'nifiCurrentDir'
+ private static final String NIFI_INSTALL_DIR = 'nifiInstallDir'
+ private static final String NIFI_ROLLBACK_DIR = 'nifiRollbackDir'
+ private static final String BACKUP_DIR = 'backupDir'
+ private static final String INSTALL_FILE = 'installFile'
+ private static final String MOVE_REPOSITORIES = 'moveRepositories'
+ private static final String OVERWRITE_CONFIGS = 'overwriteConfigs'
+ private static final String BOOTSTRAP_CONF = 'bootstrapConf'
+ private boolean moveRepositories = false
+ private final static String SUPPORTED_MINIMUM_VERSION = '1.0.0'
+ private static final List<PosixFilePermission> POSIX_PERMISSIONS =
+ [PosixFilePermission.OTHERS_EXECUTE,
+ PosixFilePermission.OTHERS_WRITE,
+ PosixFilePermission.OTHERS_READ,
+ PosixFilePermission.GROUP_EXECUTE,
+ PosixFilePermission.GROUP_WRITE,
+ PosixFilePermission.GROUP_READ,
+ PosixFilePermission.OWNER_EXECUTE,
+ PosixFilePermission.OWNER_WRITE,
+ PosixFilePermission.OWNER_READ]
+
+
+ FileManagerTool() {
+ header = buildHeader(DEFAULT_DESCRIPTION)
+ setup()
+ }
+
+ @Override
+ protected Logger getLogger() {
+ LoggerFactory.getLogger(FileManagerTool)
+ }
+
+ protected Options getOptions(){
+ final Options options = new Options()
+ options.addOption(Option.builder('o').longOpt(OPERATION).hasArg().desc('File operation (install | backup | restore)').build())
+ options.addOption(Option.builder('b').longOpt(BACKUP_DIR).hasArg().desc('Backup NiFi Directory (used with backup or restore operation)').build())
+ options.addOption(Option.builder('c').longOpt(NIFI_CURRENT_DIR).hasArg().desc('Current NiFi Installation Directory (used optionally with install or restore operation)').build())
+ options.addOption(Option.builder('d').longOpt(NIFI_INSTALL_DIR).hasArg().desc('NiFi Installation Directory (used with install or restore operation)').build())
+ options.addOption(Option.builder('i').longOpt(INSTALL_FILE).hasArg().desc('NiFi Install File').build())
+ options.addOption(Option.builder('r').longOpt(NIFI_ROLLBACK_DIR).hasArg().desc('NiFi Installation Directory (used with install or restore operation)').build())
+ options.addOption(Option.builder('t').longOpt(BOOTSTRAP_CONF).hasArg().desc('Current NiFi Bootstrap Configuration File (optional)').build())
+ options.addOption(Option.builder('m').longOpt(MOVE_REPOSITORIES).desc('Allow repositories to be moved to new/restored nifi directory from existing installation, if available (used optionally with install or restore operation)').build())
+ options.addOption(Option.builder('x').longOpt(OVERWRITE_CONFIGS).desc('Overwrite existing configuration directory with upgrade changes (used optionally with install or restore operation)').build())
+ options.addOption(Option.builder('h').longOpt(HELP_ARG).desc('Print help info (optional)').build())
+ options.addOption(Option.builder('v').longOpt(VERBOSE_ARG).desc('Set mode to verbose (optional, default is false)').build())
+
+ options
+ }
+
+ Set<PosixFilePermission> fromMode(final long mode) {
+
+ Set<PosixFilePermission> permissions = Sets.newHashSet()
+
+ POSIX_PERMISSIONS.eachWithIndex{
+ perm,index ->
+ if ((mode & (1 << index)) != 0) {
+ permissions.add(perm)
+ }
+ }
+
+ permissions
+ }
+
+ Properties getProperties(Path confFileName){
+ final Properties properties = new Properties()
+ final File confFile = confFileName.toFile()
+ properties.load(new FileInputStream(confFile))
+ properties
+ }
+
+ boolean valid(File nifiDir){
+ if(nifiDir.isDirectory() && Files.exists(Paths.get(nifiDir.absolutePath,'bin','nifi.sh'))){
+ true
+ }else {
+ false
+ }
+ }
+
+ void move(final String srcDir, final String oldDir, final String newDir){
+
+ final String oldPathName = srcDir.startsWith('./') ? oldDir + File.separator + srcDir[2..-1] : oldDir + File.separator + srcDir
+ final String newPathName = srcDir.startsWith('./') ? newDir + File.separator + srcDir[2..-1] : newDir + File.separator + srcDir
+
+ final Path oldPath = Paths.get(oldPathName)
+ final Path newPath = Paths.get(newPathName)
+
+ if(Files.exists(oldPath)) {
+ Files.move(oldPath, newPath)
+ }
+
+ }
+
+ void moveRepository(final String dirName, final String installDirName){
+
+ if(isVerbose){
+ logger.info('Moving repositories from {} to {}. Please note that repositories may be upgraded during install and become incompatible with a previous version. ',dirName,installDirName)
+ }
+
+ final String bootstrapConfFileName = dirName + File.separator + 'conf' + File.separator + 'bootstrap.conf'
+ final Properties bootstrapProperties = getProperties(Paths.get(bootstrapConfFileName))
+ final String nifiPropertiesFile = AdminUtil.getRelativeDirectory(bootstrapProperties.getProperty('conf.dir'),dirName) + File.separator +'nifi.properties'
+ final Properties nifiProperties = getProperties(Paths.get(nifiPropertiesFile))
+ final String flowFileDirectory = nifiProperties.getProperty('nifi.flowfile.repository.directory')
+ final String contentRepositoryDir = nifiProperties.getProperty('nifi.content.repository.directory.default')
+ final String provenanceRepositoryDir = nifiProperties.getProperty('nifi.provenance.repository.directory.default')
+ final String databaseDirectory = nifiProperties.getProperty('nifi.database.directory')
+
+ if(flowFileDirectory.startsWith('./')){
+ if(isVerbose){
+ logger.info('Moving flowfile repo')
+ }
+ move(flowFileDirectory,dirName,installDirName)
+ }
+
+ if(contentRepositoryDir.startsWith('./')){
+ if(isVerbose){
+ logger.info('Moving content repo')
+ }
+ move(contentRepositoryDir,dirName,installDirName)
+ }
+
+ if(provenanceRepositoryDir.startsWith('./')){
+ if(isVerbose){
+ logger.info('Moving provenance repo')
+ }
+ move(provenanceRepositoryDir,dirName,installDirName)
+ }
+
+ if(databaseDirectory.startsWith('./')){
+ if(isVerbose){
+ logger.info('Moving database repo')
+ }
+ move(databaseDirectory,dirName,installDirName)
+ }
+ }
+
+ void copyState(final String currentNiFiDirName, final String installDirName){
+
+ File stateDir = Paths.get(currentNiFiDirName,'state').toFile()
+
+ if(stateDir.exists()){
+
+ if(Files.exists(Paths.get(installDirName,'state'))){
+ Files.delete(Paths.get(installDirName,'state'))
+ }
+
+ FileUtils.copyDirectoryToDirectory(stateDir, Paths.get(installDirName).toFile())
+ }
+
+ }
+
+ protected void setPosixPermissions(final ArchiveEntry entry, final File outputFile, final ZipFile zipFile){
+ int mode = 0
+
+ if (entry instanceof TarArchiveEntry) {
+ mode = ((TarArchiveEntry) entry).getMode()
+
+ }else if(entry instanceof ZipArchiveEntry && zipFile != null){
+ mode = zipFile.getEntry(entry.name).getUnixMode()
+ }
+
+ if(mode == 0){
+ mode = outputFile.isDirectory()? TarArchiveEntry.DEFAULT_DIR_MODE: TarArchiveEntry.DEFAULT_FILE_MODE
+ }
+
+ Set<PosixFilePermission> permissions = fromMode(mode)
+ if(permissions.size() > 0) {
+ Files.setPosixFilePermissions(outputFile.toPath(), fromMode(mode))
+ }
+
+ }
+
+ protected void setPosixPermissions(final File file,List<PosixFilePermission> permissions = []){
+
+ if (SystemUtils.IS_OS_WINDOWS) {
+ file?.setReadable(permissions.contains(PosixFilePermission.OWNER_READ))
+ file?.setWritable(permissions.contains(PosixFilePermission.OWNER_WRITE))
+ file?.setExecutable(permissions.contains(PosixFilePermission.OWNER_EXECUTE))
+ } else {
+ Files.setPosixFilePermissions(file?.toPath(), permissions as Set)
+ }
+
+ }
+
+ void backup(String backupNiFiDirName, String currentNiFiDirName, String bootstrapConfFileName){
+
+ if(isVerbose){
+ logger.info('Creating backup in directory {}. Please note that repositories are not included in backup operation.',backupNiFiDirName)
+ }
+
+ final File backupNiFiDir = new File(backupNiFiDirName)
+ final Properties bootstrapProperties = getProperties(Paths.get(bootstrapConfFileName))
+ final File confDir = new File(AdminUtil.getRelativeDirectory(bootstrapProperties.getProperty('conf.dir'),currentNiFiDirName))
+ final File libDir = new File(AdminUtil.getRelativeDirectory(bootstrapProperties.getProperty('lib.dir'),currentNiFiDirName))
+
+ if( backupNiFiDir.exists() && backupNiFiDir.isDirectory()){
+ backupNiFiDir.deleteDir()
+ }
+
+ backupNiFiDir.mkdirs()
+
+ Files.createDirectory(Paths.get(backupNiFiDirName,'bootstrap_files'))
+ FileUtils.copyFileToDirectory(Paths.get(bootstrapConfFileName).toFile(),Paths.get(backupNiFiDirName,'bootstrap_files').toFile())
+ FileUtils.copyDirectoryToDirectory(Paths.get(currentNiFiDirName,'lib','bootstrap').toFile(),Paths.get(backupNiFiDirName,'bootstrap_files').toFile())
+ Files.createDirectories(Paths.get(backupNiFiDirName,'conf'))
+ Files.createDirectories(Paths.get(backupNiFiDirName,'lib'))
+ FileUtils.copyDirectoryToDirectory(confDir,Paths.get(backupNiFiDirName).toFile())
+ FileUtils.copyDirectoryToDirectory(libDir,Paths.get(backupNiFiDirName).toFile())
+ FileUtils.copyDirectoryToDirectory(Paths.get(currentNiFiDirName,'bin').toFile(),new File(backupNiFiDirName))
+ FileUtils.copyDirectoryToDirectory(Paths.get(currentNiFiDirName,'docs').toFile(),new File(backupNiFiDirName))
+ FileUtils.copyFileToDirectory(Paths.get(currentNiFiDirName,'LICENSE').toFile(),new File(backupNiFiDirName))
+ FileUtils.copyFileToDirectory(Paths.get(currentNiFiDirName,'NOTICE').toFile(),new File(backupNiFiDirName))
+ FileUtils.copyFileToDirectory(Paths.get(currentNiFiDirName,'README').toFile(),new File(backupNiFiDirName))
+
+ if(isVerbose){
+ logger.info('Backup Complete')
+ }
+
+ }
+
+ void restore(String backupNiFiDirName, String rollbackNiFiDirName, String currentNiFiDirName, String bootstrapConfFileName){
+
+ if(isVerbose){
+ logger.info('Restoring to directory:' + rollbackNiFiDirName)
+ }
+
+ final File rollbackNiFiDir = new File(rollbackNiFiDirName)
+ final File rollbackNiFiLibDir = Paths.get(rollbackNiFiDirName,'lib').toFile()
+ final File rollbackNiFiConfDir = Paths.get(rollbackNiFiDirName,'conf').toFile()
+ final Properties bootstrapProperties = getProperties(Paths.get(backupNiFiDirName,'bootstrap_files','bootstrap.conf'))
+ final File confDir = new File(AdminUtil.getRelativeDirectory(bootstrapProperties.getProperty('conf.dir'),rollbackNiFiDirName))
+ final File libDir = new File(AdminUtil.getRelativeDirectory(bootstrapProperties.getProperty('lib.dir'),rollbackNiFiDirName))
+
+
+ if(!rollbackNiFiDir.isDirectory()){
+ rollbackNiFiDir.mkdirs()
+ }
+
+ if(!rollbackNiFiLibDir.isDirectory()){
+ rollbackNiFiLibDir.mkdirs()
+ }
+
+ if(!rollbackNiFiConfDir.isDirectory()){
+ rollbackNiFiConfDir.mkdirs()
+ }
+
+ if(!libDir.isDirectory()){
+ libDir.mkdirs()
+ }
+
+ if(!confDir.isDirectory()){
+ confDir.mkdirs()
+ }
+
+ FileUtils.copyFile(Paths.get(backupNiFiDirName,'bootstrap_files','bootstrap.conf').toFile(), new File(bootstrapConfFileName))
+ FileUtils.copyDirectoryToDirectory(Paths.get(backupNiFiDirName,'bootstrap_files','bootstrap').toFile(),Paths.get(rollbackNiFiDirName,'lib').toFile())
+ FileUtils.copyDirectoryToDirectory(Paths.get(backupNiFiDirName,'bin').toFile(),new File(rollbackNiFiDirName))
+ FileUtils.copyDirectoryToDirectory(Paths.get(backupNiFiDirName,'docs').toFile(),new File(rollbackNiFiDirName))
+ FileUtils.copyDirectory(Paths.get(backupNiFiDirName,'lib').toFile(),libDir)
+ FileUtils.copyDirectory(Paths.get(backupNiFiDirName,'conf').toFile(),confDir)
+ FileUtils.copyFileToDirectory(Paths.get(backupNiFiDirName,'LICENSE').toFile(),new File(rollbackNiFiDirName))
+ FileUtils.copyFileToDirectory(Paths.get(backupNiFiDirName,'NOTICE').toFile(),new File(rollbackNiFiDirName))
+ FileUtils.copyFileToDirectory(Paths.get(backupNiFiDirName,'README').toFile(),new File(rollbackNiFiDirName))
+
+ final File binDir = Paths.get(rollbackNiFiDirName,'bin').toFile()
+ binDir.listFiles().each { setPosixPermissions(it,[PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE,
+ PosixFilePermission.GROUP_READ, PosixFilePermission.GROUP_EXECUTE, PosixFilePermission.OTHERS_READ,
+ PosixFilePermission.OTHERS_EXECUTE]) }
+
+ if(currentNiFiDirName && moveRepositories) {
+ moveRepository(currentNiFiDirName, rollbackNiFiDirName)
+ }
+
+ if(isVerbose){
+ logger.info('Restore Completed.')
+ }
+
+ }
+
+ String extract(final File installFile, final File installDirName){
+
+ if(isVerbose){
+ logger.info('Beginning extraction using {} into installation directory {}',installFile.absolutePath,installDirName.absolutePath)
+ }
+
+ final String extension = FilenameUtils.getExtension(installFile.getName())
+ final InputStream fis = extension.equals('gz') ? new GzipCompressorInputStream(new FileInputStream(installFile)) : new FileInputStream(installFile)
+ final ArchiveInputStream inputStream = new ArchiveStreamFactory().createArchiveInputStream(new BufferedInputStream(fis))
+ final ZipFile zipFile = extension.equals('zip') ? new ZipFile(installFile) : null
+
+ ArchiveEntry entry = inputStream.nextEntry
+
+ if(entry != null){
+
+ String archiveRootDir = null
+
+ while(entry != null){
+
+ if(archiveRootDir == null & entry.name.toLowerCase().startsWith('nifi')){
+
+ archiveRootDir = entry.name.indexOf(File.separator) > -1 ? entry.name[0..entry.name.indexOf(File.separator)-1] : entry.name
+
+ if(isVerbose){
+ logger.info('Upgrade root directory: {}', archiveRootDir)
+ }
+
+ File archiveRootDirFile = Paths.get(installDirName.getAbsolutePath(),archiveRootDir).toFile()
+
+ if(archiveRootDirFile.exists()){
+ archiveRootDirFile.deleteDir()
+ }
+ archiveRootDirFile.mkdirs()
+ }
+
+ if(isVerbose){
+ logger.info('Extracting file: {} ',entry.name)
+ }
+
+ if(archiveRootDir && entry.name.startsWith(archiveRootDir)) {
+
+ final File outputFile = Paths.get(installDirName.getAbsolutePath(),entry.name).toFile()
+
+ if (entry.isDirectory()) {
+
+ if (!outputFile.exists()) {
+ if (!outputFile.mkdirs()) {
+ throw new IllegalStateException('Could not create directory :' + outputFile.getAbsolutePath())
+ }
+ }
+
+ } else {
+
+ File parentDirectory = outputFile.getParentFile()
+
+ if(!parentDirectory.exists()){
+ parentDirectory.mkdirs()
+ }
+
+ final OutputStream outputFileStream = new FileOutputStream(outputFile)
+ IOUtils.copy(inputStream, outputFileStream)
+ outputFileStream.close()
+ }
+
+ if(!SystemUtils.IS_OS_WINDOWS){
+ setPosixPermissions(entry,outputFile,zipFile)
+ }
+ }
+
+ entry = inputStream.nextEntry
+ }
+
+ return archiveRootDir
+
+ }else{
+ throw new RuntimeException('Attempting to extract installation file however it is empty: '+installFile.getName())
+ }
+
+ }
+
+ void install(final String installFileName, final String installDirName, final String currentNiFiDirName, final String bootstrapConfFileName, final Boolean overwriteConfigs){
+
+ final File installFile = new File(installFileName)
+
+ if(isVerbose){
+ logger.info('Beginning installation into directory:' + installDirName)
+ }
+
+ if(installFile.exists()){
+
+ final File installDir = new File(installDirName)
+
+ if(!installDir.exists()){
+ installDir.mkdirs()
+ }
+
+ final String installRootDirName = extract(installFile,installDir)
+ final File installRootDir = Paths.get(installDirName,installRootDirName).toFile()
+
+ if(valid(installRootDir)){
+
+ if(currentNiFiDirName && bootstrapConfFileName){
+ copyState(currentNiFiDirName,installRootDir.absolutePath)
+ if(moveRepositories) {
+ moveRepository(currentNiFiDirName,installRootDir.absolutePath)
+ }
+ final ConfigMigrator configMigrator = new ConfigMigrator(isVerbose,overwriteConfigs)
+ configMigrator.run(currentNiFiDirName,bootstrapConfFileName,installRootDir.canonicalPath)
+ }
+
+ }else{
+ throw new RuntimeException('Extract failed: Invalid NiFi Installation. Check the install path provided and retry.')
+ }
+
+ }else{
+ throw new RuntimeException('Installation file provided does not exist')
+ }
+
+ if(isVerbose){
+ logger.info('Installation Complete')
+ }
+
+ }
+
+ void parseInstall(final CommandLine commandLine){
+
+ if(commandLine.hasOption(MOVE_REPOSITORIES)){
+ this.moveRepositories = true
+ }
+
+ if(!commandLine.hasOption(INSTALL_FILE)){
+ throw new ParseException('Missing -i option')
+ } else if(!commandLine.hasOption(NIFI_INSTALL_DIR)){
+ throw new ParseException('Missing -d option')
+ } else if (!commandLine.hasOption(NIFI_CURRENT_DIR) && moveRepositories){
+ throw new ParseException('Missing -c option: Moving repositories requires current nifi directory')
+ }
+
+ final String installFileName = commandLine.getOptionValue(INSTALL_FILE)
+ final String nifiCurrentDirName = commandLine.getOptionValue(NIFI_CURRENT_DIR)
+ final String nifiInstallDirName = commandLine.getOptionValue(NIFI_INSTALL_DIR)
+ final Boolean overwriteConfigs = commandLine.hasOption(OVERWRITE_CONFIGS)
+ final String bootstrapConfFileName = commandLine.hasOption(BOOTSTRAP_CONF) ? commandLine.getOptionValue(BOOTSTRAP_CONF) : nifiCurrentDirName ?
+ Paths.get(nifiCurrentDirName,'conf','bootstrap.conf').toString() : null
+
+ if (Files.notExists(Paths.get(installFileName))) {
+ throw new ParseException('Missing installation file: ' + installFileName)
+ }
+
+ if (nifiCurrentDirName && Files.notExists(Paths.get(nifiCurrentDirName))) {
+ throw new ParseException('Current NiFi installation path does not exist: ' + nifiCurrentDirName)
+ }
+
+ if(nifiCurrentDirName && bootstrapConfFileName && !supportedNiFiMinimumVersion(nifiCurrentDirName, bootstrapConfFileName, SUPPORTED_MINIMUM_VERSION)) {
+ throw new UnsupportedOperationException('File Manager Tool only supports NiFi versions 1.0.0 or higher.')
+ }
+
+ install(installFileName, nifiInstallDirName, !nifiCurrentDirName ? null : Paths.get(nifiCurrentDirName).toFile().getCanonicalPath(), bootstrapConfFileName, overwriteConfigs)
+
+ }
+
+ void parseBackup(final CommandLine commandLine){
+
+ if(!commandLine.hasOption(BACKUP_DIR)){
+ throw new ParseException('Missing -b option')
+ } else if(!commandLine.hasOption(NIFI_CURRENT_DIR)){
+ throw new ParseException('Missing -c option')
+ }
+
+ final String backupDirName = commandLine.getOptionValue(BACKUP_DIR)
+ final String nifiCurrentDirName = commandLine.getOptionValue(NIFI_CURRENT_DIR)
+
+ if (Files.notExists(Paths.get(nifiCurrentDirName))) {
+ throw new ParseException('Current NiFi installation link does not exist: ' + nifiCurrentDirName)
+ }
+
+ final String bootstrapConfFileName = commandLine.hasOption(BOOTSTRAP_CONF) ? commandLine.getOptionValue(BOOTSTRAP_CONF) : Paths.get(nifiCurrentDirName,'conf','bootstrap.conf').toString()
+
+ if(supportedNiFiMinimumVersion(nifiCurrentDirName, bootstrapConfFileName, SUPPORTED_MINIMUM_VERSION)) {
+ backup(backupDirName, Paths.get(nifiCurrentDirName).toFile().getCanonicalPath(),bootstrapConfFileName)
+ }else{
+ throw new UnsupportedOperationException('File Manager Tool only supports NiFi versions 1.0.0 or higher.')
+ }
+
+ }
+
+ void parseRestore(final CommandLine commandLine){
+
+ if(commandLine.hasOption(MOVE_REPOSITORIES)){
+ this.moveRepositories = true
+ }
+
+ if(!commandLine.hasOption(BACKUP_DIR)) {
+ throw new ParseException('Missing -b option')
+ }else if(!commandLine.hasOption(NIFI_ROLLBACK_DIR)){
+ throw new ParseException('Missing -r option')
+ }else if (!commandLine.hasOption(NIFI_CURRENT_DIR) && moveRepositories){
+ throw new ParseException('Missing -c option: Moving repositories requires current nifi directory')
+ }
+
+ final String backupDirName = commandLine.getOptionValue(BACKUP_DIR)
+ final String nifiRollbackDirName = commandLine.getOptionValue(NIFI_ROLLBACK_DIR)
+ final String nifiCurrentDirName = commandLine.getOptionValue(NIFI_CURRENT_DIR)
+
+ if (Files.notExists(Paths.get(backupDirName)) || !Files.isDirectory(Paths.get(backupDirName))) {
+ throw new ParseException('Missing or invalid backup directory: ' + backupDirName)
+ }
+
+ if (nifiCurrentDirName && Files.notExists(Paths.get(nifiCurrentDirName))) {
+ throw new ParseException('Current NiFi installation path does not exist: ' + nifiCurrentDirName)
+ }
+
+ if(!supportedNiFiMinimumVersion(backupDirName, Paths.get(backupDirName,'bootstrap_files','bootstrap.conf').toString(), SUPPORTED_MINIMUM_VERSION)) {
+ throw new UnsupportedOperationException('File Manager Tool only supports NiFi versions 1.0.0 or higher.')
+ }
+
+ final String bootstrapConfFileName = commandLine.hasOption(BOOTSTRAP_CONF) ? commandLine.getOptionValue(BOOTSTRAP_CONF) : Paths.get(nifiRollbackDirName,'conf','bootstrap.conf').toString()
+ restore(backupDirName, nifiRollbackDirName, !nifiCurrentDirName ? null : Paths.get(nifiCurrentDirName).toFile().getCanonicalPath(), bootstrapConfFileName)
+
+ }
+
+ void parse(final String[] args) throws ParseException, IllegalArgumentException {
+
+ CommandLine commandLine = new DefaultParser().parse(options, args)
+
+ if (commandLine.hasOption(HELP_ARG)) {
+ printUsage(null)
+ } else if (commandLine.hasOption(OPERATION)) {
+
+ if(commandLine.hasOption(VERBOSE_ARG)){
+ this.isVerbose = true
+ }
+
+ String operation = commandLine.getOptionValue(OPERATION).toLowerCase()
+
+ if(operation.equals('install')){
+ parseInstall(commandLine)
+ }else if(operation.equals('backup')){
+ parseBackup(commandLine)
+ }else if(operation.equals('restore')){
+ parseRestore(commandLine)
+ }else{
+ throw new ParseException('Invalid operation value:' + operation)
+ }
+
+ }else{
+ throw new ParseException('Missing -o option')
+ }
+
+ }
+
+ public static void main(String[] args) {
+ FileManagerTool tool = new FileManagerTool()
+
+ try {
+ tool.parse(args)
+ } catch (Exception e) {
+ tool.printUsage(e.getLocalizedMessage())
+ System.exit(1)
+ }
+
+ System.exit(0)
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/8ef4fddd/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/nodemanager/NodeManagerTool.groovy
----------------------------------------------------------------------
diff --git a/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/nodemanager/NodeManagerTool.groovy b/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/nodemanager/NodeManagerTool.groovy
index e78adec..3290fc3 100644
--- a/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/nodemanager/NodeManagerTool.groovy
+++ b/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/nodemanager/NodeManagerTool.groovy
@@ -30,6 +30,7 @@ import org.apache.commons.cli.ParseException
import org.apache.nifi.properties.NiFiPropertiesLoader
import org.apache.nifi.toolkit.admin.client.ClientFactory
import org.apache.nifi.toolkit.admin.client.NiFiClientFactory
+import org.apache.nifi.toolkit.admin.util.AdminUtil
import org.apache.nifi.util.NiFiProperties
import org.apache.nifi.util.StringUtils
import org.apache.nifi.web.api.dto.NodeDTO
@@ -72,7 +73,7 @@ public class NodeManagerTool extends AbstractAdminTool {
@Override
protected Logger getLogger() {
- LoggerFactory.getLogger(NodeManagerTool.class)
+ LoggerFactory.getLogger(NodeManagerTool)
}
protected Options getOptions(){
@@ -168,31 +169,40 @@ public class NodeManagerTool extends AbstractAdminTool {
void disconnectNode(final Client client, NiFiProperties niFiProperties, List<String> activeUrls, final String proxyDN){
final ClusterEntity clusterEntity = NiFiClientUtil.getCluster(client, niFiProperties, activeUrls,proxyDN)
NodeDTO currentNode = getCurrentNode(clusterEntity,niFiProperties)
- for(String activeUrl: activeUrls) {
- try {
- final String url = activeUrl + NODE_ENDPOINT + File.separator + currentNode.nodeId
- updateNode(url, client, currentNode, STATUS.DISCONNECTING,proxyDN)
- return
- } catch (Exception ex){
- logger.warn("Could not connect to node on "+activeUrl+". Exception: "+ex.toString())
+ if(currentNode != null){
+ for(String activeUrl: activeUrls) {
+ try {
+ final String url = activeUrl + NODE_ENDPOINT + File.separator + currentNode.nodeId
+ updateNode(url, client, currentNode, STATUS.DISCONNECTING,proxyDN)
+ return
+ } catch (Exception ex){
+ logger.warn("Could not connect to node on "+activeUrl+". Exception: "+ex.toString())
+ }
}
+ throw new RuntimeException("Could not successfully complete request")
+ }else{
+ throw new RuntimeException("Current node could not be found in the cluster")
}
- throw new RuntimeException("Could not successfully complete request")
}
void connectNode(final Client client, NiFiProperties niFiProperties,List<String> activeUrls, final String proxyDN){
final ClusterEntity clusterEntity = NiFiClientUtil.getCluster(client, niFiProperties, activeUrls,proxyDN)
NodeDTO currentNode = getCurrentNode(clusterEntity,niFiProperties)
- for(String activeUrl: activeUrls) {
- try {
- final String url = activeUrl + NODE_ENDPOINT + File.separator + currentNode.nodeId
- updateNode(url, client, currentNode, STATUS.CONNECTING,proxyDN)
- return
- } catch (Exception ex){
- logger.warn("Could not connect to node on "+activeUrl+". Exception: "+ex.toString())
+
+ if(currentNode != null) {
+ for(String activeUrl: activeUrls) {
+ try {
+ final String url = activeUrl + NODE_ENDPOINT + File.separator + currentNode.nodeId
+ updateNode(url, client, currentNode, STATUS.CONNECTING,proxyDN)
+ return
+ } catch (Exception ex){
+ logger.warn("Could not connect to node on "+activeUrl+". Exception: "+ex.toString())
+ }
}
+ throw new RuntimeException("Could not successfully complete request")
+ }else{
+ throw new RuntimeException("Current node could not be found in the cluster")
}
- throw new RuntimeException("Could not successfully complete request")
}
void removeNode(final Client client, NiFiProperties niFiProperties, List<String> activeUrls, final String proxyDN){
@@ -250,15 +260,15 @@ public class NodeManagerTool extends AbstractAdminTool {
if(commandLine.hasOption(BOOTSTRAP_CONF) && commandLine.hasOption(NIFI_INSTALL_DIR) && commandLine.hasOption(OPERATION)) {
if(commandLine.hasOption(VERBOSE_ARG)){
- this.isVerbose = true;
+ this.isVerbose = true
}
final String bootstrapConfFileName = commandLine.getOptionValue(BOOTSTRAP_CONF)
final String proxyDN = commandLine.getOptionValue(PROXY_DN)
final File bootstrapConf = new File(bootstrapConfFileName)
- Properties bootstrapProperties = getBootstrapConf(Paths.get(bootstrapConfFileName))
- String nifiConfDir = getRelativeDirectory(bootstrapProperties.getProperty("conf.dir"), bootstrapConf.getCanonicalFile().getParentFile().getParentFile().getCanonicalPath())
- String nifiLibDir = getRelativeDirectory(bootstrapProperties.getProperty("lib.dir"), bootstrapConf.getCanonicalFile().getParentFile().getParentFile().getCanonicalPath())
+ Properties bootstrapProperties = AdminUtil.getBootstrapConf(Paths.get(bootstrapConfFileName))
+ String nifiConfDir = AdminUtil.getRelativeDirectory(bootstrapProperties.getProperty("conf.dir"), bootstrapConf.getCanonicalFile().getParentFile().getParentFile().getCanonicalPath())
+ String nifiLibDir = AdminUtil.getRelativeDirectory(bootstrapProperties.getProperty("lib.dir"), bootstrapConf.getCanonicalFile().getParentFile().getParentFile().getCanonicalPath())
String nifiPropertiesFileName = nifiConfDir + File.separator +"nifi.properties"
final String key = NiFiPropertiesLoader.extractKeyFromBootstrapFile(bootstrapConfFileName)
final NiFiProperties niFiProperties = NiFiPropertiesLoader.withKey(key).load(nifiPropertiesFileName)
@@ -270,7 +280,7 @@ public class NodeManagerTool extends AbstractAdminTool {
final String nifiInstallDir = commandLine.getOptionValue(NIFI_INSTALL_DIR)
- if(supportedNiFiMinimumVersion(nifiConfDir,nifiLibDir,SUPPORTED_MINIMUM_VERSION)){
+ if(AdminUtil.supportedNiFiMinimumVersion(nifiConfDir,nifiLibDir,SUPPORTED_MINIMUM_VERSION)){
final Client client = clientFactory.getClient(niFiProperties,nifiInstallDir)
@@ -335,8 +345,8 @@ public class NodeManagerTool extends AbstractAdminTool {
try{
tool.parse(clientFactory,args)
- } catch (ParseException | RuntimeException e ) {
- tool.printUsage(e.getLocalizedMessage());
+ } catch (Exception e ) {
+ tool.printUsage(e.getLocalizedMessage())
System.exit(1)
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/8ef4fddd/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/notify/NotificationTool.groovy
----------------------------------------------------------------------
diff --git a/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/notify/NotificationTool.groovy b/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/notify/NotificationTool.groovy
index 215aee8..0405dbf 100644
--- a/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/notify/NotificationTool.groovy
+++ b/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/notify/NotificationTool.groovy
@@ -31,6 +31,7 @@ import org.apache.nifi.properties.NiFiPropertiesLoader
import org.apache.nifi.toolkit.admin.AbstractAdminTool
import org.apache.nifi.toolkit.admin.client.ClientFactory
import org.apache.nifi.toolkit.admin.client.NiFiClientFactory
+import org.apache.nifi.toolkit.admin.util.AdminUtil
import org.apache.nifi.util.NiFiProperties
import org.apache.nifi.web.api.dto.BulletinDTO
import org.apache.nifi.web.api.entity.BulletinEntity
@@ -65,7 +66,7 @@ public class NotificationTool extends AbstractAdminTool {
@Override
protected Logger getLogger() {
- LoggerFactory.getLogger(NotificationTool.class)
+ LoggerFactory.getLogger(NotificationTool)
}
protected Options getOptions(){
@@ -139,22 +140,22 @@ public class NotificationTool extends AbstractAdminTool {
if(commandLine.hasOption(BOOTSTRAP_CONF) && commandLine.hasOption(NOTIFICATION_MESSAGE) && commandLine.hasOption(NIFI_INSTALL_DIR)) {
if(commandLine.hasOption(VERBOSE_ARG)){
- this.isVerbose = true;
+ this.isVerbose = true
}
final String bootstrapConfFileName = commandLine.getOptionValue(BOOTSTRAP_CONF)
final File bootstrapConf = new File(bootstrapConfFileName)
- final Properties bootstrapProperties = getBootstrapConf(Paths.get(bootstrapConfFileName))
+ final Properties bootstrapProperties = AdminUtil.getBootstrapConf(Paths.get(bootstrapConfFileName))
final String proxyDN = commandLine.getOptionValue(PROXY_DN)
final String parentPathName = bootstrapConf.getCanonicalFile().getParentFile().getParentFile().getCanonicalPath()
- final String nifiConfDir = getRelativeDirectory(bootstrapProperties.getProperty("conf.dir"),parentPathName)
- final String nifiLibDir = getRelativeDirectory(bootstrapProperties.getProperty("lib.dir"),parentPathName)
+ final String nifiConfDir = AdminUtil.getRelativeDirectory(bootstrapProperties.getProperty("conf.dir"),parentPathName)
+ final String nifiLibDir = AdminUtil.getRelativeDirectory(bootstrapProperties.getProperty("lib.dir"),parentPathName)
final String nifiPropertiesFileName = nifiConfDir + File.separator +"nifi.properties"
final String notificationMessage = commandLine.getOptionValue(NOTIFICATION_MESSAGE)
final String notificationLevel = commandLine.getOptionValue(NOTIFICATION_LEVEL)
final String nifiInstallDir = commandLine.getOptionValue(NIFI_INSTALL_DIR)
- if(supportedNiFiMinimumVersion(nifiConfDir, nifiLibDir, SUPPORTED_MINIMUM_VERSION)){
+ if(AdminUtil.supportedNiFiMinimumVersion(nifiConfDir, nifiLibDir, SUPPORTED_MINIMUM_VERSION)){
if(isVerbose){
logger.info("Attempting to connect with nifi using properties:", nifiPropertiesFileName)
}
@@ -185,8 +186,8 @@ public class NotificationTool extends AbstractAdminTool {
try{
tool.parse(clientFactory,args)
- } catch (ParseException | UnsupportedOperationException | RuntimeException e) {
- tool.printUsage(e.message);
+ } catch (Exception e) {
+ tool.printUsage(e.message)
System.exit(1)
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/8ef4fddd/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/util/AdminUtil.groovy
----------------------------------------------------------------------
diff --git a/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/util/AdminUtil.groovy b/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/util/AdminUtil.groovy
index 9dc0090..f69f218 100644
--- a/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/util/AdminUtil.groovy
+++ b/nifi-toolkit/nifi-toolkit-admin/src/main/groovy/org/apache/nifi/toolkit/admin/util/AdminUtil.groovy
@@ -19,6 +19,9 @@ package org.apache.nifi.toolkit.admin.util
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry
import org.apache.commons.compress.archivers.zip.ZipFile
import org.apache.commons.lang3.StringUtils
+import org.apache.commons.lang3.SystemUtils
+
+import java.nio.file.Path
class AdminUtil {
@@ -62,8 +65,46 @@ class AdminUtil {
if(StringUtils.isEmpty(nifiVersion)){
nifiVersion = getNiFiVersionFromNar(nifiLibDir)
}
- return nifiVersion.replace("-SNAPSHOT","")
+ nifiVersion.replace("-SNAPSHOT","")
+
+ }
+ public static Properties getBootstrapConf(Path bootstrapConfFileName) {
+ Properties bootstrapProperties = new Properties()
+ File bootstrapConf = bootstrapConfFileName.toFile()
+ bootstrapProperties.load(new FileInputStream(bootstrapConf))
+ bootstrapProperties
}
+ public static String getRelativeDirectory(String directory, String rootDirectory) {
+ if (directory.startsWith("./")) {
+ final String directoryUpdated = SystemUtils.IS_OS_WINDOWS ? File.separator + directory[2..-1] : directory[1..-1]
+ rootDirectory + directoryUpdated
+ } else {
+ directory
+ }
+ }
+
+ public static Boolean supportedNiFiMinimumVersion(final String nifiConfDirName, final String nifiLibDirName, final String supportedMinimumVersion){
+ final File nifiConfDir = new File(nifiConfDirName)
+ final File nifiLibDir = new File (nifiLibDirName)
+ final String versionStr = getNiFiVersion(nifiConfDir,nifiLibDir)
+
+ if(!org.apache.nifi.util.StringUtils.isEmpty(versionStr)){
+ Version version = new Version(versionStr.replace("-","."),".")
+ Version minVersion = new Version(supportedMinimumVersion,".")
+ Version.VERSION_COMPARATOR.compare(version,minVersion) >= 0
+ }else{
+ false
+ }
+ }
+
+ public static Boolean supportedVersion(String minimumVersion, String maximumVersion, String incomingVersion) {
+ Version version = new Version(incomingVersion,incomingVersion[1])
+ Version supportedMinimum = new Version(minimumVersion,minimumVersion[1])
+ Version supportedMaximum = new Version(maximumVersion,maximumVersion[1])
+ Version.VERSION_COMPARATOR.compare(version,supportedMinimum) >= 0 && Version.VERSION_COMPARATOR.compare(version,supportedMaximum) <= 0
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/8ef4fddd/nifi-toolkit/nifi-toolkit-admin/src/test/groovy/org/apache/nifi/toolkit/admin/configmigrator/ConfigMigratorSpec.groovy
----------------------------------------------------------------------
diff --git a/nifi-toolkit/nifi-toolkit-admin/src/test/groovy/org/apache/nifi/toolkit/admin/configmigrator/ConfigMigratorSpec.groovy b/nifi-toolkit/nifi-toolkit-admin/src/test/groovy/org/apache/nifi/toolkit/admin/configmigrator/ConfigMigratorSpec.groovy
new file mode 100644
index 0000000..fb82a22
--- /dev/null
+++ b/nifi-toolkit/nifi-toolkit-admin/src/test/groovy/org/apache/nifi/toolkit/admin/configmigrator/ConfigMigratorSpec.groovy
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.toolkit.admin.configmigrator
+
+import groovy.xml.XmlUtil
+import org.apache.commons.io.FileUtils
+import org.apache.commons.lang3.SystemUtils
+import org.junit.Rule
+import org.junit.contrib.java.lang.system.SystemOutRule
+import spock.lang.Specification
+import org.junit.contrib.java.lang.system.ExpectedSystemExit
+
+import java.nio.file.Files
+import java.nio.file.attribute.PosixFilePermission
+
+class ConfigMigratorSpec extends Specification{
+
+ @Rule
+ public final ExpectedSystemExit exit = ExpectedSystemExit.none()
+
+ @Rule
+ public final SystemOutRule systemOutRule = new SystemOutRule().enableLog()
+
+
+ def "get rules directory name"(){
+
+ setup:
+
+ def config = new ConfigMigrator(false,false)
+ def nifiVersion = "1.1.0"
+ def nifiUpgradeVersion = "1.3.0"
+
+ when:
+
+ def rulesDirs = config.getRulesDirectoryName(nifiVersion,nifiUpgradeVersion)
+
+ then:
+ rulesDirs.size() == 2
+ rulesDirs[0].endsWith("rules/v1_2_0")
+ rulesDirs[1].endsWith("rules/v1_3_0")
+
+ }
+
+ def "get script rule name"(){
+
+ setup:
+ def config = new ConfigMigrator(false,false)
+ def fileName = "flow.xml.gz"
+
+ when:
+
+ def script = config.getScriptRuleName(fileName)
+
+ then:
+
+ script == "flow-xml-gz.groovy"
+
+ }
+
+ def "parse argument and migrate property config successfully"(){
+
+ setup:
+
+ def File tmpDir = setupTmpDir()
+ def config = new ConfigMigrator(true,false)
+ def bootstrapFile = new File("src/test/resources/conf/bootstrap.conf")
+ def upgradeConfDir = new File("src/test/resources/upgrade")
+ def File workingFile = new File("target/tmp/upgrade")
+
+ if(workingFile.exists()) {
+ workingFile.delete()
+ }
+
+ FileUtils.copyDirectory(upgradeConfDir,workingFile)
+ def Properties updatedProperties = new Properties()
+ def Properties bootstrapProperties = new Properties()
+
+ when:
+
+ config.run("src/test/resources/",bootstrapFile.path,workingFile.path)
+ updatedProperties.load(new FileInputStream(workingFile.path + "/conf/nifi.properties"))
+ bootstrapProperties.load(new FileInputStream(workingFile.path + "/conf/bootstrap.conf"))
+
+ then:
+ updatedProperties.getProperty("nifi.cluster.node.protocol.port") == "8300"
+ bootstrapProperties.getProperty("java.arg.2") == "-Xms512m"
+ bootstrapProperties.getProperty("lib.dir") == "./lib"
+
+ cleanup:
+
+ tmpDir.deleteOnExit()
+
+ }
+
+ def "parse argument and move over configs due to no rules successfully"(){
+
+ setup:
+
+ def File tmpDir = setupTmpDir()
+ def config = new ConfigMigrator(true,false)
+ def bootstrapFile = new File("src/test/resources/conf/bootstrap.conf")
+ def upgradeConfDir = new File("src/test/resources/no_rules")
+ def File workingFile = new File("target/tmp/no_rules")
+
+ if(workingFile.exists()) {
+ workingFile.delete()
+ }
+
+ FileUtils.copyDirectory(upgradeConfDir,workingFile)
+ def Properties updatedProperties = new Properties()
+ def Properties bootstrapProperties = new Properties()
+
+ when:
+
+ config.run("src/test/resources/",bootstrapFile.path,workingFile.path)
+ updatedProperties.load(new FileInputStream(workingFile.path + "/conf/nifi.properties"))
+ bootstrapProperties.load(new FileInputStream(workingFile.path + "/conf/bootstrap.conf"))
+
+ then:
+ updatedProperties.getProperty("nifi.cluster.node.protocol.port") == "8300"
+ updatedProperties.getProperty("nifi.cluster.is.node") == "true"
+ bootstrapProperties.getProperty("java.arg.1")
+
+ cleanup:
+
+ tmpDir.deleteOnExit()
+
+ }
+
+ def "parse arguments and migrate property config successfully with override"(){
+
+ setup:
+
+ def File tmpDir = setupTmpDir()
+ def config = new ConfigMigrator(true,true)
+ def nifiConfDir = new File("src/test/resources/conf")
+ def nifiLibDir = new File("src/test/resources/lib")
+ def externalConfDir = new File("src/test/resources/external/conf")
+ def upgradeConfDir = new File("src/test/resources/upgrade")
+
+ def File workingFile = new File("target/tmp/conf")
+ def File workingLibFile = new File("target/tmp/lib")
+ def File externalWorkingFile = new File("target/tmp/external/conf")
+ def File upgradeWorkingFile = new File("target/tmp/upgrade")
+
+
+ if(workingFile.exists()) {
+ workingFile.delete()
+ }
+
+ if(externalWorkingFile.exists()){
+ externalWorkingFile.delete()
+ }
+
+ if(upgradeWorkingFile.exists()){
+ upgradeWorkingFile.delete()
+ }
+
+ FileUtils.copyDirectory(nifiConfDir,workingFile)
+ FileUtils.copyDirectory(nifiLibDir,workingLibFile)
+ FileUtils.copyDirectory(externalConfDir,externalWorkingFile)
+ FileUtils.copyDirectory(upgradeConfDir,upgradeWorkingFile)
+
+ def bootstrapFile = new File("target/tmp/external/conf/bootstrap.conf")
+ def Properties updatedNiFiProperties = new Properties()
+ def Properties updatedBootstrapProperties = new Properties()
+ def File updatedLoginProvidersFile
+ def xml
+
+ when:
+ config.run("target/tmp/external",bootstrapFile.path,upgradeWorkingFile.path)
+ updatedNiFiProperties.load(new FileInputStream(workingFile.path + "/nifi.properties"))
+ updatedBootstrapProperties.load(new FileInputStream(upgradeWorkingFile.path + "/conf/bootstrap.conf"))
+ updatedLoginProvidersFile = new File(workingFile.path + "/login-identity-providers.xml")
+ xml = new XmlSlurper().parse(updatedLoginProvidersFile)
+
+
+ then:
+ updatedNiFiProperties.getProperty("nifi.cluster.node.protocol.port") == "8300"
+ updatedBootstrapProperties.getProperty("java.arg.2") == "-Xms512m"
+ updatedBootstrapProperties.getProperty("lib.dir") == "./lib"
+ xml.depthFirst().findAll { it.name() == "fake"}.size() == 1
+
+ cleanup:
+
+ tmpDir.deleteOnExit()
+
+ }
+
+ def setFilePermissions(File file, List<PosixFilePermission> permissions = []) {
+ if (SystemUtils.IS_OS_WINDOWS) {
+ file?.setReadable(permissions.contains(PosixFilePermission.OWNER_READ))
+ file?.setWritable(permissions.contains(PosixFilePermission.OWNER_WRITE))
+ file?.setExecutable(permissions.contains(PosixFilePermission.OWNER_EXECUTE))
+ } else {
+ Files.setPosixFilePermissions(file?.toPath(), permissions as Set)
+ }
+ }
+
+ def setupTmpDir(String tmpDirPath = "target/tmp/") {
+ File tmpDir = new File(tmpDirPath)
+ tmpDir.mkdirs()
+ setFilePermissions(tmpDir, [PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE,
+ PosixFilePermission.GROUP_READ, PosixFilePermission.GROUP_WRITE, PosixFilePermission.GROUP_EXECUTE,
+ PosixFilePermission.OTHERS_READ, PosixFilePermission.OTHERS_WRITE, PosixFilePermission.OTHERS_EXECUTE])
+ tmpDir
+ }
+
+}