You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by oz...@apache.org on 2016/11/17 23:26:17 UTC
nifi git commit: NIFI-3059 Adds --ignore-source option to the
ZooKeeper Migrator to allow data read from a source zookeeper to be written
back to the same zookeeper Added unit test to test the --ignore-source option
Repository: nifi
Updated Branches:
refs/heads/master c4be80068 -> 06d7ecd32
NIFI-3059 Adds --ignore-source option to the ZooKeeper Migrator to allow data read from a source zookeeper to be written back to the same zookeeper
Added unit test to test the --ignore-source option
This closes #1242
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/06d7ecd3
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/06d7ecd3
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/06d7ecd3
Branch: refs/heads/master
Commit: 06d7ecd324ff8c352c512f3f4709f77786f46325
Parents: c4be800
Author: Jeff Storck <jt...@gmail.com>
Authored: Thu Nov 17 16:17:46 2016 -0500
Committer: Oleg Zhurakousky <ol...@suitcase.io>
Committed: Thu Nov 17 18:24:58 2016 -0500
----------------------------------------------------------------------
.../toolkit/zkmigrator/ZooKeeperMigrator.java | 4 +--
.../zkmigrator/ZooKeeperMigratorMain.java | 12 ++++++---
.../zkmigrator/ZooKeeperMigratorTest.groovy | 26 ++++++++++++++++++++
3 files changed, 37 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/06d7ecd3/nifi-toolkit/nifi-toolkit-zookeeper-migrator/src/main/java/org/apache/nifi/toolkit/zkmigrator/ZooKeeperMigrator.java
----------------------------------------------------------------------
diff --git a/nifi-toolkit/nifi-toolkit-zookeeper-migrator/src/main/java/org/apache/nifi/toolkit/zkmigrator/ZooKeeperMigrator.java b/nifi-toolkit/nifi-toolkit-zookeeper-migrator/src/main/java/org/apache/nifi/toolkit/zkmigrator/ZooKeeperMigrator.java
index c2ad166..c15286e 100644
--- a/nifi-toolkit/nifi-toolkit-zookeeper-migrator/src/main/java/org/apache/nifi/toolkit/zkmigrator/ZooKeeperMigrator.java
+++ b/nifi-toolkit/nifi-toolkit-zookeeper-migrator/src/main/java/org/apache/nifi/toolkit/zkmigrator/ZooKeeperMigrator.java
@@ -120,7 +120,7 @@ class ZooKeeperMigrator {
}
}
- void writeZooKeeper(InputStream zkData, AuthMode authMode, byte[] authData) throws IOException, ExecutionException, InterruptedException {
+ void writeZooKeeper(InputStream zkData, AuthMode authMode, byte[] authData, boolean ignoreSource) throws IOException, ExecutionException, InterruptedException {
ZooKeeper zooKeeper = getZooKeeper(zooKeeperEndpointConfig, authMode, authData);
JsonReader jsonReader = new JsonReader(new BufferedReader(new InputStreamReader(zkData)));
Gson gson = new GsonBuilder().create();
@@ -132,7 +132,7 @@ class ZooKeeperMigrator {
LOGGER.info("Source data was obtained from ZooKeeper: {}", sourceZooKeeperEndpointConfig);
Preconditions.checkArgument(!Strings.isNullOrEmpty(sourceZooKeeperEndpointConfig.getConnectString()) && !Strings.isNullOrEmpty(sourceZooKeeperEndpointConfig.getPath()),
"Source ZooKeeper %s from %s is invalid", sourceZooKeeperEndpointConfig, zkData);
- Preconditions.checkState(!zooKeeperEndpointConfig.equals(sourceZooKeeperEndpointConfig),
+ Preconditions.checkArgument( !(zooKeeperEndpointConfig.equals(sourceZooKeeperEndpointConfig) && !ignoreSource),
"Source ZooKeeper config %s for the data provided can not be the same as the configured destination ZooKeeper config %s",
sourceZooKeeperEndpointConfig, zooKeeperEndpointConfig);
http://git-wip-us.apache.org/repos/asf/nifi/blob/06d7ecd3/nifi-toolkit/nifi-toolkit-zookeeper-migrator/src/main/java/org/apache/nifi/toolkit/zkmigrator/ZooKeeperMigratorMain.java
----------------------------------------------------------------------
diff --git a/nifi-toolkit/nifi-toolkit-zookeeper-migrator/src/main/java/org/apache/nifi/toolkit/zkmigrator/ZooKeeperMigratorMain.java b/nifi-toolkit/nifi-toolkit-zookeeper-migrator/src/main/java/org/apache/nifi/toolkit/zkmigrator/ZooKeeperMigratorMain.java
index c8488a1..012618e 100644
--- a/nifi-toolkit/nifi-toolkit-zookeeper-migrator/src/main/java/org/apache/nifi/toolkit/zkmigrator/ZooKeeperMigratorMain.java
+++ b/nifi-toolkit/nifi-toolkit-zookeeper-migrator/src/main/java/org/apache/nifi/toolkit/zkmigrator/ZooKeeperMigratorMain.java
@@ -40,8 +40,8 @@ public class ZooKeeperMigratorMain {
private static final String JAVA_HOME = "JAVA_HOME";
private static final String NIFI_TOOLKIT_HOME = "NIFI_TOOLKIT_HOME";
private static final String HEADER = System.lineSeparator() + "A tool for importing and exporting data from ZooKeeper." + System.lineSeparator() + System.lineSeparator();
- private static final String FOOTER = new StringBuilder(System.lineSeparator()).append("Java home: ")
- .append(System.getenv(JAVA_HOME)).append(System.lineSeparator()).append("NiFi Toolkit home: ").append(System.getenv(NIFI_TOOLKIT_HOME)).toString();
+ private static final String FOOTER = System.lineSeparator() + "Java home: " +
+ System.getenv(JAVA_HOME) + System.lineSeparator() + "NiFi Toolkit home: " + System.getenv(NIFI_TOOLKIT_HOME);
private static final Option OPTION_ZK_MIGRATOR_HELP = Option.builder("h")
.longOpt("help")
@@ -81,6 +81,10 @@ public class ZooKeeperMigratorMain {
.hasArg()
.argName("filename")
.build();
+ private static final Option OPTION_IGNORE_SOURCE = Option.builder()
+ .longOpt("ignore-source")
+ .desc("ignores the source ZooKeeper endpoint specified in the exported data")
+ .build();
private static Options createOptions() {
final Options options = new Options();
@@ -88,6 +92,7 @@ public class ZooKeeperMigratorMain {
options.addOption(OPTION_ZK_ENDPOINT);
options.addOption(OPTION_ZK_AUTH_INFO);
options.addOption(OPTION_FILE);
+ options.addOption(OPTION_IGNORE_SOURCE);
final OptionGroup optionGroupAuth = new OptionGroup().addOption(OPTION_ZK_AUTH_INFO).addOption(OPTION_ZK_KRB_CONF_FILE);
optionGroupAuth.setRequired(false);
options.addOptionGroup(optionGroupAuth);
@@ -125,6 +130,7 @@ public class ZooKeeperMigratorMain {
final String filename = commandLine.getOptionValue(OPTION_FILE.getOpt());
final String auth = commandLine.getOptionValue(OPTION_ZK_AUTH_INFO.getOpt());
final String jaasFilename = commandLine.getOptionValue(OPTION_ZK_KRB_CONF_FILE.getOpt());
+ final boolean ignoreSource = commandLine.hasOption(OPTION_IGNORE_SOURCE.getLongOpt());
final AuthMode authMode;
final byte[] authData;
if (auth != null) {
@@ -143,7 +149,7 @@ public class ZooKeeperMigratorMain {
if (mode.equals(Mode.READ)) {
zookeeperMigrator.readZooKeeper(filename != null ? new FileOutputStream(Paths.get(filename).toFile()) : output, authMode, authData);
} else {
- zookeeperMigrator.writeZooKeeper(filename != null ? new FileInputStream(Paths.get(filename).toFile()) : System.in, authMode, authData);
+ zookeeperMigrator.writeZooKeeper(filename != null ? new FileInputStream(Paths.get(filename).toFile()) : System.in, authMode, authData, ignoreSource);
}
}
} catch (ParseException e) {
http://git-wip-us.apache.org/repos/asf/nifi/blob/06d7ecd3/nifi-toolkit/nifi-toolkit-zookeeper-migrator/src/test/java/org/apache/nifi/toolkit/zkmigrator/ZooKeeperMigratorTest.groovy
----------------------------------------------------------------------
diff --git a/nifi-toolkit/nifi-toolkit-zookeeper-migrator/src/test/java/org/apache/nifi/toolkit/zkmigrator/ZooKeeperMigratorTest.groovy b/nifi-toolkit/nifi-toolkit-zookeeper-migrator/src/test/java/org/apache/nifi/toolkit/zkmigrator/ZooKeeperMigratorTest.groovy
index 7444fd2..6c7476d 100644
--- a/nifi-toolkit/nifi-toolkit-zookeeper-migrator/src/test/java/org/apache/nifi/toolkit/zkmigrator/ZooKeeperMigratorTest.groovy
+++ b/nifi-toolkit/nifi-toolkit-zookeeper-migrator/src/test/java/org/apache/nifi/toolkit/zkmigrator/ZooKeeperMigratorTest.groovy
@@ -206,6 +206,32 @@ class ZooKeeperMigratorTest extends Specification {
'127.0.0.1:2181/path/node' || _
}
+ def "Test ignore source"() {
+ given:
+ def server = new TestingServer()
+ def connectString = "$server.connectString"
+ def dataPath = 'target/test-data-ignore-source.json'
+
+ when: "data is read from the source zookeeper"
+ ZooKeeperMigratorMain.main(['-r', '-z', connectString, '-f', dataPath] as String[])
+
+ then: "verify the data has been written the output file"
+ new File(dataPath).exists()
+
+ when: "data is sent to the same zookeeper as the the source zookeeper without ignore source"
+ ZooKeeperMigratorMain.main(['-s', '-z', connectString, '-f', dataPath] as String[])
+
+ then: "verify that a runtime exception is thrown with an illegal argument exception as the cause"
+ def e = thrown(RuntimeException)
+ e.cause.class == IllegalArgumentException
+
+ when: "data is sent to the same zookeeper as the source zookeeper with ignore source option is set"
+ ZooKeeperMigratorMain.main(['-s', '-z', connectString, '-f', dataPath, '--ignore-source'] as String[])
+
+ then: "no exceptions are thrown"
+ noExceptionThrown()
+ }
+
def List<String> getChildren(ZooKeeper client, String path, List<String> ag) {
def children = client.getChildren(path, null)
ag.add path