You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ka...@apache.org on 2019/05/18 20:15:22 UTC
[phoenix-queryserver] branch master updated: PHOENIX-5255 Create
Orchestrator for QueryServerCanaryTool in phoenix-queryserver project
This is an automated email from the ASF dual-hosted git repository.
karanmehta93 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/phoenix-queryserver.git
The following commit(s) were added to refs/heads/master by this push:
new d6f93a5 PHOENIX-5255 Create Orchestrator for QueryServerCanaryTool in phoenix-queryserver project
d6f93a5 is described below
commit d6f93a542b31ca417af97751bf5d37a67af6963d
Author: Swaroopa Kadam <sw...@gmail.com>
AuthorDate: Sat May 18 13:15:18 2019 -0700
PHOENIX-5255 Create Orchestrator for QueryServerCanaryTool in phoenix-queryserver project
---
pom.xml | 45 ++++++-
queryserver-orchestrator/pom.xml | 73 +++++++++++
.../QueryServerCanaryOrchestrator.java | 141 +++++++++++++++++++++
.../orchestrator/TestExecutorClient.java | 129 +++++++++++++++++++
.../queryserver/orchestrator/ToolWrapper.java | 43 +++++++
5 files changed, 429 insertions(+), 2 deletions(-)
diff --git a/pom.xml b/pom.xml
index 0f87d75..caff061 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1,4 +1,24 @@
<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
@@ -30,6 +50,7 @@
<module>load-balancer</module>
<module>assembly</module>
<module>phoenix-client</module>
+ <module>queryserver-orchestrator</module>
</modules>
<repositories>
@@ -53,7 +74,7 @@
<hbase.version>1.4.0</hbase.version>
<hadoop-two.version>2.7.5</hadoop-two.version>
<curator.version>2.12.0</curator.version>
- <phoenix.version>4.15.0-HBase-1.4-SNAPSHOT</phoenix.version>
+ <phoenix.version>4.15.0-HBase-1.3-SNAPSHOT</phoenix.version>
<!-- Dependency versions -->
<protobuf-java.version>2.5.0</protobuf-java.version>
@@ -233,7 +254,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
- <version>2.4.3</version>
+ <version>3.2.0</version>
</plugin>
<plugin>
<!-- Allows us to get the apache-ds bundle artifacts -->
@@ -508,6 +529,16 @@
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
+ <artifactId>curator-recipes</artifactId>
+ <version>${curator.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-framework</artifactId>
+ <version>${curator.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
<artifactId>curator-test</artifactId>
<version>${curator.version}</version>
</dependency>
@@ -571,6 +602,16 @@
<artifactId>javax.servlet-api</artifactId>
<version>${servlet.api.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>1.8.0-alpha2</version>
+ </dependency>
+ <dependency>
+ <groupId>net.sourceforge.argparse4j</groupId>
+ <artifactId>argparse4j</artifactId>
+ <version>0.8.1</version>
+ </dependency>
</dependencies>
</dependencyManagement>
diff --git a/queryserver-orchestrator/pom.xml b/queryserver-orchestrator/pom.xml
new file mode 100644
index 0000000..de00fe9
--- /dev/null
+++ b/queryserver-orchestrator/pom.xml
@@ -0,0 +1,73 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>phoenix-queryserver</artifactId>
+ <groupId>org.apache.phoenix</groupId>
+ <version>1.0.0-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>queryserver-orchestrator</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.phoenix</groupId>
+ <artifactId>queryserver-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.calcite.avatica</groupId>
+ <artifactId>avatica-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>net.sourceforge.argparse4j</groupId>
+ <artifactId>argparse4j</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-framework</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-recipes</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.phoenix</groupId>
+ <artifactId>phoenix-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ </dependency>
+ </dependencies>
+
+</project>
\ No newline at end of file
diff --git a/queryserver-orchestrator/src/main/java/org/apache/phoenix/queryserver/orchestrator/QueryServerCanaryOrchestrator.java b/queryserver-orchestrator/src/main/java/org/apache/phoenix/queryserver/orchestrator/QueryServerCanaryOrchestrator.java
new file mode 100644
index 0000000..4279233
--- /dev/null
+++ b/queryserver-orchestrator/src/main/java/org/apache/phoenix/queryserver/orchestrator/QueryServerCanaryOrchestrator.java
@@ -0,0 +1,141 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.queryserver.orchestrator;
+
+import net.sourceforge.argparse4j.ArgumentParsers;
+import net.sourceforge.argparse4j.inf.ArgumentParser;
+import net.sourceforge.argparse4j.inf.ArgumentParserException;
+import net.sourceforge.argparse4j.inf.Namespace;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.BoundedExponentialBackoffRetry;
+
+import org.apache.curator.utils.CloseableUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/*
+ This is a main class to which initiates the relevant clients
+ to run orchestrator for queryserver canary
+ */
+
+public class QueryServerCanaryOrchestrator {
+ private static final Logger LOGGER = LoggerFactory.getLogger(QueryServerCanaryOrchestrator.class);
+
+ private static String ZK_PATH = "/pqs/leader";
+
+ private static Namespace parseArgs(String[] args) {
+
+ ArgumentParser parser = ArgumentParsers.newFor("PQS Canary Orchestrator").build()
+ .description("PQS Canary Orchestrator");
+
+ parser.addArgument("--zkurl", "-zk").type(String.class).help("URL for Zookeeper");
+
+ parser.addArgument("--zkpath", "-zkp").type(String.class).nargs("?").setDefault(ZK_PATH)
+ .help("ZKNode path default: " + ZK_PATH);
+
+ parser.addArgument("--hostname", "-hn").type(String.class).nargs("?").setDefault("localhost").help("Hostname on "
+ + "which PQS is running.");
+
+ parser.addArgument("--port", "-p").type(String.class).nargs("?").setDefault("8765").help("Port on which PQS " +
+ "is running.");
+
+ parser.addArgument("--constring", "-cs").type(String.class).nargs("?").help("Pass an " +
+ "explicit " + "connection String to connect to PQS. default: null");
+
+ parser.addArgument("--timeout", "-t").type(String.class).nargs("?").setDefault("120")
+ .help("Maximum time for which the app should run before returning error. default: 120 sec");
+
+ parser.addArgument("--testtable", "-tt").type(String.class).nargs("?").setDefault
+ ("PQSTEST").help("Custom name for the test table. default: PQSTEST");
+
+ parser.addArgument("--testschema", "-ts").type(String.class).nargs("?").setDefault
+ ("TEST").help("Custom name for the test table. default: TEST");
+
+ parser.addArgument("--logsinkclass", "-lsc").type(String.class).nargs("?").setDefault
+ ("org.apache.phoenix.tool.PhoenixCanaryTool$StdOutSink")
+ .help("Path to a Custom implementation for log sink class. default: stdout");
+
+ parser.addArgument("--interval", "-in").type(String.class).nargs("?").setDefault("900")
+ .help("Time interval between 2 consecutive test suite runs");
+
+
+ Namespace res = null;
+ try {
+ res = parser.parseArgs(args);
+ } catch (ArgumentParserException e) {
+ parser.handleError(e);
+ }
+ return res;
+ }
+
+ public static Map<String, String> getArgs(Namespace cArgs) {
+
+ Map<String, String> params = new HashMap<>();
+
+ for (Map.Entry<String, Object> entry : cArgs.getAttrs().entrySet()) {
+ params.put(entry.getKey(), (String) entry.getValue());
+ }
+ return params;
+ }
+
+ public static void main(String[] args) {
+
+ LOGGER.info("Starting PQS Canary Orchestrator...");
+ ToolWrapper tool = new ToolWrapper();
+ try {
+ Namespace cArgs = parseArgs(args);
+ if (cArgs == null) {
+ throw new RuntimeException("Argument parsing failed");
+ }
+
+ Map<String, String> params = getArgs(cArgs);
+ ZK_PATH = params.get("zkpath");
+
+ CuratorFramework curatorClient = CuratorFrameworkFactory.newClient(params.get("zkurl"), new
+ BoundedExponentialBackoffRetry(1000, 60000, 50000));
+ curatorClient.start();
+
+ TestExecutorClient testExecutorClient = new TestExecutorClient(curatorClient, ZK_PATH, params, tool);
+
+ try {
+ testExecutorClient.start();
+ while (!testExecutorClient.isDone()) {
+ Thread.sleep(1000);
+ }
+ } catch (Exception ex) {
+ LOGGER.error("The Main thread was interrupted", ex);
+ } finally {
+ LOGGER.info("Shutdown Hook for PQS Canary Orchestrator running...");
+ CloseableUtils.closeQuietly(testExecutorClient);
+ CloseableUtils.closeQuietly(curatorClient);
+ LOGGER.info("Closed Curator Client");
+ }
+
+ } catch (Exception e) {
+ LOGGER.error("Error in PQS Canary Orchestrator. ", e);
+ throw new RuntimeException(e);
+ }
+ LOGGER.info("Exiting PQS Canary Orchestrator...");
+ }
+
+}
diff --git a/queryserver-orchestrator/src/main/java/org/apache/phoenix/queryserver/orchestrator/TestExecutorClient.java b/queryserver-orchestrator/src/main/java/org/apache/phoenix/queryserver/orchestrator/TestExecutorClient.java
new file mode 100644
index 0000000..d252ed2
--- /dev/null
+++ b/queryserver-orchestrator/src/main/java/org/apache/phoenix/queryserver/orchestrator/TestExecutorClient.java
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.phoenix.queryserver.orchestrator;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.recipes.leader.CancelLeadershipException;
+import org.apache.curator.framework.recipes.leader.LeaderSelector;
+import org.apache.curator.framework.recipes.leader.LeaderSelectorListenerAdapter;
+import org.apache.curator.framework.state.ConnectionState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/*
+ This is a class which selects the leader among all the orchestrator hosts using curator framework
+ and appropriately calls ToolWrapper.
+ */
+
+public class TestExecutorClient extends LeaderSelectorListenerAdapter implements Closeable {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(TestExecutorClient.class);
+ private final LeaderSelector leaderSelector;
+
+ private Map<String, String> params;
+ private ToolWrapper toolWrapper;
+ private Boolean isRunning = false;
+
+ public TestExecutorClient(CuratorFramework client, String path, Map<String, String> params, ToolWrapper tool) {
+
+ // create a leader selector using the given path for management
+ // all participants in a given leader selection must use the same path
+ // ExampleClient here is also a LeaderSelectorListener but this isn't required
+ this.leaderSelector = new LeaderSelector(client, path, this);
+
+ // for most cases you will want your instance to requeue when it relinquishes leadership
+ this.leaderSelector.autoRequeue();
+ this.params = params;
+ this.toolWrapper = tool;
+ }
+
+ public void close() {
+ leaderSelector.close();
+ }
+
+ public void start() {
+ leaderSelector.start();
+ isRunning = true;
+ }
+
+ @Override
+ public void takeLeadership(CuratorFramework client) throws Exception {
+ // we are now the leader. This method should not return until we want to relinquish
+ // leadership
+ LOGGER.info("Took leadership.");
+
+ while (true) {
+ LOGGER.info("Starting test case suite execution.");
+ executeQueryServerCanaryTool();
+
+ LOGGER.info("Test suite execution completed. Waiting for " + params.get("interval")
+ + " secs before executing next run.");
+
+ TimeUnit.SECONDS.sleep(Integer.parseInt(params.get("interval")));
+ }
+ }
+
+ @Override
+ public void stateChanged(CuratorFramework client, ConnectionState newState) {
+ LOGGER.info("ZK Connection State Changed to [{}]", newState.name());
+ switch(newState) {
+ case CONNECTED:
+ LOGGER.info("The host is connected at" +System.currentTimeMillis());
+ break;
+ case LOST:
+ LOGGER.info("The host lost connection at "+ System.currentTimeMillis());
+ isRunning = false;
+ throw new CancelLeadershipException();
+ case READ_ONLY:
+ LOGGER.info("The connection state is Read only since "+ System.currentTimeMillis());
+ break;
+ case RECONNECTED:
+ LOGGER.info("Reconnected to the ZK Path at " +System.currentTimeMillis());
+ break;
+ case SUSPENDED:
+ isRunning = false;
+ throw new CancelLeadershipException();
+ default:
+ }
+ }
+
+ @VisibleForTesting
+ public void executeQueryServerCanaryTool() {
+ List<String> cmd = new ArrayList<>();
+ for (Map.Entry<String, String> entry : params.entrySet()) {
+ if (entry.getValue() != null) {
+ cmd.add("--" + entry.getKey());
+ cmd.add(entry.getValue());
+ }
+ }
+ LOGGER.info("Test Suit execution started.");
+ toolWrapper.executeMain(cmd.toArray(new String[cmd.size()]));
+ }
+
+ public boolean isDone() {
+ return !isRunning;
+ }
+}
diff --git a/queryserver-orchestrator/src/main/java/org/apache/phoenix/queryserver/orchestrator/ToolWrapper.java b/queryserver-orchestrator/src/main/java/org/apache/phoenix/queryserver/orchestrator/ToolWrapper.java
new file mode 100644
index 0000000..62e00a4
--- /dev/null
+++ b/queryserver-orchestrator/src/main/java/org/apache/phoenix/queryserver/orchestrator/ToolWrapper.java
@@ -0,0 +1,43 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.queryserver.orchestrator;
+
+import com.google.common.base.Joiner;
+import org.apache.phoenix.tool.PhoenixCanaryTool;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+
+/*
+ This is a wrapper class runs the PhoenixCanaryTool.
+ */
+public class ToolWrapper {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(ToolWrapper.class);
+
+ public void executeMain(String [] args){
+ if(Arrays.asList(args).contains(null)){
+ LOGGER.error("Bad argument list passed to executeMain. Skipping QueryServerCanaryTool.");
+ return;
+ }
+ LOGGER.info("Passing args to QueryServerCanaryTool: " + Joiner.on(",").join(args));
+ PhoenixCanaryTool.main(args);
+ }
+}