You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by mu...@apache.org on 2015/11/17 01:32:56 UTC
phoenix git commit: PHOENIX-2359 Configuration for PQS to use
Protobuf serialization instead of JSON (elserj)
Repository: phoenix
Updated Branches:
refs/heads/4.x-HBase-0.98 c9b76f8be -> 4e38a6e12
PHOENIX-2359 Configuration for PQS to use Protobuf serialization instead of JSON (elserj)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/4e38a6e1
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/4e38a6e1
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/4e38a6e1
Branch: refs/heads/4.x-HBase-0.98
Commit: 4e38a6e12af926010ecbddb91270f23081ab15b0
Parents: c9b76f8
Author: Mujtaba <mu...@apache.org>
Authored: Mon Nov 16 16:32:42 2015 -0800
Committer: Mujtaba <mu...@apache.org>
Committed: Mon Nov 16 16:32:42 2015 -0800
----------------------------------------------------------------------
bin/sqlline-thin.py | 30 ++++++-
.../org/apache/phoenix/query/QueryServices.java | 13 ++-
.../phoenix/query/QueryServicesOptions.java | 16 ++++
.../queryserver/client/ThinClientUtil.java | 11 ++-
.../phoenix/end2end/QueryServerBasicsIT.java | 4 +-
.../apache/phoenix/queryserver/server/Main.java | 89 +++++++++++---------
pom.xml | 2 +-
7 files changed, 116 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/4e38a6e1/bin/sqlline-thin.py
----------------------------------------------------------------------
diff --git a/bin/sqlline-thin.py b/bin/sqlline-thin.py
index 9877def..2e237ed 100755
--- a/bin/sqlline-thin.py
+++ b/bin/sqlline-thin.py
@@ -40,6 +40,7 @@ phoenix_utils.setPath()
url = "localhost:8765"
sqlfile = ""
+serialization_key = 'phoenix.queryserver.serialization'
def usage_and_exit():
sys.exit("usage: sqlline-thin.py [host[:port]] [sql_file]")
@@ -53,6 +54,29 @@ def cleanup_url(url):
url = url + ":8765"
return url
+def get_serialization():
+ default_serialization='PROTOBUF'
+ env=os.environ.copy()
+ hbase_cmd = phoenix_utils.which('hbase')
+ if hbase_cmd is None:
+ print 'Failed to find hbase executable on PATH, defaulting serialization to %s.' % default_serialization
+ return default_serialization
+
+ env['HBASE_CONF_DIR'] = phoenix_utils.hbase_conf_dir
+ proc = subprocess.Popen([hbase_cmd, 'org.apache.hadoop.hbase.util.HBaseConfTool', serialization_key],
+ env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ (stdout, stderr) = proc.communicate()
+ if proc.returncode != 0:
+ print 'Failed to extract serialization from hbase-site.xml, defaulting to %s.' % default_serialization
+ return default_serialization
+ # Don't expect this to happen, but give a default value just in case
+ if stdout is None:
+ return default_serialization
+
+ stdout = stdout.strip()
+ if stdout == 'null':
+ return default_serialization
+ return stdout
if len(sys.argv) == 1:
pass
@@ -81,6 +105,8 @@ if os.name == 'nt':
# HBase/Phoenix client side property override
hbase_config_path = os.getenv('HBASE_CONF_DIR', phoenix_utils.current_dir)
+serialization = get_serialization()
+
java_home = os.getenv('JAVA_HOME')
# load hbase-env.??? to extract JAVA_HOME, HBASE_PID_DIR, HBASE_LOG_DIR
@@ -112,10 +138,10 @@ else:
java = 'java'
java_cmd = java + ' -cp "' + phoenix_utils.hbase_conf_dir + os.pathsep + phoenix_utils.phoenix_thin_client_jar + \
- os.pathsep + phoenix_utils.hadoop_conf + os.pathsep + phoenix_utils.hadoop_classpath + '" -Dlog4j.configuration=file:' + \
+ os.pathsep + phoenix_utils.hadoop_conf + os.pathsep + phoenix_utils.hadoop_classpath + '" -Dlog4j.configuration=file:' + \
os.path.join(phoenix_utils.current_dir, "log4j.properties") + \
" sqlline.SqlLine -d org.apache.phoenix.queryserver.client.Driver " + \
- " -u jdbc:phoenix:thin:url=" + url + \
+ " -u jdbc:phoenix:thin:url='" + url + ";serialization=" + serialization + "'" + \
" -n none -p none --color=" + colorSetting + " --fastConnect=false --verbose=true " + \
" --isolation=TRANSACTION_READ_COMMITTED " + sqlfile
http://git-wip-us.apache.org/repos/asf/phoenix/blob/4e38a6e1/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
index 70d8f5e..e4dc57a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -169,7 +169,18 @@ public interface QueryServices extends SQLCloseable {
public static final String ALLOW_VIEWS_ADD_NEW_CF_BASE_TABLE = "phoenix.schema.view.newcf";
public static final String RETURN_SEQUENCE_VALUES_ATTRIB = "phoenix.sequence.returnValues";
public static final String EXTRA_JDBC_ARGUMENTS_ATTRIB = "phoenix.jdbc.extra.arguments";
-
+
+ // queryserver configuration keys
+ public static final String QUERY_SERVER_SERIALIZATION_ATTRIB = "phoenix.queryserver.serialization";
+ public static final String QUERY_SERVER_META_FACTORY_ATTRIB = "phoenix.queryserver.metafactory.class";
+ public static final String QUERY_SERVER_HTTP_PORT_ATTRIB = "phoenix.queryserver.http.port";
+ public static final String QUERY_SERVER_ENV_LOGGING_ATTRIB = "phoenix.queryserver.envvars.logging.disabled";
+ public static final String QUERY_SERVER_ENV_LOGGING_SKIPWORDS_ATTRIB = "phoenix.queryserver.envvars.logging.skipwords";
+ public static final String QUERY_SERVER_KEYTAB_FILENAME_ATTRIB = "phoenix.queryserver.keytab.file";
+ public static final String QUERY_SERVER_KERBEROS_PRINCIPAL_ATTRIB = "phoenix.queryserver.kerberos.principal";
+ public static final String QUERY_SERVER_DNS_NAMESERVER_ATTRIB = "phoenix.queryserver.dns.nameserver";
+ public static final String QUERY_SERVER_DNS_INTERFACE_ATTRIB = "phoenix.queryserver.dns.interface";
+ public static final String QUERY_SERVER_HBASE_SECURITY_CONF_ATTRIB = "hbase.security.authentication";
/**
* Get executor service used for parallel scans
http://git-wip-us.apache.org/repos/asf/phoenix/blob/4e38a6e1/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
index 3a4a51f..be68427 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
@@ -68,6 +68,8 @@ import static org.apache.phoenix.query.QueryServices.THREAD_TIMEOUT_MS_ATTRIB;
import static org.apache.phoenix.query.QueryServices.USE_BYTE_BASED_REGEX_ATTRIB;
import static org.apache.phoenix.query.QueryServices.USE_INDEXES_ATTRIB;
+import java.util.HashSet;
+import java.util.Set;
import java.util.Map.Entry;
import org.apache.hadoop.conf.Configuration;
@@ -201,6 +203,20 @@ public class QueryServicesOptions {
public static final boolean DEFAULT_RETURN_SEQUENCE_VALUES = false;
public static final String DEFAULT_EXTRA_JDBC_ARGUMENTS = "";
+ // QueryServer defaults -- ensure ThinClientUtil is also updated since phoenix-server-client
+ // doesn't depend on phoenix-core.
+ public static final String DEFAULT_QUERY_SERVER_SERIALIZATION = "PROTOBUF";
+ public static final int DEFAULT_QUERY_SERVER_HTTP_PORT = 8765;
+ @SuppressWarnings("serial")
+ public static final Set<String> DEFAULT_QUERY_SERVER_SKIP_WORDS = new HashSet<String>() {
+ {
+ add("secret");
+ add("passwd");
+ add("password");
+ add("credential");
+ }
+ };
+
private final Configuration config;
private QueryServicesOptions(Configuration config) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/4e38a6e1/phoenix-server-client/src/main/java/org/apache/phoenix/queryserver/client/ThinClientUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-server-client/src/main/java/org/apache/phoenix/queryserver/client/ThinClientUtil.java b/phoenix-server-client/src/main/java/org/apache/phoenix/queryserver/client/ThinClientUtil.java
index 0ef1c8c..af56c33 100644
--- a/phoenix-server-client/src/main/java/org/apache/phoenix/queryserver/client/ThinClientUtil.java
+++ b/phoenix-server-client/src/main/java/org/apache/phoenix/queryserver/client/ThinClientUtil.java
@@ -21,6 +21,9 @@ package org.apache.phoenix.queryserver.client;
* Utilities for thin clients.
*/
public final class ThinClientUtil {
+ // The default serialization is also defined in QueryServicesOptions. phoenix-server-client
+ // currently doesn't depend on phoenix-core so we have to deal with the duplication.
+ private static final String DEFAULT_SERIALIZATION = "PROTOBUF";
private ThinClientUtil() {}
@@ -29,7 +32,11 @@ public final class ThinClientUtil {
}
public static String getConnectionUrl(String protocol, String hostname, int port) {
- String urlFmt = Driver.CONNECT_STRING_PREFIX + "url=%s://%s:%s";
- return String.format(urlFmt, protocol, hostname, port);
+ return getConnectionUrl(protocol, hostname, port, DEFAULT_SERIALIZATION);
+ }
+
+ public static String getConnectionUrl(String protocol, String hostname, int port, String serialization) {
+ String urlFmt = Driver.CONNECT_STRING_PREFIX + "url=%s://%s:%s;serialization=%s";
+ return String.format(urlFmt, protocol, hostname, port, serialization);
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/4e38a6e1/phoenix-server/src/it/java/org/apache/phoenix/end2end/QueryServerBasicsIT.java
----------------------------------------------------------------------
diff --git a/phoenix-server/src/it/java/org/apache/phoenix/end2end/QueryServerBasicsIT.java b/phoenix-server/src/it/java/org/apache/phoenix/end2end/QueryServerBasicsIT.java
index 3b18c4e..3003f31 100644
--- a/phoenix-server/src/it/java/org/apache/phoenix/end2end/QueryServerBasicsIT.java
+++ b/phoenix-server/src/it/java/org/apache/phoenix/end2end/QueryServerBasicsIT.java
@@ -20,8 +20,8 @@ package org.apache.phoenix.end2end;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.queryserver.client.ThinClientUtil;
-import org.apache.phoenix.queryserver.server.Main;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -57,7 +57,7 @@ public class QueryServerBasicsIT extends BaseHBaseManagedTimeIT {
@BeforeClass
public static void beforeClass() throws Exception {
CONF = getTestClusterConfig();
- CONF.setInt(Main.QUERY_SERVER_HTTP_PORT_KEY, 0);
+ CONF.setInt(QueryServices.QUERY_SERVER_HTTP_PORT_ATTRIB, 0);
String url = getUrl();
AVATICA_SERVER = new QueryServerThread(new String[] { url }, CONF,
QueryServerBasicsIT.class.getName());
http://git-wip-us.apache.org/repos/asf/phoenix/blob/4e38a6e1/phoenix-server/src/main/java/org/apache/phoenix/queryserver/server/Main.java
----------------------------------------------------------------------
diff --git a/phoenix-server/src/main/java/org/apache/phoenix/queryserver/server/Main.java b/phoenix-server/src/main/java/org/apache/phoenix/queryserver/server/Main.java
index 9f9bfc7..106d422 100644
--- a/phoenix-server/src/main/java/org/apache/phoenix/queryserver/server/Main.java
+++ b/phoenix-server/src/main/java/org/apache/phoenix/queryserver/server/Main.java
@@ -19,9 +19,10 @@ package org.apache.phoenix.queryserver.server;
import com.google.common.annotations.VisibleForTesting;
import org.apache.calcite.avatica.Meta;
+import org.apache.calcite.avatica.remote.Driver;
import org.apache.calcite.avatica.remote.LocalService;
import org.apache.calcite.avatica.remote.Service;
-import org.apache.calcite.avatica.server.AvaticaHandler;
+import org.apache.calcite.avatica.server.HandlerFactory;
import org.apache.calcite.avatica.server.HttpServer;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -33,6 +34,9 @@ import org.apache.hadoop.net.DNS;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.query.QueryServicesOptions;
+import org.eclipse.jetty.server.Handler;
import java.lang.management.ManagementFactory;
import java.lang.management.RuntimeMXBean;
@@ -48,36 +52,8 @@ import java.util.concurrent.TimeUnit;
*/
public final class Main extends Configured implements Tool, Runnable {
- public static final String QUERY_SERVER_META_FACTORY_KEY =
- "phoenix.queryserver.metafactory.class";
-
- public static final String QUERY_SERVER_HTTP_PORT_KEY =
- "phoenix.queryserver.http.port";
- public static final int DEFAULT_HTTP_PORT = 8765;
-
- public static final String QUERY_SERVER_ENV_LOGGING_KEY =
- "phoenix.queryserver.envvars.logging.disabled";
- public static final String QUERY_SERVER_ENV_LOGGING_SKIPWORDS_KEY =
- "phoenix.queryserver.envvars.logging.skipwords";
-
- public static final String KEYTAB_FILENAME_KEY = "phoenix.queryserver.keytab.file";
- public static final String KERBEROS_PRINCIPAL_KEY = "phoenix.queryserver.kerberos.principal";
- public static final String DNS_NAMESERVER_KEY = "phoenix.queryserver.dns.nameserver";
- public static final String DNS_INTERFACE_KEY = "phoenix.queryserver.dns.interface";
- public static final String HBASE_SECURITY_CONF_KEY = "hbase.security.authentication";
-
protected static final Log LOG = LogFactory.getLog(Main.class);
- @SuppressWarnings("serial")
- private static final Set<String> DEFAULT_SKIP_WORDS = new HashSet<String>() {
- {
- add("secret");
- add("passwd");
- add("password");
- add("credential");
- }
- };
-
private final String[] argv;
private final CountDownLatch runningLatch = new CountDownLatch(1);
private HttpServer server = null;
@@ -107,10 +83,10 @@ public final class Main extends Configured implements Tool, Runnable {
*/
public static void logProcessInfo(Configuration conf) {
// log environment variables unless asked not to
- if (conf == null || !conf.getBoolean(QUERY_SERVER_ENV_LOGGING_KEY, false)) {
- Set<String> skipWords = new HashSet<String>(DEFAULT_SKIP_WORDS);
+ if (conf == null || !conf.getBoolean(QueryServices.QUERY_SERVER_ENV_LOGGING_ATTRIB, false)) {
+ Set<String> skipWords = new HashSet<String>(QueryServicesOptions.DEFAULT_QUERY_SERVER_SKIP_WORDS);
if (conf != null) {
- String[] confSkipWords = conf.getStrings(QUERY_SERVER_ENV_LOGGING_SKIPWORDS_KEY);
+ String[] confSkipWords = conf.getStrings(QueryServices.QUERY_SERVER_ENV_LOGGING_SKIPWORDS_ATTRIB);
if (confSkipWords != null) {
skipWords.addAll(Arrays.asList(confSkipWords));
}
@@ -183,26 +159,29 @@ public final class Main extends Configured implements Tool, Runnable {
logProcessInfo(getConf());
try {
// handle secure cluster credentials
- if ("kerberos".equalsIgnoreCase(getConf().get(HBASE_SECURITY_CONF_KEY))) {
+ if ("kerberos".equalsIgnoreCase(getConf().get(QueryServices.QUERY_SERVER_HBASE_SECURITY_CONF_ATTRIB))) {
String hostname = Strings.domainNamePointerToHostName(DNS.getDefaultHost(
- getConf().get(DNS_INTERFACE_KEY, "default"),
- getConf().get(DNS_NAMESERVER_KEY, "default")));
+ getConf().get(QueryServices.QUERY_SERVER_DNS_INTERFACE_ATTRIB, "default"),
+ getConf().get(QueryServices.QUERY_SERVER_DNS_NAMESERVER_ATTRIB, "default")));
if (LOG.isDebugEnabled()) {
- LOG.debug("Login to " + hostname + " using " + getConf().get(KEYTAB_FILENAME_KEY)
- + " and principal " + getConf().get(KERBEROS_PRINCIPAL_KEY) + ".");
+ LOG.debug("Login to " + hostname + " using " + getConf().get(QueryServices.QUERY_SERVER_KEYTAB_FILENAME_ATTRIB)
+ + " and principal " + getConf().get(QueryServices.QUERY_SERVER_KERBEROS_PRINCIPAL_ATTRIB) + ".");
}
- SecurityUtil.login(getConf(), KEYTAB_FILENAME_KEY, KERBEROS_PRINCIPAL_KEY, hostname);
+ SecurityUtil.login(getConf(), QueryServices.QUERY_SERVER_KEYTAB_FILENAME_ATTRIB,
+ QueryServices.QUERY_SERVER_KERBEROS_PRINCIPAL_ATTRIB, hostname);
LOG.info("Login successful.");
}
Class<? extends PhoenixMetaFactory> factoryClass = getConf().getClass(
- QUERY_SERVER_META_FACTORY_KEY, PhoenixMetaFactoryImpl.class, PhoenixMetaFactory.class);
- int port = getConf().getInt(QUERY_SERVER_HTTP_PORT_KEY, DEFAULT_HTTP_PORT);
+ QueryServices.QUERY_SERVER_META_FACTORY_ATTRIB, PhoenixMetaFactoryImpl.class, PhoenixMetaFactory.class);
+ int port = getConf().getInt(QueryServices.QUERY_SERVER_HTTP_PORT_ATTRIB,
+ QueryServicesOptions.DEFAULT_QUERY_SERVER_HTTP_PORT);
LOG.debug("Listening on port " + port);
PhoenixMetaFactory factory =
factoryClass.getDeclaredConstructor(Configuration.class).newInstance(getConf());
Meta meta = factory.create(Arrays.asList(args));
+ final HandlerFactory handlerFactory = new HandlerFactory();
Service service = new LocalService(meta);
- server = new HttpServer(port, new AvaticaHandler(service));
+ server = new HttpServer(port, getHandler(getConf(), service, handlerFactory));
server.start();
runningLatch.countDown();
server.join();
@@ -214,6 +193,34 @@ public final class Main extends Configured implements Tool, Runnable {
}
}
+ /**
+ * Instantiates the Handler for use by the Avatica (Jetty) server.
+ *
+ * @param conf The configuration
+ * @param service The Avatica Service implementation
+ * @param handlerFactory Factory used for creating a Handler
+ * @return The Handler to use based on the configuration.
+ */
+ Handler getHandler(Configuration conf, Service service, HandlerFactory handlerFactory) {
+ String serializationName = conf.get(QueryServices.QUERY_SERVER_SERIALIZATION_ATTRIB,
+ QueryServicesOptions.DEFAULT_QUERY_SERVER_SERIALIZATION);
+
+ Driver.Serialization serialization;
+ // Otherwise, use what was provided in the configuration
+ try {
+ serialization = Driver.Serialization.valueOf(serializationName);
+ } catch (Exception e) {
+ LOG.error("Unknown message serialization type for " + serializationName);
+ throw e;
+ }
+
+ Handler handler = handlerFactory.getHandler(service, serialization);
+
+ LOG.info("Instantiated " + handler.getClass() + " for QueryServer");
+
+ return handler;
+ }
+
@Override public void run() {
try {
retCode = run(argv);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/4e38a6e1/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index ca52392..d52c8db 100644
--- a/pom.xml
+++ b/pom.xml
@@ -110,7 +110,7 @@
<collections.version>3.2.1</collections.version>
<jodatime.version>2.7</jodatime.version>
<joni.version>2.1.2</joni.version>
- <calcite.version>1.3.0-incubating</calcite.version>
+ <calcite.version>1.5.0</calcite.version>
<jettyVersion>8.1.7.v20120910</jettyVersion>
<!-- Test Dependencies -->