You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by su...@apache.org on 2016/05/25 11:23:32 UTC
lens git commit: LENS-1155 : Seeing INTERNAL_SERVER_ERROR for JDBC
queries due to connection unavailability
Repository: lens
Updated Branches:
refs/heads/master a4b4b85d8 -> 5c25e68fe
LENS-1155 : Seeing INTERNAL_SERVER_ERROR for JDBC queries due to connection unavailability
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/5c25e68f
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/5c25e68f
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/5c25e68f
Branch: refs/heads/master
Commit: 5c25e68fe71670df6c72d3b8dc08fcbcad6011c4
Parents: a4b4b85
Author: SushilMohanty <su...@apache.org>
Authored: Wed May 25 16:53:22 2016 +0530
Committer: sushilmohanty <su...@apache.org>
Committed: Wed May 25 16:53:22 2016 +0530
----------------------------------------------------------------------
.../org/apache/lens/driver/jdbc/JDBCDriver.java | 3 +-
.../jdbc/MaxJDBCConnectionCheckConstraint.java | 48 ++++++++++++++++++++
...MaxJDBCConnectionCheckConstraintFactory.java | 38 ++++++++++++++++
.../src/main/resources/jdbcdriver-default.xml | 3 +-
.../apache/lens/driver/jdbc/TestJdbcDriver.java | 37 +++++++++++++++
.../apache/lens/server/api/util/LensUtil.java | 2 +-
6 files changed, 128 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/5c25e68f/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
index 044a19c..8047302 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
@@ -94,6 +94,7 @@ public class JDBCDriver extends AbstractLensDriver {
private ExecutorService asyncQueryPool;
/** The query context map. */
+ @Getter
private ConcurrentHashMap<QueryHandle, JdbcQueryContext> queryContextMap;
/** The conf. */
@@ -458,7 +459,7 @@ public class JDBCDriver extends AbstractLensDriver {
final int maxPoolSize = parseInt(this.conf.get(JDBC_POOL_MAX_SIZE.getConfigKey()));
final int maxConcurrentQueries
= parseInt(this.conf.get(MaxConcurrentDriverQueriesConstraintFactory.MAX_CONCURRENT_QUERIES_KEY));
- checkState(maxPoolSize == maxConcurrentQueries, "maxPoolSize:" + maxPoolSize + " maxConcurrentQueries:"
+ checkState(maxPoolSize >= maxConcurrentQueries, "maxPoolSize:" + maxPoolSize + " maxConcurrentQueries:"
+ maxConcurrentQueries);
queryContextMap = new ConcurrentHashMap<>();
http://git-wip-us.apache.org/repos/asf/lens/blob/5c25e68f/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraint.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraint.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraint.java
new file mode 100644
index 0000000..82b5647
--- /dev/null
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraint.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.driver.jdbc;
+
+import org.apache.lens.server.api.driver.LensDriver;
+import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.query.collect.EstimatedImmutableQueryCollection;
+import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class MaxJDBCConnectionCheckConstraint implements QueryLaunchingConstraint {
+
+ private final int poolMaxSize;
+
+ public MaxJDBCConnectionCheckConstraint(final int poolMaxSize) {
+ this.poolMaxSize = poolMaxSize;
+ }
+
+ @Override
+ public boolean allowsLaunchOf(final QueryContext candidateQuery,
+ EstimatedImmutableQueryCollection launchedQueries) {
+ final LensDriver selectedDriver = candidateQuery.getSelectedDriver();
+ final boolean canLaunch = (selectedDriver instanceof JDBCDriver)
+ && (((JDBCDriver) selectedDriver).getQueryContextMap().size() < poolMaxSize);
+
+ log.debug("canLaunch:{}", canLaunch);
+ return canLaunch;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/lens/blob/5c25e68f/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraintFactory.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraintFactory.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraintFactory.java
new file mode 100644
index 0000000..477e197
--- /dev/null
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraintFactory.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.driver.jdbc;
+
+import static org.apache.lens.driver.jdbc.JDBCDriverConfConstants.ConnectionPoolProperties.JDBC_POOL_MAX_SIZE;
+
+import org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory;
+import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint;
+
+import org.apache.hadoop.conf.Configuration;
+
+public class MaxJDBCConnectionCheckConstraintFactory implements
+ ConfigBasedObjectCreationFactory<QueryLaunchingConstraint> {
+
+ @Override
+ public MaxJDBCConnectionCheckConstraint create(Configuration conf) {
+ final int poolMaxSize = Integer.parseInt(conf.get(JDBC_POOL_MAX_SIZE.getConfigKey()));
+
+ return new MaxJDBCConnectionCheckConstraint(poolMaxSize);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/5c25e68f/lens-driver-jdbc/src/main/resources/jdbcdriver-default.xml
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/resources/jdbcdriver-default.xml b/lens-driver-jdbc/src/main/resources/jdbcdriver-default.xml
index b5b6164..b446f7a 100644
--- a/lens-driver-jdbc/src/main/resources/jdbcdriver-default.xml
+++ b/lens-driver-jdbc/src/main/resources/jdbcdriver-default.xml
@@ -228,7 +228,8 @@
<property>
<name>lens.driver.jdbc.query.launching.constraint.factories</name>
- <value>org.apache.lens.server.api.query.constraint.MaxConcurrentDriverQueriesConstraintFactory</value>
+ <value>org.apache.lens.server.api.query.constraint.MaxConcurrentDriverQueriesConstraintFactory,
+ org.apache.lens.driver.jdbc.MaxJDBCConnectionCheckConstraintFactory</value>
<description>Factories used to instantiate constraints enforced on queries by driver. A query will be launched
only if all constraints pass. Every Factory should be an implementation of
org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of
http://git-wip-us.apache.org/repos/asf/lens/blob/5c25e68f/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
index 67f6c1f..491aa69 100644
--- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
+++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
@@ -411,6 +411,43 @@ public class TestJdbcDriver {
}
}
+ @Test
+ public void testJDBCMaxConnectionConstraintCheck() throws Exception {
+ close();
+ // Create table execute_test
+ createTable("max_connection_test");
+ // Insert some data into table
+ insertData("max_connection_test");
+
+ MaxJDBCConnectionCheckConstraintFactory factory = new MaxJDBCConnectionCheckConstraintFactory();
+ MaxJDBCConnectionCheckConstraint constraint = factory.create(driver.getConf());
+
+ // check constraint in driver
+ assertTrue(driver.getQueryConstraints().toString().contains("MaxJDBCConnectionCheckConstraint"));
+
+ String query;
+ QueryContext context = createQueryContext("SELECT * FROM max_connection_test");
+
+ for (int i = 1; i <= JDBC_POOL_MAX_SIZE.getDefaultValue(); i++) {
+ query = "SELECT " + i + " FROM max_connection_test";
+ context = createQueryContext(query);
+ driver.executeAsync(context);
+ }
+
+ //pool max size is same as number of query context hold on driver
+ assertEquals(driver.getQueryContextMap().size(), JDBC_POOL_MAX_SIZE.getDefaultValue());
+
+ //new query shouldn't be allowed
+ QueryContext newcontext = createQueryContext("SELECT 123 FROM max_connection_test");
+ assertFalse(constraint.allowsLaunchOf(newcontext, null));
+
+ //close one query and launch the previous query again
+ driver.closeQuery(context.getQueryHandle());
+ assertTrue(constraint.allowsLaunchOf(newcontext, null));
+ close();
+ }
+
+
/**
* Data provider for test case {@link #testExecuteWithPreFetch()}
* @return
http://git-wip-us.apache.org/repos/asf/lens/blob/5c25e68f/lens-server-api/src/main/java/org/apache/lens/server/api/util/LensUtil.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/util/LensUtil.java b/lens-server-api/src/main/java/org/apache/lens/server/api/util/LensUtil.java
index a0691a5..8261d8a 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/util/LensUtil.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/util/LensUtil.java
@@ -85,7 +85,7 @@ public final class LensUtil {
for (String factoryName : factoryNames) {
if (StringUtils.isNotBlank(factoryName)) {
- final T implementation = getImplementation(factoryName, conf);
+ final T implementation = getImplementation(factoryName.trim(), conf);
implSet.add(implementation);
}
}