You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by pb...@apache.org on 2018/03/30 07:59:23 UTC
[1/3] phoenix git commit: PHOENIX-4675 Better parsing around the
allowed UDF jar directory configuration
Repository: phoenix
Updated Branches:
refs/heads/4.x-cdh5.11 881d7aacb -> 0fa6d947e
PHOENIX-4675 Better parsing around the allowed UDF jar directory configuration
The parsing/validation logic on the allowed path for UDF jar loading was lacking
in that it didn't correctly handle a trailing slash.
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/bd4d15bb
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/bd4d15bb
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/bd4d15bb
Branch: refs/heads/4.x-cdh5.11
Commit: bd4d15bbd8c562a0a24b72989477cf22cc481ef5
Parents: 881d7aa
Author: Josh Elser <el...@apache.org>
Authored: Wed Mar 28 00:06:45 2018 +0100
Committer: Pedro Boado <pb...@apache.org>
Committed: Fri Mar 30 08:58:23 2018 +0100
----------------------------------------------------------------------
.../phoenix/end2end/UserDefinedFunctionsIT.java | 42 ++++++++++----------
.../expression/function/UDFExpression.java | 21 ++++++----
2 files changed, 35 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/bd4d15bb/phoenix-core/src/it/java/org/apache/phoenix/end2end/UserDefinedFunctionsIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UserDefinedFunctionsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UserDefinedFunctionsIT.java
index 943119d..ebb2462 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UserDefinedFunctionsIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UserDefinedFunctionsIT.java
@@ -201,7 +201,6 @@ public class UserDefinedFunctionsIT extends BaseOwnClusterIT {
public void cleanUpAfterTest() throws Exception {
Connection conn = driver.connect(url, EMPTY_PROPS);
Statement stmt = conn.createStatement();
- ResultSet rs = stmt.executeQuery("list jars");
stmt.execute("delete jar '"+ util.getConfiguration().get(QueryServices.DYNAMIC_JARS_DIR_KEY)+"/"+"myjar1.jar'");
stmt.execute("delete jar '"+ util.getConfiguration().get(QueryServices.DYNAMIC_JARS_DIR_KEY)+"/"+"myjar2.jar'");
stmt.execute("delete jar '"+ util.getConfiguration().get(QueryServices.DYNAMIC_JARS_DIR_KEY)+"/"+"myjar3.jar'");
@@ -280,7 +279,8 @@ public class UserDefinedFunctionsIT extends BaseOwnClusterIT {
util.startMiniDFSCluster(1);
util.startMiniZKCluster(1);
String string = util.getConfiguration().get("fs.defaultFS");
- conf.set(DYNAMIC_JARS_DIR_KEY, string+"/hbase/tmpjars");
+ // PHOENIX-4675 setting the trailing slash implicitly tests that we're doing some path normalization
+ conf.set(DYNAMIC_JARS_DIR_KEY, string+"/hbase/tmpjars/");
util.startMiniHBaseCluster(1, 1);
UDFExpression.setConfig(conf);
@@ -297,20 +297,21 @@ public class UserDefinedFunctionsIT extends BaseOwnClusterIT {
@Test
public void testListJars() throws Exception {
Connection conn = driver.connect(url, EMPTY_PROPS);
+ Path jarPath = new Path(util.getConfiguration().get(QueryServices.DYNAMIC_JARS_DIR_KEY));
Statement stmt = conn.createStatement();
ResultSet rs = stmt.executeQuery("list jars");
assertTrue(rs.next());
- assertEquals(util.getConfiguration().get(QueryServices.DYNAMIC_JARS_DIR_KEY)+"/"+"myjar1.jar", rs.getString("jar_location"));
+ assertEquals(new Path(jarPath, "myjar1.jar").toString(), rs.getString("jar_location"));
assertTrue(rs.next());
- assertEquals(util.getConfiguration().get(QueryServices.DYNAMIC_JARS_DIR_KEY)+"/"+"myjar2.jar", rs.getString("jar_location"));
+ assertEquals(new Path(jarPath, "myjar2.jar").toString(), rs.getString("jar_location"));
assertTrue(rs.next());
- assertEquals(util.getConfiguration().get(QueryServices.DYNAMIC_JARS_DIR_KEY)+"/"+"myjar3.jar", rs.getString("jar_location"));
+ assertEquals(new Path(jarPath, "myjar3.jar").toString(), rs.getString("jar_location"));
assertTrue(rs.next());
- assertEquals(util.getConfiguration().get(QueryServices.DYNAMIC_JARS_DIR_KEY)+"/"+"myjar4.jar", rs.getString("jar_location"));
+ assertEquals(new Path(jarPath, "myjar4.jar").toString(), rs.getString("jar_location"));
assertTrue(rs.next());
- assertEquals(util.getConfiguration().get(QueryServices.DYNAMIC_JARS_DIR_KEY)+"/"+"myjar5.jar", rs.getString("jar_location"));
+ assertEquals(new Path(jarPath, "myjar5.jar").toString(), rs.getString("jar_location"));
assertTrue(rs.next());
- assertEquals(util.getConfiguration().get(QueryServices.DYNAMIC_JARS_DIR_KEY)+"/"+"myjar6.jar", rs.getString("jar_location"));
+ assertEquals(new Path(jarPath, "myjar6.jar").toString(), rs.getString("jar_location"));
assertFalse(rs.next());
}
@@ -320,30 +321,31 @@ public class UserDefinedFunctionsIT extends BaseOwnClusterIT {
Statement stmt = conn.createStatement();
ResultSet rs = stmt.executeQuery("list jars");
assertTrue(rs.next());
- assertEquals(util.getConfiguration().get(QueryServices.DYNAMIC_JARS_DIR_KEY)+"/"+"myjar1.jar", rs.getString("jar_location"));
+ Path jarPath = new Path(util.getConfiguration().get(QueryServices.DYNAMIC_JARS_DIR_KEY));
+ assertEquals(new Path(jarPath, "myjar1.jar").toString(), rs.getString("jar_location"));
assertTrue(rs.next());
- assertEquals(util.getConfiguration().get(QueryServices.DYNAMIC_JARS_DIR_KEY)+"/"+"myjar2.jar", rs.getString("jar_location"));
+ assertEquals(new Path(jarPath, "myjar2.jar").toString(), rs.getString("jar_location"));
assertTrue(rs.next());
- assertEquals(util.getConfiguration().get(QueryServices.DYNAMIC_JARS_DIR_KEY)+"/"+"myjar3.jar", rs.getString("jar_location"));
+ assertEquals(new Path(jarPath, "myjar3.jar").toString(), rs.getString("jar_location"));
assertTrue(rs.next());
- assertEquals(util.getConfiguration().get(QueryServices.DYNAMIC_JARS_DIR_KEY)+"/"+"myjar4.jar", rs.getString("jar_location"));
+ assertEquals(new Path(jarPath, "myjar4.jar").toString(), rs.getString("jar_location"));
assertTrue(rs.next());
- assertEquals(util.getConfiguration().get(QueryServices.DYNAMIC_JARS_DIR_KEY)+"/"+"myjar5.jar", rs.getString("jar_location"));
+ assertEquals(new Path(jarPath, "myjar5.jar").toString(), rs.getString("jar_location"));
assertTrue(rs.next());
- assertEquals(util.getConfiguration().get(QueryServices.DYNAMIC_JARS_DIR_KEY)+"/"+"myjar6.jar", rs.getString("jar_location"));
+ assertEquals(new Path(jarPath, "myjar6.jar").toString(), rs.getString("jar_location"));
assertFalse(rs.next());
- stmt.execute("delete jar '"+ util.getConfiguration().get(QueryServices.DYNAMIC_JARS_DIR_KEY)+"/"+"myjar4.jar'");
+ stmt.execute("delete jar '"+ new Path(jarPath, "myjar4.jar").toString() + "'");
rs = stmt.executeQuery("list jars");
assertTrue(rs.next());
- assertEquals(util.getConfiguration().get(QueryServices.DYNAMIC_JARS_DIR_KEY)+"/"+"myjar1.jar", rs.getString("jar_location"));
+ assertEquals(new Path(jarPath, "myjar1.jar").toString(), rs.getString("jar_location"));
assertTrue(rs.next());
- assertEquals(util.getConfiguration().get(QueryServices.DYNAMIC_JARS_DIR_KEY)+"/"+"myjar2.jar", rs.getString("jar_location"));
+ assertEquals(new Path(jarPath, "myjar2.jar").toString(), rs.getString("jar_location"));
assertTrue(rs.next());
- assertEquals(util.getConfiguration().get(QueryServices.DYNAMIC_JARS_DIR_KEY)+"/"+"myjar3.jar", rs.getString("jar_location"));
+ assertEquals(new Path(jarPath, "myjar3.jar").toString(), rs.getString("jar_location"));
assertTrue(rs.next());
- assertEquals(util.getConfiguration().get(QueryServices.DYNAMIC_JARS_DIR_KEY)+"/"+"myjar5.jar", rs.getString("jar_location"));
+ assertEquals(new Path(jarPath, "myjar5.jar").toString(), rs.getString("jar_location"));
assertTrue(rs.next());
- assertEquals(util.getConfiguration().get(QueryServices.DYNAMIC_JARS_DIR_KEY)+"/"+"myjar6.jar", rs.getString("jar_location"));
+ assertEquals(new Path(jarPath, "myjar6.jar").toString(), rs.getString("jar_location"));
assertFalse(rs.next());
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/bd4d15bb/phoenix-core/src/main/java/org/apache/phoenix/expression/function/UDFExpression.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/UDFExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/UDFExpression.java
index c8636fd..80a569a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/UDFExpression.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/UDFExpression.java
@@ -173,21 +173,18 @@ public class UDFExpression extends ScalarFunction {
public static DynamicClassLoader getClassLoader(final PName tenantId, final String jarPath) {
DynamicClassLoader cl = tenantIdSpecificCls.get(tenantId);
- String parent = null;
+ Path parent = null;
if (cl != null) return cl;
if(jarPath != null && !jarPath.isEmpty()) {
cl = pathSpecificCls.get(jarPath);
if (cl != null) return cl;
- Path path = new Path(jarPath);
- if(jarPath.endsWith(".jar")) {
- parent = path.getParent().toString();
- } else {
- parent = path.toString();
- }
+ parent = getPathForParent(jarPath);
}
+ // Parse the DYNAMIC_JARS_DIR_KEY value as a Path if it's present in the configuration
+ Path allowedDynamicJarsPath = config.get(DYNAMIC_JARS_DIR_KEY) != null ? new Path(config.get(DYNAMIC_JARS_DIR_KEY)) : null;
// The case jarPath is not provided, or it is provided and the jar is inside hbase.dynamic.jars.dir
if (jarPath == null || jarPath.isEmpty()
- || config.get(DYNAMIC_JARS_DIR_KEY) != null && (parent != null && parent.equals(config.get(DYNAMIC_JARS_DIR_KEY)))) {
+ || (allowedDynamicJarsPath != null && parent != null && parent.equals(allowedDynamicJarsPath))) {
cl = tenantIdSpecificCls.get(tenantId);
if (cl == null) {
cl = new DynamicClassLoader(config, UDFExpression.class.getClassLoader());
@@ -204,6 +201,14 @@ public class UDFExpression extends ScalarFunction {
throw new SecurityException("Loading jars from " + jarPath + " is not allowed. The only location that is allowed is "+ config.get(DYNAMIC_JARS_DIR_KEY));
}
}
+
+ public static Path getPathForParent(String jarPath) {
+ Path path = new Path(jarPath);
+ if (jarPath.endsWith(".jar")) {
+ return path.getParent();
+ }
+ return path;
+ }
@VisibleForTesting
public static void setConfig(Configuration conf) {
[2/3] phoenix git commit: PHOENIX-4677 Add explicit commons-cli
dependency
Posted by pb...@apache.org.
PHOENIX-4677 Add explicit commons-cli dependency
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/235def6a
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/235def6a
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/235def6a
Branch: refs/heads/4.x-cdh5.11
Commit: 235def6aa8a096f2f2ddbfc466143e75aca29676
Parents: bd4d15b
Author: Josh Elser <el...@apache.org>
Authored: Wed Mar 28 20:21:07 2018 +0100
Committer: Pedro Boado <pb...@apache.org>
Committed: Fri Mar 30 08:58:35 2018 +0100
----------------------------------------------------------------------
phoenix-core/pom.xml | 4 ++++
phoenix-kafka/pom.xml | 4 ++++
phoenix-pherf/pom.xml | 4 ++++
phoenix-pig/pom.xml | 4 ++++
pom.xml | 6 +++++-
5 files changed, 21 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/235def6a/phoenix-core/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-core/pom.xml b/phoenix-core/pom.xml
index 2cb4c81..b07cbbb 100644
--- a/phoenix-core/pom.xml
+++ b/phoenix-core/pom.xml
@@ -355,6 +355,10 @@
<artifactId>htrace-core</artifactId>
</dependency>
<dependency>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ </dependency>
+ <dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/235def6a/phoenix-kafka/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-kafka/pom.xml b/phoenix-kafka/pom.xml
index c2cb7db..8718f5f 100644
--- a/phoenix-kafka/pom.xml
+++ b/phoenix-kafka/pom.xml
@@ -153,6 +153,10 @@
<artifactId>htrace-core</artifactId>
</dependency>
<dependency>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ </dependency>
+ <dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/235def6a/phoenix-pherf/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-pherf/pom.xml b/phoenix-pherf/pom.xml
index 6741bf3..7831f35 100644
--- a/phoenix-pherf/pom.xml
+++ b/phoenix-pherf/pom.xml
@@ -60,6 +60,10 @@
<artifactId>commons-math3</artifactId>
<version>3.3</version>
</dependency>
+ <dependency>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ </dependency>
<!-- Test Dependencies -->
<dependency>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/235def6a/phoenix-pig/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-pig/pom.xml b/phoenix-pig/pom.xml
index 31f38b8..e5d0d52 100644
--- a/phoenix-pig/pom.xml
+++ b/phoenix-pig/pom.xml
@@ -172,6 +172,10 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/235def6a/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 1479b75..73964c0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -122,7 +122,6 @@
<log4j.version>1.2.17</log4j.version>
<slf4j.version>${cdh.slf4j.version}</slf4j.version>
<protobuf-java.version>2.5.0</protobuf-java.version>
- <commons-configuration.version>1.6</commons-configuration.version>
<commons-io.version>2.1</commons-io.version>
<commons-lang.version>2.5</commons-lang.version>
<commons-logging.version>1.2</commons-logging.version>
@@ -936,6 +935,11 @@
<version>${slf4j.version}</version>
</dependency>
<dependency>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ <version>${commons-cli.version}</version>
+ </dependency>
+ <dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
<version>${commons-logging.version}</version>
[3/3] phoenix git commit: PHOENIX-4671 Fix minor size accounting bug
for MutationSize.
Posted by pb...@apache.org.
PHOENIX-4671 Fix minor size accounting bug for MutationSize.
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/0fa6d947
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/0fa6d947
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/0fa6d947
Branch: refs/heads/4.x-cdh5.11
Commit: 0fa6d947ebfc237e854b206f35af3a558e28a8ee
Parents: 235def6
Author: Lars Hofhansl <la...@apache.org>
Authored: Thu Mar 29 22:42:53 2018 +0100
Committer: Pedro Boado <pb...@apache.org>
Committed: Fri Mar 30 08:58:50 2018 +0100
----------------------------------------------------------------------
.../end2end/UpsertSelectAutoCommitIT.java | 28 ++++++++++++++++++++
.../apache/phoenix/execute/MutationState.java | 1 +
2 files changed, 29 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/0fa6d947/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectAutoCommitIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectAutoCommitIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectAutoCommitIT.java
index 6b781a0..38d48d6 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectAutoCommitIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectAutoCommitIT.java
@@ -23,15 +23,19 @@ import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.sql.Connection;
import java.sql.Date;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
+import java.sql.SQLException;
import java.sql.Statement;
import java.util.Properties;
+import org.apache.phoenix.exception.SQLExceptionCode;
+import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.util.PropertiesUtil;
import org.junit.Test;
@@ -173,4 +177,28 @@ public class UpsertSelectAutoCommitIT extends ParallelStatsDisabledIT {
conn.close();
}
+ @Test
+ public void testMaxMutationSize() throws Exception {
+ Properties connectionProperties = new Properties();
+ connectionProperties.setProperty(QueryServices.MAX_MUTATION_SIZE_ATTRIB, "3");
+ connectionProperties.setProperty(QueryServices.MAX_MUTATION_SIZE_BYTES_ATTRIB, "50000");
+ PhoenixConnection connection =
+ (PhoenixConnection) DriverManager.getConnection(getUrl(), connectionProperties);
+ connection.setAutoCommit(true);
+ String fullTableName = generateUniqueName();
+ try (Statement stmt = connection.createStatement()) {
+ stmt.execute(
+ "CREATE TABLE " + fullTableName + " (pk INTEGER PRIMARY KEY, v1 INTEGER, v2 INTEGER)");
+ stmt.execute(
+ "CREATE SEQUENCE " + fullTableName + "_seq cache 1000");
+ stmt.execute("UPSERT INTO " + fullTableName + " VALUES (NEXT VALUE FOR " + fullTableName + "_seq, rand(), rand())");
+ }
+ try (Statement stmt = connection.createStatement()) {
+ for (int i=0; i<16; i++) {
+ stmt.execute("UPSERT INTO " + fullTableName + " SELECT NEXT VALUE FOR " + fullTableName + "_seq, rand(), rand() FROM " + fullTableName);
+ }
+ }
+ connection.close();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/0fa6d947/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
index 727b424..f6d11a0 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
@@ -1563,6 +1563,7 @@ public class MutationState implements SQLCloseable {
public void clear(){
rowKeyToRowMutationState.clear();
+ estimatedSize = 0;
}
public Collection<RowMutationState> values() {