You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ja...@apache.org on 2018/05/09 20:51:08 UTC
[1/7] phoenix git commit: PHOENIX-4732 Improve config defaults for
async secondary index MR jobs
Repository: phoenix
Updated Branches:
refs/heads/5.x-HBase-2.0 254c0a2ae -> ff0b8089d
PHOENIX-4732 Improve config defaults for async secondary index MR jobs
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/933ef6aa
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/933ef6aa
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/933ef6aa
Branch: refs/heads/5.x-HBase-2.0
Commit: 933ef6aaeeccc98c9a62c49c01c4cf71547c614f
Parents: 254c0a2
Author: James Taylor <jt...@salesforce.com>
Authored: Tue May 8 15:26:44 2018 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Wed May 9 13:44:20 2018 -0700
----------------------------------------------------------------------
.../index/automation/PhoenixMRJobSubmitter.java | 46 +++++++++++++++++++-
.../apache/phoenix/util/PhoenixMRJobUtil.java | 18 +++-----
2 files changed, 51 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/933ef6aa/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/automation/PhoenixMRJobSubmitter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/automation/PhoenixMRJobSubmitter.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/automation/PhoenixMRJobSubmitter.java
index 662f0c9..59ced6c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/automation/PhoenixMRJobSubmitter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/automation/PhoenixMRJobSubmitter.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
+import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.mapreduce.index.IndexTool;
@@ -69,8 +70,28 @@ public class PhoenixMRJobSubmitter {
// Lock to elect a master node that submits the Phoenix Secondary Index MR Jobs
private static final String PHOENIX_LOCKS_PARENT =
"/phoenix/automated-mr-index-build-leader-election";
+ public static final String PHOENIX_MR_CLIENT_SCANNER_TIMEOUT_PERIOD =
+ "phoenix.mr.client.scanner.timeout.period";
+ public static final String PHOENIX_MR_RPC_TIMEOUT =
+ "phoenix.mr.rpc.timeout";
+ public static final String PHOENIX_MR_TASK_TIMEOUT =
+ "phoenix.mr.task.timeout";
+ public static final String PHOENIX_MR_CLIENT_RETRIES_NUMBER =
+ "phoenix.mr.client.retries.number";
+ public static final String PHOENIX_MR_CLIENT_PAUSE =
+ "phoenix.mr.client.retries.number";
+ public static final String PHOENIX_MR_ZK_RECOVERY_RETRY =
+ "phoenix.mr.zk.recovery.retry";
private static final String AUTO_INDEX_BUILD_LOCK_NAME = "ActiveStandbyElectorLock";
-
+ private static final int DEFAULT_TIMEOUT_IN_MILLIS = 600000;
+ public static final int DEFAULT_MR_CLIENT_SCANNER_TIMEOUT_PERIOD = DEFAULT_TIMEOUT_IN_MILLIS;
+ public static final int DEFAULT_MR_RPC_TIMEOUT = DEFAULT_TIMEOUT_IN_MILLIS;
+ public static final int DEFAULT_MR_TASK_TIMEOUT = DEFAULT_TIMEOUT_IN_MILLIS;
+ // Reduced HBase/Zookeeper Client Retries
+ public static final int DEFAULT_MR_CLIENT_RETRIES_NUMBER = 10;
+ public static final int DEFAULT_MR_CLIENT_PAUSE = 1000;
+ public static final int DEFAULT_MR_ZK_RECOVERY_RETRY = 1;
+
public static final String CANDIDATE_INDEX_INFO_QUERY = "SELECT "
+ PhoenixDatabaseMetaData.INDEX_TYPE + ","
+ PhoenixDatabaseMetaData.DATA_TABLE_NAME + ", "
@@ -103,7 +124,28 @@ public class PhoenixMRJobSubmitter {
}
this.conf = conf;
- PhoenixMRJobUtil.updateTimeoutsToFailFast(conf);
+ // Have Phoenix specific properties for defaults to enable potential override
+ conf.setLong(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD,
+ conf.getLong(PHOENIX_MR_CLIENT_SCANNER_TIMEOUT_PERIOD,
+ DEFAULT_MR_CLIENT_SCANNER_TIMEOUT_PERIOD));
+ conf.setLong(HConstants.HBASE_RPC_TIMEOUT_KEY,
+ conf.getLong(PHOENIX_MR_RPC_TIMEOUT,
+ DEFAULT_MR_RPC_TIMEOUT));
+ conf.setLong(MRJobConfig.TASK_TIMEOUT,
+ conf.getLong(PHOENIX_MR_TASK_TIMEOUT,
+ DEFAULT_MR_TASK_TIMEOUT));
+
+ // Reduced HBase Client Retries
+ conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
+ conf.getInt(PHOENIX_MR_CLIENT_RETRIES_NUMBER,
+ DEFAULT_MR_CLIENT_RETRIES_NUMBER));
+ conf.setInt(HConstants.HBASE_CLIENT_PAUSE,
+ conf.getInt(PHOENIX_MR_CLIENT_PAUSE,
+ DEFAULT_MR_CLIENT_PAUSE));
+ conf.setInt("zookeeper.recovery.retry",
+ conf.getInt(PHOENIX_MR_ZK_RECOVERY_RETRY,
+ DEFAULT_MR_ZK_RECOVERY_RETRY));
+
String schedulerType =
conf.get(PhoenixMRJobUtil.PHOENIX_MR_SCHEDULER_TYPE_NAME,
MR_SCHEDULER_TYPE.NONE.toString());
http://git-wip-us.apache.org/repos/asf/phoenix/blob/933ef6aa/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixMRJobUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixMRJobUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixMRJobUtil.java
index 74bbaad..bf0d03b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixMRJobUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixMRJobUtil.java
@@ -52,19 +52,17 @@ public class PhoenixMRJobUtil {
private static final String ACTIVE_STANDBY_ELECTOR_LOCK = "ActiveStandbyElectorLock";
private static final String RM_APPS_GET_ENDPOINT = "/ws/v1/cluster/apps";
- // Reduced HBase Client Retries
- private static final int CLIENT_RETRIES_NUMBER = 2;
- private static final long CLIENT_PAUSE_TIME = 1000;
- private static final int ZOOKEEPER_RECOVERY_RETRY_COUNT = 1;
-
public static final String PHOENIX_INDEX_MR_QUEUE_NAME_PROPERTY =
"phoenix.index.mr.scheduler.capacity.queuename";
public static final String PHOENIX_INDEX_MR_MAP_MEMORY_PROPERTY =
"phoenix.index.mr.scheduler.capacity.mapMemoryMB";
+ public static final String PHOENIX_MR_CONCURRENT_MAP_LIMIT_PROPERTY =
+ "phoenix.mr.concurrent.map.limit";
// Default MR Capacity Scheduler Configurations for Phoenix MR Index Build
// Jobs
public static final String DEFAULT_QUEUE_NAME = "default";
+ public static final int DEFAULT_MR_CONCURRENT_MAP_LIMIT = 20;
public static final int DEFAULT_MAP_MEMROY_MB = 5120;
public static final String XMX_OPT = "-Xmx";
@@ -208,12 +206,6 @@ public class PhoenixMRJobUtil {
return rmPort;
}
- public static void updateTimeoutsToFailFast(Configuration conf) {
- conf.set("hbase.client.retries.number", String.valueOf(CLIENT_RETRIES_NUMBER));
- conf.set("zookeeper.recovery.retry", String.valueOf(ZOOKEEPER_RECOVERY_RETRY_COUNT));
- conf.set("hbase.client.pause", String.valueOf(CLIENT_PAUSE_TIME));
- }
-
/**
* This method set the configuration values for Capacity scheduler.
* @param conf - Configuration to which Capacity Queue information to be added
@@ -221,6 +213,10 @@ public class PhoenixMRJobUtil {
public static void updateCapacityQueueInfo(Configuration conf) {
conf.set(MRJobConfig.QUEUE_NAME,
conf.get(PHOENIX_INDEX_MR_QUEUE_NAME_PROPERTY, DEFAULT_QUEUE_NAME));
+
+ conf.setInt(MRJobConfig.JOB_RUNNING_MAP_LIMIT,
+ conf.getInt(PHOENIX_MR_CONCURRENT_MAP_LIMIT_PROPERTY, DEFAULT_MR_CONCURRENT_MAP_LIMIT));
+
int mapMemoryMB = conf.getInt(PHOENIX_INDEX_MR_MAP_MEMORY_PROPERTY, DEFAULT_MAP_MEMROY_MB);
conf.setInt(MRJobConfig.MAP_MEMORY_MB, mapMemoryMB);
[3/7] phoenix git commit: PHOENIX-3856 StatementContext class
constructor not honouring supplied scan object (Sukumar Maddineni)
Posted by ja...@apache.org.
PHOENIX-3856 StatementContext class constructor not honouring supplied scan object (Sukumar Maddineni)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/23b0faa8
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/23b0faa8
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/23b0faa8
Branch: refs/heads/5.x-HBase-2.0
Commit: 23b0faa87333040d5634903c5c1d4c8405d75c41
Parents: 933ef6a
Author: James Taylor <jt...@salesforce.com>
Authored: Thu May 3 09:32:57 2018 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Wed May 9 13:44:22 2018 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/phoenix/compile/StatementContext.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/23b0faa8/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
index c105046..3e5c8f2 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
@@ -98,7 +98,7 @@ public class StatementContext {
}
public StatementContext(PhoenixStatement statement, Scan scan) {
- this(statement, FromCompiler.EMPTY_TABLE_RESOLVER, new Scan(), new SequenceManager(statement));
+ this(statement, FromCompiler.EMPTY_TABLE_RESOLVER, scan, new SequenceManager(statement));
}
public StatementContext(PhoenixStatement statement, ColumnResolver resolver) {
[2/7] phoenix git commit: PHOENIX-3270 Remove @Ignore tag for
TransactionIT.testNonTxToTxTableFailure()
Posted by ja...@apache.org.
PHOENIX-3270 Remove @Ignore tag for TransactionIT.testNonTxToTxTableFailure()
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/294ff589
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/294ff589
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/294ff589
Branch: refs/heads/5.x-HBase-2.0
Commit: 294ff5898b4e1b5c0de4aa6b261cc83fdfd349ee
Parents: 23b0faa
Author: James Taylor <jt...@salesforce.com>
Authored: Thu May 3 09:52:57 2018 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Wed May 9 13:44:22 2018 -0700
----------------------------------------------------------------------
.../it/java/org/apache/phoenix/tx/ParameterizedTransactionIT.java | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/294ff589/phoenix-core/src/it/java/org/apache/phoenix/tx/ParameterizedTransactionIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/tx/ParameterizedTransactionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/tx/ParameterizedTransactionIT.java
index ee3a600..099808c 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/tx/ParameterizedTransactionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/tx/ParameterizedTransactionIT.java
@@ -58,7 +58,6 @@ import org.apache.phoenix.transaction.PhoenixTransactionContext;
import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.TestUtil;
-import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@@ -325,7 +324,6 @@ public class ParameterizedTransactionIT extends ParallelStatsDisabledIT {
assertFalse(rs.next());
}
- @Ignore
@Test
public void testNonTxToTxTableFailure() throws Exception {
String nonTxTableName = generateUniqueName();
[6/7] phoenix git commit: PHOENIX-4678 ndexScrutinyTool generates
malformed query due to incorrect table name(s) (Josh Elser)
Posted by ja...@apache.org.
PHOENIX-4678 ndexScrutinyTool generates malformed query due to incorrect table name(s) (Josh Elser)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/9516b14a
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/9516b14a
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/9516b14a
Branch: refs/heads/5.x-HBase-2.0
Commit: 9516b14ab3103c387bd06f1938bdd33f45e893f1
Parents: 5d53215
Author: James Taylor <jt...@salesforce.com>
Authored: Wed May 9 12:50:22 2018 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Wed May 9 13:49:36 2018 -0700
----------------------------------------------------------------------
phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9516b14a/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
index 0168bee..51d583c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
@@ -1132,7 +1132,7 @@ public class SchemaUtil {
* @return the byte representation of the HTable
*/
public static String getQualifiedTableName(String schemaName, String tableName) {
- if (schemaName != null) {
+ if (schemaName != null && !schemaName.isEmpty()) {
return String.format("%s.%s", normalizeIdentifier(schemaName),
normalizeIdentifier(tableName));
} else {
[5/7] phoenix git commit: PHOENIX-4623 Inconsistent physical view
index name
Posted by ja...@apache.org.
PHOENIX-4623 Inconsistent physical view index name
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/5d532156
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/5d532156
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/5d532156
Branch: refs/heads/5.x-HBase-2.0
Commit: 5d53215696912bf2557867cd1ac334f84b6492fd
Parents: 33a6d9d
Author: James Taylor <jt...@salesforce.com>
Authored: Tue May 8 16:29:41 2018 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Wed May 9 13:49:36 2018 -0700
----------------------------------------------------------------------
.../org/apache/phoenix/end2end/BaseViewIT.java | 7 +++----
.../java/org/apache/phoenix/end2end/ViewIT.java | 2 +-
.../phoenix/schema/stats/StatsCollectorIT.java | 2 +-
.../org/apache/phoenix/util/MetaDataUtil.java | 18 +++++-------------
.../java/org/apache/phoenix/util/UpgradeUtil.java | 2 +-
5 files changed, 11 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5d532156/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java
index 92966f1..b606fc6 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java
@@ -46,7 +46,6 @@ import org.apache.phoenix.util.MetaDataUtil;
import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.ScanUtil;
import org.apache.phoenix.util.SchemaUtil;
-import org.apache.phoenix.util.TestUtil;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
@@ -66,7 +65,7 @@ public abstract class BaseViewIT extends ParallelStatsEnabledIT {
if (transactional) {
optionBuilder.append(" TRANSACTIONAL=true ");
}
- this.schemaName = TestUtil.DEFAULT_SCHEMA_NAME;
+ this.schemaName = "S_" + generateUniqueName();
this.tableDDLOptions = optionBuilder.toString();
this.tableName = "T_" + generateUniqueName();
this.fullTableName = SchemaUtil.getTableName(schemaName, tableName);
@@ -167,7 +166,7 @@ public abstract class BaseViewIT extends ParallelStatsEnabledIT {
ResultSet rs;
Connection conn = DriverManager.getConnection(getUrl());
String viewIndexName1 = "I_" + generateUniqueName();
- String viewIndexPhysicalName = MetaDataUtil.getViewIndexName(schemaName, tableName);
+ String viewIndexPhysicalName = MetaDataUtil.getViewIndexPhysicalName(fullTableName);
if (localIndex) {
conn.createStatement().execute("CREATE LOCAL INDEX " + viewIndexName1 + " on " + viewName + "(k3)");
} else {
@@ -232,7 +231,7 @@ public abstract class BaseViewIT extends ParallelStatsEnabledIT {
rs = conn.createStatement().executeQuery("EXPLAIN " + query);
String physicalTableName;
if (localIndex) {
- physicalTableName = tableName;
+ physicalTableName = fullTableName;
assertEquals("CLIENT PARALLEL "+ (saltBuckets == null ? 1 : saltBuckets) +"-WAY RANGE SCAN OVER " + fullTableName +" [" + (2) + ",'foo']\n"
+ " SERVER FILTER BY FIRST KEY ONLY\n"
+ "CLIENT MERGE SORT",QueryUtil.getExplainPlan(rs));
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5d532156/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIT.java
index e277c18..72dd26f 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIT.java
@@ -69,7 +69,7 @@ public class ViewIT extends BaseViewIT {
String ddl = "CREATE TABLE " + fullTableName + " (k INTEGER NOT NULL PRIMARY KEY, v1 DATE) "+ tableDDLOptions;
conn.createStatement().execute(ddl);
String fullParentViewName = "V_" + generateUniqueName();
- ddl = "CREATE VIEW " + fullParentViewName + " (v2 VARCHAR) AS SELECT * FROM " + tableName + " WHERE k > 5";
+ ddl = "CREATE VIEW " + fullParentViewName + " (v2 VARCHAR) AS SELECT * FROM " + fullTableName + " WHERE k > 5";
conn.createStatement().execute(ddl);
try {
conn.createStatement().execute("UPSERT INTO " + fullParentViewName + " VALUES(1)");
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5d532156/phoenix-core/src/it/java/org/apache/phoenix/schema/stats/StatsCollectorIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/schema/stats/StatsCollectorIT.java b/phoenix-core/src/it/java/org/apache/phoenix/schema/stats/StatsCollectorIT.java
index f7a2edd..3af0d09 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/schema/stats/StatsCollectorIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/schema/stats/StatsCollectorIT.java
@@ -785,7 +785,7 @@ public abstract class StatsCollectorIT extends BaseUniqueNamesOwnClusterIT {
String viewIndex = "VI_" + generateUniqueName();
ddl = "CREATE INDEX " + viewIndex + " ON " + view + " (b)";
conn.createStatement().execute(ddl);
- String viewIndexTableName = MetaDataUtil.getViewIndexTableName(baseTable);
+ String viewIndexTableName = MetaDataUtil.getViewIndexPhysicalName(baseTable);
statsCollector = getDefaultStatsCollectorForTable(viewIndexTableName);
statsCollector.init();
assertEquals(defaultGuidePostWidth, statsCollector.getGuidePostDepth());
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5d532156/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
index e31d9ab..ca36a57 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
@@ -353,23 +353,15 @@ public class MetaDataUtil {
return getIndexPhysicalName(physicalTableName, VIEW_INDEX_TABLE_PREFIX);
}
- public static String getViewIndexTableName(String tableName) {
- return VIEW_INDEX_TABLE_PREFIX + tableName;
- }
-
- public static String getViewIndexSchemaName(String schemaName) {
- return schemaName;
- }
-
- public static String getViewIndexName(String schemaName, String tableName) {
- return SchemaUtil.getTableName(getViewIndexSchemaName(schemaName), getViewIndexTableName(tableName));
+ public static String getViewIndexPhysicalName(String physicalTableName) {
+ return getIndexPhysicalName(physicalTableName, VIEW_INDEX_TABLE_PREFIX);
}
- public static byte[] getIndexPhysicalName(byte[] physicalTableName, String indexPrefix) {
- return getIndexPhysicalName(Bytes.toString(physicalTableName), indexPrefix).getBytes();
+ private static byte[] getIndexPhysicalName(byte[] physicalTableName, String indexPrefix) {
+ return Bytes.toBytes(getIndexPhysicalName(Bytes.toString(physicalTableName), indexPrefix));
}
- public static String getIndexPhysicalName(String physicalTableName, String indexPrefix) {
+ private static String getIndexPhysicalName(String physicalTableName, String indexPrefix) {
if (physicalTableName.contains(QueryConstants.NAMESPACE_SEPARATOR)) {
String schemaName = SchemaUtil.getSchemaNameFromFullName(physicalTableName,
QueryConstants.NAMESPACE_SEPARATOR);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5d532156/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
index f57249c..1634159 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
@@ -604,7 +604,7 @@ public class UpgradeUtil {
viewPTable = PhoenixRuntime.getTable(globalConnection, fullViewName);
globalConnection.createStatement().execute(String.format(disableIndexDDL, indexName, fullViewName));
}
- String indexPhysicalTableName = MetaDataUtil.getViewIndexTableName(viewPTable.getPhysicalName().getString());
+ String indexPhysicalTableName = MetaDataUtil.getViewIndexPhysicalName(viewPTable.getPhysicalName().getString());
if (physicalTables.add(indexPhysicalTableName)) {
final TableName tableName = TableName.valueOf(indexPhysicalTableName);
if(admin.tableExists(tableName)) {
[4/7] phoenix git commit: PHOENIX-4727 Parameterize transaction tests
with TRANSACTION_PROVIDER
Posted by ja...@apache.org.
PHOENIX-4727 Parameterize transaction tests with TRANSACTION_PROVIDER
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/33a6d9d8
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/33a6d9d8
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/33a6d9d8
Branch: refs/heads/5.x-HBase-2.0
Commit: 33a6d9d86d61485393d245ec4c9d1ad39c82cb4f
Parents: 294ff58
Author: James Taylor <jt...@salesforce.com>
Authored: Mon May 7 18:23:03 2018 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Wed May 9 13:49:34 2018 -0700
----------------------------------------------------------------------
.../phoenix/end2end/index/MutableIndexIT.java | 22 +++---
.../phoenix/tx/ParameterizedTransactionIT.java | 1 +
.../org/apache/phoenix/tx/TransactionIT.java | 75 +++++++++++++++-----
.../java/org/apache/phoenix/util/TestUtil.java | 6 +-
4 files changed, 75 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/33a6d9d8/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java
index d69a4a5..a934fb6 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java
@@ -81,11 +81,11 @@ public class MutableIndexIT extends ParallelStatsDisabledIT {
protected final boolean localIndex;
private final String tableDDLOptions;
- public MutableIndexIT(boolean localIndex, boolean transactional, boolean columnEncoded) {
+ public MutableIndexIT(Boolean localIndex, String txProvider, Boolean columnEncoded) {
this.localIndex = localIndex;
StringBuilder optionBuilder = new StringBuilder();
- if (transactional) {
- optionBuilder.append("TRANSACTIONAL=true");
+ if (txProvider != null) {
+ optionBuilder.append("TRANSACTIONAL=true," + PhoenixDatabaseMetaData.TRANSACTION_PROVIDER + "='" + txProvider + "'");
}
if (!columnEncoded) {
if (optionBuilder.length()!=0)
@@ -107,13 +107,15 @@ public class MutableIndexIT extends ParallelStatsDisabledIT {
}
@Parameters(name="MutableIndexIT_localIndex={0},transactional={1},columnEncoded={2}") // name is used by failsafe as file name in reports
- public static Collection<Boolean[]> data() {
- // Temporarily disable transactional tests
- return Arrays.asList(new Boolean[][] {
- { false, false, false }, { false, false, true },
- //{ false, true, false }, { false, true, true },
- { true, false, false }, { true, false, true } });
- //{ true, true, false }, { true, true, true } });
+ public static Collection<Object[]> data() {
+ return Arrays.asList(new Object[][] {
+ { false, null, false }, { false, null, true },
+ //{ false, "TEPHRA", false }, { false, "TEPHRA", true },
+ //{ false, "OMID", false }, { false, "OMID", true },
+ { true, null, false }, { true, null, true },
+ //{ true, "TEPHRA", false }, { true, "TEPHRA", true },
+ //{ true, "OMID", false }, { true, "OMID", true },
+ });
}
@Test
http://git-wip-us.apache.org/repos/asf/phoenix/blob/33a6d9d8/phoenix-core/src/it/java/org/apache/phoenix/tx/ParameterizedTransactionIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/tx/ParameterizedTransactionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/tx/ParameterizedTransactionIT.java
index 099808c..749b056 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/tx/ParameterizedTransactionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/tx/ParameterizedTransactionIT.java
@@ -58,6 +58,7 @@ import org.apache.phoenix.transaction.PhoenixTransactionContext;
import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.TestUtil;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/33a6d9d8/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java
index 88fba47..baaac8d 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java
@@ -32,6 +32,8 @@ import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
+import java.util.Arrays;
+import java.util.Collection;
import java.util.Properties;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
@@ -44,6 +46,7 @@ import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.query.QueryServicesOptions;
+import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTableKey;
import org.apache.phoenix.transaction.PhoenixTransactionContext;
import org.apache.phoenix.transaction.TransactionFactory;
@@ -53,16 +56,33 @@ import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.StringUtil;
import org.apache.phoenix.util.TestUtil;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+@RunWith(Parameterized.class)
public class TransactionIT extends ParallelStatsDisabledIT {
+ private final String txProvider;
+ private final String tableDDLOptions;
+
+ public TransactionIT(String provider) {
+ txProvider = provider;
+ tableDDLOptions = PhoenixDatabaseMetaData.TRANSACTION_PROVIDER + "='" + provider + "'";
+ }
+ @Parameters(name="TransactionIT_provider={0}") // name is used by failsafe as file name in reports
+ public static Collection<String[]> data() {
+ return Arrays.asList(new String[][] {
+ {"TEPHRA"/*,"OMID"*/}});
+ }
+
@Test
public void testQueryWithSCN() throws Exception {
String tableName = generateUniqueName();
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
try (Connection conn = DriverManager.getConnection(getUrl(), props);) {
conn.createStatement().execute(
- "CREATE TABLE " + tableName + " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR) TRANSACTIONAL=true");
+ "CREATE TABLE " + tableName + " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR) TRANSACTIONAL=true," + tableDDLOptions);
}
props.put(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(EnvironmentEdgeManager.currentTimeMillis()));
try (Connection conn = DriverManager.getConnection(getUrl(), props);) {
@@ -86,7 +106,7 @@ public class TransactionIT extends ParallelStatsDisabledIT {
Statement stmt = conn.createStatement();
stmt.execute("CREATE TABLE " + tableName + "(k VARCHAR PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)");
stmt.execute("DROP TABLE " + tableName);
- stmt.execute("CREATE TABLE " + tableName + "(k VARCHAR PRIMARY KEY, v1 VARCHAR, v2 VARCHAR) TRANSACTIONAL=true");
+ stmt.execute("CREATE TABLE " + tableName + "(k VARCHAR PRIMARY KEY, v1 VARCHAR, v2 VARCHAR) TRANSACTIONAL=true," + tableDDLOptions);
stmt.execute("CREATE INDEX " + tableName + "_IDX ON " + tableName + " (v1) INCLUDE(v2)");
assertTrue(conn.unwrap(PhoenixConnection.class).getTable(new PTableKey(null, tableName)).isTransactional());
assertTrue(conn.unwrap(PhoenixConnection.class).getTable(new PTableKey(null, tableName + "_IDX")).isTransactional());
@@ -100,7 +120,7 @@ public class TransactionIT extends ParallelStatsDisabledIT {
conn.setAutoCommit(false);
Statement stmt = conn.createStatement();
try {
- stmt.execute("CREATE TABLE " + tableName + "(k VARCHAR, v VARCHAR, d DATE NOT NULL, CONSTRAINT PK PRIMARY KEY(k,d ROW_TIMESTAMP)) TRANSACTIONAL=true");
+ stmt.execute("CREATE TABLE " + tableName + "(k VARCHAR, v VARCHAR, d DATE NOT NULL, CONSTRAINT PK PRIMARY KEY(k,d ROW_TIMESTAMP)) TRANSACTIONAL=true," + tableDDLOptions);
fail();
}
catch(SQLException e) {
@@ -124,7 +144,7 @@ public class TransactionIT extends ParallelStatsDisabledIT {
String transactTableName = generateUniqueName();
Statement stmt = conn.createStatement();
stmt.execute("CREATE TABLE " + transactTableName + " (k VARCHAR PRIMARY KEY, v1 VARCHAR, v2 VARCHAR) " +
- "TRANSACTIONAL=true");
+ "TRANSACTIONAL=true," + tableDDLOptions);
conn.commit();
DatabaseMetaData dbmd = conn.getMetaData();
@@ -132,7 +152,11 @@ public class TransactionIT extends ParallelStatsDisabledIT {
assertTrue(rs.next());
assertEquals("Transactional table was not marked as transactional in JDBC API.",
"true", rs.getString(PhoenixDatabaseMetaData.TRANSACTIONAL));
- assertEquals(TransactionFactory.Provider.TEPHRA.name(), rs.getString(PhoenixDatabaseMetaData.TRANSACTION_PROVIDER));
+ assertEquals(txProvider, rs.getString(PhoenixDatabaseMetaData.TRANSACTION_PROVIDER));
+
+ // Ensure round-trip-ability of TRANSACTION_PROVIDER
+ PTable table = PhoenixRuntime.getTableNoCache(conn, transactTableName);
+ assertEquals(txProvider, table.getTransactionProvider().name());
String nonTransactTableName = generateUniqueName();
Statement stmt2 = conn.createStatement();
@@ -160,7 +184,7 @@ public class TransactionIT extends ParallelStatsDisabledIT {
// TODO: we should support having a transactional table defined for a connectionless connection
try (Connection conn = DriverManager.getConnection(getUrl())) {
String transactTableName = generateUniqueName();
- conn.createStatement().execute("CREATE TABLE " + transactTableName + " (k integer not null primary key, v bigint) TRANSACTIONAL=true");
+ conn.createStatement().execute("CREATE TABLE " + transactTableName + " (k integer not null primary key, v bigint) TRANSACTIONAL=true," + tableDDLOptions);
conn.createStatement().execute("UPSERT INTO " + transactTableName + " VALUES(0,0) ON DUPLICATE KEY UPDATE v = v + 1");
fail();
} catch (SQLException e) {
@@ -171,13 +195,28 @@ public class TransactionIT extends ParallelStatsDisabledIT {
@Test
public void testProperties() throws Exception {
String nonTxTableName = generateUniqueName();
+ String txTableName = generateUniqueName();
+ String idx1 = generateUniqueName();
+ String idx2 = generateUniqueName();
Connection conn = DriverManager.getConnection(getUrl());
conn.createStatement().execute("CREATE TABLE " + nonTxTableName + "1(k INTEGER PRIMARY KEY, a.v VARCHAR, b.v VARCHAR, c.v VARCHAR) TTL=1000");
- conn.createStatement().execute("CREATE INDEX idx1 ON " + nonTxTableName + "1(a.v, b.v) TTL=1000");
- conn.createStatement().execute("CREATE INDEX idx2 ON " + nonTxTableName + "1(c.v) INCLUDE (a.v, b.v) TTL=1000");
+ conn.createStatement().execute("CREATE INDEX " + idx1 + " ON " + nonTxTableName + "1(a.v, b.v) TTL=1000");
+ conn.createStatement().execute("CREATE INDEX " + idx2 + " ON " + nonTxTableName + "1(c.v) INCLUDE (a.v, b.v) TTL=1000");
- conn.createStatement().execute("ALTER TABLE " + nonTxTableName + "1 SET TRANSACTIONAL=true");
+ try {
+ conn.createStatement().execute("ALTER TABLE " + nonTxTableName + "1 SET TRANSACTIONAL=true," + tableDDLOptions);
+ if (TransactionFactory.Provider.OMID.name().equals(txProvider)) {
+ fail("Omid shouldn't allow converting a non transactional table to be transactional");
+ }
+ } catch (SQLException e) { // Should fail for Omid, but not Tephra
+ if (TransactionFactory.Provider.TEPHRA.name().equals(txProvider)) {
+ throw e;
+ }
+ assertEquals(SQLExceptionCode.CANNOT_ALTER_TABLE_FROM_NON_TXN_TO_TXNL.getErrorCode(), e.getErrorCode());
+ // FIXME: should verify Omid table properties too, but the checks below won't be valid for Omid
+ return;
+ }
TableDescriptor desc = conn.unwrap(PhoenixConnection.class).getQueryServices().getTableDescriptor(Bytes.toBytes(nonTxTableName + "1"));
for (ColumnFamilyDescriptor colDesc : desc.getColumnFamilies()) {
@@ -187,7 +226,7 @@ public class TransactionIT extends ParallelStatsDisabledIT {
assertEquals(1000, Integer.parseInt(Bytes.toString(propertyTTL)));
}
- desc = conn.unwrap(PhoenixConnection.class).getQueryServices().getTableDescriptor(Bytes.toBytes("IDX1"));
+ desc = conn.unwrap(PhoenixConnection.class).getQueryServices().getTableDescriptor(Bytes.toBytes(idx1));
for (ColumnFamilyDescriptor colDesc : desc.getColumnFamilies()) {
assertEquals(QueryServicesOptions.DEFAULT_MAX_VERSIONS_TRANSACTIONAL, colDesc.getMaxVersions());
assertEquals(1000, colDesc.getTimeToLive());
@@ -195,7 +234,7 @@ public class TransactionIT extends ParallelStatsDisabledIT {
assertEquals(1000, Integer.parseInt(Bytes.toString(propertyTTL)));
}
- desc = conn.unwrap(PhoenixConnection.class).getQueryServices().getTableDescriptor(Bytes.toBytes("IDX2"));
+ desc = conn.unwrap(PhoenixConnection.class).getQueryServices().getTableDescriptor(Bytes.toBytes(idx2));
for (ColumnFamilyDescriptor colDesc : desc.getColumnFamilies()) {
assertEquals(QueryServicesOptions.DEFAULT_MAX_VERSIONS_TRANSACTIONAL, colDesc.getMaxVersions());
assertEquals(1000, colDesc.getTimeToLive());
@@ -221,7 +260,7 @@ public class TransactionIT extends ParallelStatsDisabledIT {
}
conn.createStatement().execute("CREATE TABLE " + nonTxTableName + "3(k INTEGER PRIMARY KEY, a.v VARCHAR, b.v VARCHAR, c.v VARCHAR)");
- conn.createStatement().execute("ALTER TABLE " + nonTxTableName + "3 SET TRANSACTIONAL=true, b.VERSIONS=10, c.VERSIONS=20");
+ conn.createStatement().execute("ALTER TABLE " + nonTxTableName + "3 SET TRANSACTIONAL=true, b.VERSIONS=10, c.VERSIONS=20," + tableDDLOptions);
desc = conn.unwrap(PhoenixConnection.class).getQueryServices().getTableDescriptor(Bytes.toBytes( nonTxTableName + "3"));
assertEquals(QueryServicesOptions.DEFAULT_MAX_VERSIONS_TRANSACTIONAL, desc.getColumnFamily(Bytes.toBytes("A")).getMaxVersions());
assertEquals(10, desc.getColumnFamily(Bytes.toBytes("B")).getMaxVersions());
@@ -229,21 +268,21 @@ public class TransactionIT extends ParallelStatsDisabledIT {
conn.createStatement().execute("CREATE TABLE " + nonTxTableName + "4(k INTEGER PRIMARY KEY, a.v VARCHAR, b.v VARCHAR, c.v VARCHAR)");
try {
- conn.createStatement().execute("ALTER TABLE " + nonTxTableName + "4 SET TRANSACTIONAL=true, VERSIONS=1");
+ conn.createStatement().execute("ALTER TABLE " + nonTxTableName + "4 SET TRANSACTIONAL=true, VERSIONS=1," + tableDDLOptions);
fail();
} catch (SQLException e) {
assertEquals(SQLExceptionCode.TX_MAX_VERSIONS_MUST_BE_GREATER_THAN_ONE.getErrorCode(), e.getErrorCode());
}
try {
- conn.createStatement().execute("ALTER TABLE " + nonTxTableName + "4 SET TRANSACTIONAL=true, b.VERSIONS=1");
+ conn.createStatement().execute("ALTER TABLE " + nonTxTableName + "4 SET TRANSACTIONAL=true, b.VERSIONS=1," + tableDDLOptions);
fail();
} catch (SQLException e) {
assertEquals(SQLExceptionCode.TX_MAX_VERSIONS_MUST_BE_GREATER_THAN_ONE.getErrorCode(), e.getErrorCode());
}
- conn.createStatement().execute("CREATE TABLE TX_TABLE1(k INTEGER PRIMARY KEY, v VARCHAR) TTL=1000, TRANSACTIONAL=true");
- desc = conn.unwrap(PhoenixConnection.class).getQueryServices().getTableDescriptor(Bytes.toBytes("TX_TABLE1"));
+ conn.createStatement().execute("CREATE TABLE " + txTableName + "(k INTEGER PRIMARY KEY, v VARCHAR) TTL=1000, TRANSACTIONAL=true," + tableDDLOptions);
+ desc = conn.unwrap(PhoenixConnection.class).getQueryServices().getTableDescriptor(Bytes.toBytes(txTableName));
for (ColumnFamilyDescriptor colDesc : desc.getColumnFamilies()) {
assertEquals(QueryServicesOptions.DEFAULT_MAX_VERSIONS_TRANSACTIONAL, colDesc.getMaxVersions());
assertEquals(ColumnFamilyDescriptorBuilder.DEFAULT_TTL, colDesc.getTimeToLive());
@@ -258,7 +297,7 @@ public class TransactionIT extends ParallelStatsDisabledIT {
String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + transTableName;
try (Connection conn1 = DriverManager.getConnection(getUrl());
Connection conn2 = DriverManager.getConnection(getUrl())) {
- TestUtil.createTransactionalTable(conn1, fullTableName);
+ TestUtil.createTransactionalTable(conn1, fullTableName, tableDDLOptions);
conn1.setAutoCommit(false);
conn2.setAutoCommit(false);
String selectSql = "SELECT * FROM "+fullTableName;
@@ -297,7 +336,7 @@ public class TransactionIT extends ParallelStatsDisabledIT {
conn.setAutoCommit(false);
try {
Statement stmt = conn.createStatement();
- stmt.execute("CREATE TABLE " + fullTableName + "(k VARCHAR PRIMARY KEY, v1 VARCHAR, v2 VARCHAR) TRANSACTIONAL=true");
+ stmt.execute("CREATE TABLE " + fullTableName + "(k VARCHAR PRIMARY KEY, v1 VARCHAR, v2 VARCHAR) TRANSACTIONAL=true," + tableDDLOptions);
stmt.executeUpdate("upsert into " + fullTableName + " values('x', 'a', 'a')");
conn.commit();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/33a6d9d8/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
index 38643a4..cb082f7 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
@@ -832,7 +832,11 @@ public class TestUtil {
}
public static void createTransactionalTable(Connection conn, String tableName) throws SQLException {
- conn.createStatement().execute("create table " + tableName + TestUtil.TEST_TABLE_SCHEMA + "TRANSACTIONAL=true");
+ createTransactionalTable(conn, tableName, "");
+ }
+
+ public static void createTransactionalTable(Connection conn, String tableName, String extraProps) throws SQLException {
+ conn.createStatement().execute("create table " + tableName + TestUtil.TEST_TABLE_SCHEMA + "TRANSACTIONAL=true" + (extraProps.length() == 0 ? "" : ("," + extraProps)));
}
public static void dumpTable(Table table) throws IOException {
[7/7] phoenix git commit: PHOENIX-4646 The data exceeds the max
capacity for the data type error for valid scenarios (Sergey Soldatov)
Posted by ja...@apache.org.
PHOENIX-4646 The data exceeds the max capacity for the data type error for valid scenarios (Sergey Soldatov)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/ff0b8089
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/ff0b8089
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/ff0b8089
Branch: refs/heads/5.x-HBase-2.0
Commit: ff0b8089d8c15154cbe875a535c33a62d72fe26a
Parents: 9516b14
Author: James Taylor <jt...@salesforce.com>
Authored: Wed May 9 13:12:26 2018 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Wed May 9 13:49:36 2018 -0700
----------------------------------------------------------------------
.../apache/phoenix/end2end/UpsertSelectIT.java | 19 +++++++++++++++++++
.../apache/phoenix/schema/types/PVarchar.java | 10 ++++++----
2 files changed, 25 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/ff0b8089/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java
index 275d72d..e6feea6 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java
@@ -1486,6 +1486,25 @@ public class UpsertSelectIT extends ParallelStatsDisabledIT {
}
}
+ @Test // See https://issues.apache.org/jira/browse/PHOENIX-4646
+ public void testLengthLimitedVarchar() throws Exception {
+ String tableName1 = generateUniqueName();
+ String tableName2 = generateUniqueName();
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ conn.setAutoCommit(true);
+ conn.createStatement().execute("create table " + tableName1 + "(name varchar(160) primary key, id varchar(120), address varchar(160))");
+ conn.createStatement().execute("create table " + tableName2 + "(name varchar(160) primary key, id varchar(10), address varchar(10))");
+ conn.createStatement().execute("upsert into " + tableName1 + " values('test','test','test')");
+ conn.createStatement().execute("upsert into " + tableName2 + " select * from " + tableName1);
+ ResultSet rs = conn.createStatement().executeQuery("select * from " + tableName2);
+ assertTrue(rs.next());
+ assertEquals("test", rs.getString(1));
+ assertEquals("test", rs.getString(2));
+ assertEquals("test", rs.getString(2));
+ assertFalse(rs.next());
+ }
+ }
+
private static Connection getTenantConnection(String tenantId) throws Exception {
Properties props = PropertiesUtil.deepCopy(TestUtil.TEST_PROPERTIES);
props.setProperty(TENANT_ID_ATTRIB, tenantId);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/ff0b8089/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PVarchar.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PVarchar.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PVarchar.java
index fb5a045..a99925c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PVarchar.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PVarchar.java
@@ -102,15 +102,17 @@ public class PVarchar extends PDataType<String> {
SortOrder sortOrder, Integer maxLength, Integer scale,
Integer desiredMaxLength, Integer desiredScale) {
if (ptr.getLength() != 0 && desiredMaxLength != null) {
- if (maxLength == null) {
+ if (maxLength == null || maxLength > desiredMaxLength) {
if (value != null) { // Use value if provided
maxLength = value.toString().length();
} else {
- coerceBytes(ptr, value, srcType, maxLength, scale, sortOrder, desiredMaxLength, desiredScale, sortOrder, true);
- maxLength = StringUtil.calculateUTF8Length(ptr.get(), ptr.getOffset(), ptr.getLength(), sortOrder);
+ coerceBytes(ptr, value, srcType, maxLength, scale, sortOrder, desiredMaxLength,
+ desiredScale, sortOrder, true);
+ maxLength = StringUtil
+ .calculateUTF8Length(ptr.get(), ptr.getOffset(), ptr.getLength(), sortOrder);
}
+ return maxLength <= desiredMaxLength;
}
- return maxLength <= desiredMaxLength;
}
return true;
}