You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by sk...@apache.org on 2020/02/04 23:49:46 UTC
[phoenix] branch 4.x-HBase-1.5 updated: PHOENIX-5713: Incorrectly handled view and view indexes with/without … (#704)
This is an automated email from the ASF dual-hosted git repository.
skadam pushed a commit to branch 4.x-HBase-1.5
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/4.x-HBase-1.5 by this push:
new 36c642b PHOENIX-5713: Incorrectly handled view and view indexes with/without … (#704)
36c642b is described below
commit 36c642b1f46088f8031a4f7ba5dee9e4ee3427e5
Author: Swaroopa Kadam <sw...@gmail.com>
AuthorDate: Tue Feb 4 15:42:36 2020 -0800
PHOENIX-5713: Incorrectly handled view and view indexes with/without … (#704)
* PHOENIX-5713: Incorrectly handled view and view indexes with/without namespace in IndexScrutinyMapper#getTtl()
---
.../NonParameterizedIndexScrutinyToolIT.java | 56 ++++++++-
.../mapreduce/index/IndexScrutinyMapper.java | 46 ++++---
.../phoenix/index/IndexScrutinyMapperTest.java | 132 +++++++++++++++++++++
3 files changed, 209 insertions(+), 25 deletions(-)
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/NonParameterizedIndexScrutinyToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/NonParameterizedIndexScrutinyToolIT.java
index cd73af5..93a723d 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/NonParameterizedIndexScrutinyToolIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/NonParameterizedIndexScrutinyToolIT.java
@@ -23,6 +23,7 @@ import org.apache.phoenix.mapreduce.index.IndexScrutinyMapperForTest;
import org.apache.phoenix.mapreduce.index.IndexScrutinyTool;
import org.apache.phoenix.util.EnvironmentEdgeManager;
import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.SchemaUtil;
import org.junit.Test;
import java.sql.Connection;
@@ -144,32 +145,75 @@ public class NonParameterizedIndexScrutinyToolIT extends IndexScrutinyToolBaseIT
@Test
public void testScrutinyOnRowsNearExpiry() throws Exception {
+ String schema = generateUniqueName();
String dataTableName = generateUniqueName();
String indexTableName = generateUniqueName();
+ String dataTableFullName = SchemaUtil.getTableName(schema, dataTableName);
String dataTableDDL = "CREATE TABLE %s (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR, "
+ "ZIP INTEGER) TTL="+TEST_TABLE_TTL;
String indexTableDDL = "CREATE INDEX %s ON %s (NAME) INCLUDE (ZIP)";
String upsertData = "UPSERT INTO %s VALUES (?, ?, ?)";
- int initialDelta = -5000; //insert row with 5 secs before current timestamp
IndexScrutinyMapperForTest.ScrutinyTestClock
- testClock = new IndexScrutinyMapperForTest.ScrutinyTestClock(initialDelta);
+ testClock = new IndexScrutinyMapperForTest.ScrutinyTestClock(0);
try (Connection conn =
DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES))) {
- conn.createStatement().execute(String.format(dataTableDDL, dataTableName));
+ conn.createStatement().execute(String.format(dataTableDDL, dataTableFullName));
conn.createStatement().execute(String.format(indexTableDDL, indexTableName,
- dataTableName));
+ dataTableFullName));
// insert two rows
PreparedStatement
upsertDataStmt = conn.prepareStatement(String.format(upsertData,
- dataTableName));
+ dataTableFullName));
EnvironmentEdgeManager.injectEdge(testClock);
upsertRow(upsertDataStmt, 1, "name-1", 98051);
upsertRow(upsertDataStmt, 2, "name-2", 98052);
conn.commit();
- List<Job> completedJobs = runScrutiny(null, dataTableName, indexTableName);
+ List<Job> completedJobs = runScrutiny(schema, dataTableName, indexTableName);
+ Job job = completedJobs.get(0);
+ assertTrue(job.isSuccessful());
+ Counters counters = job.getCounters();
+ assertEquals(2, getCounterValue(counters, EXPIRED_ROW_COUNT));
+ assertEquals(0, getCounterValue(counters, VALID_ROW_COUNT));
+ assertEquals(0, getCounterValue(counters, INVALID_ROW_COUNT));
+ }
+ }
+
+ @Test
+ public void testScrutinyOnRowsNearExpiry_viewIndex() throws Exception {
+ String schemaName = "S"+generateUniqueName();
+ String dataTableName = "T"+generateUniqueName();
+ String dataTableFullName = SchemaUtil.getTableName(schemaName,dataTableName);
+ String viewIndexName = "VI"+generateUniqueName();
+ String viewName = "V"+generateUniqueName();
+ String viewFullName = SchemaUtil.getTableName(schemaName,viewName);
+ String dataTableDDL = "CREATE TABLE %s (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR, "
+ + "ZIP INTEGER) TTL="+TEST_TABLE_TTL;
+ String viewDDL = "CREATE VIEW %s AS SELECT * FROM %s";
+ String indexTableDDL = "CREATE INDEX %s ON %s (NAME) INCLUDE (ZIP)";
+ String upsertData = "UPSERT INTO %s VALUES (?, ?, ?)";
+ IndexScrutinyMapperForTest.ScrutinyTestClock
+ testClock = new IndexScrutinyMapperForTest.ScrutinyTestClock(0);
+
+ try (Connection conn =
+ DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES))) {
+ conn.createStatement().execute(String.format(dataTableDDL, dataTableFullName));
+ conn.createStatement().execute(String.format(viewDDL, viewFullName, dataTableFullName));
+ conn.createStatement().execute(String.format(indexTableDDL, viewIndexName,
+ viewFullName));
+ // insert two rows
+ PreparedStatement
+ upsertDataStmt = conn.prepareStatement(String.format(upsertData,
+ viewFullName));
+
+ EnvironmentEdgeManager.injectEdge(testClock);
+ upsertRow(upsertDataStmt, 1, "name-1", 98051);
+ upsertRow(upsertDataStmt, 2, "name-2", 98052);
+ conn.commit();
+
+ List<Job> completedJobs = runScrutiny(schemaName, viewName, viewIndexName);
Job job = completedJobs.get(0);
assertTrue(job.isSuccessful());
Counters counters = job.getCounters();
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexScrutinyMapper.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexScrutinyMapper.java
index 9099f05..2359b4a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexScrutinyMapper.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexScrutinyMapper.java
@@ -33,6 +33,7 @@ import java.util.Map;
import java.util.Properties;
import java.util.Set;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.codec.binary.Hex;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
@@ -50,6 +51,8 @@ import org.apache.phoenix.mapreduce.index.IndexScrutinyTool.SourceTable;
import org.apache.phoenix.mapreduce.util.ConnectionUtil;
import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil;
import org.apache.phoenix.parse.HintNode.Hint;
+import org.apache.phoenix.query.ConnectionQueryServices;
+import org.apache.phoenix.query.ConnectionQueryServicesImpl;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.util.ColumnInfo;
@@ -297,31 +300,36 @@ public class IndexScrutinyMapper extends Mapper<NullWritable, PhoenixIndexDBWrit
}
private long getTableTtl() throws SQLException, IOException {
- PTable psourceTable = PhoenixRuntime.getTable(connection, qSourceTable);
- if (psourceTable.getType() == PTableType.INDEX
- && psourceTable.getIndexType() == PTable.IndexType.LOCAL) {
+ PTable pSourceTable = PhoenixRuntime.getTable(connection, qSourceTable);
+ if (pSourceTable.getType() == PTableType.INDEX
+ && pSourceTable.getIndexType() == PTable.IndexType.LOCAL) {
return Integer.MAX_VALUE;
}
- String schema = psourceTable.getSchemaName().toString();
- String table = getSourceTableName(psourceTable);
- Admin admin = connection.unwrap(PhoenixConnection.class).getQueryServices().getAdmin();
- String fullTableName = SchemaUtil.getQualifiedTableName(schema, table);
- HTableDescriptor tableDesc = admin.getTableDescriptor(TableName.valueOf(fullTableName));
- return tableDesc.getFamily(SchemaUtil.getEmptyColumnFamily(psourceTable)).getTimeToLive();
+ ConnectionQueryServices
+ cqsi = connection.unwrap(PhoenixConnection.class).getQueryServices();
+ Admin admin = cqsi.getAdmin();
+ String physicalTable = getSourceTableName(pSourceTable,
+ SchemaUtil.isNamespaceMappingEnabled(null, cqsi.getProps()));
+ HTableDescriptor tableDesc = admin.getTableDescriptor(TableName.valueOf(physicalTable));
+ return tableDesc.getFamily(SchemaUtil.getEmptyColumnFamily(pSourceTable)).getTimeToLive();
}
- private String getSourceTableName(PTable psourceTable) {
- String sourcePhysicalName = psourceTable.getPhysicalName().getString();
- String table;
- if (psourceTable.getType() == PTableType.VIEW) {
- table = sourcePhysicalName;
- } else if (MetaDataUtil.isViewIndex(sourcePhysicalName)) {
- table = SchemaUtil.getParentTableNameFromIndexTable(sourcePhysicalName,
- MetaDataUtil.VIEW_INDEX_TABLE_PREFIX);
+ @VisibleForTesting
+ public static String getSourceTableName(PTable pSourceTable, boolean isNamespaceEnabled) {
+ String sourcePhysicalName = pSourceTable.getPhysicalName().getString();
+ String physicalTable, table, schema;
+ if (pSourceTable.getType() == PTableType.VIEW
+ || MetaDataUtil.isViewIndex(sourcePhysicalName)) {
+ // in case of view and view index ptable, getPhysicalName() returns hbase tables
+ // i.e. without _IDX_ and with _IDX_ respectively
+ physicalTable = sourcePhysicalName;
} else {
- table = psourceTable.getTableName().toString();
+ table = pSourceTable.getTableName().toString();
+ schema = pSourceTable.getSchemaName().toString();
+ physicalTable = SchemaUtil
+ .getPhysicalHBaseTableName(schema, table, isNamespaceEnabled).toString();
}
- return table;
+ return physicalTable;
}
protected Map<String, Pair<Long, List<Object>>> buildTargetStatement(PreparedStatement targetStatement)
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/index/IndexScrutinyMapperTest.java b/phoenix-core/src/test/java/org/apache/phoenix/index/IndexScrutinyMapperTest.java
new file mode 100644
index 0000000..863593b
--- /dev/null
+++ b/phoenix-core/src/test/java/org/apache/phoenix/index/IndexScrutinyMapperTest.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.index;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.mapreduce.index.IndexScrutinyMapper;
+import org.apache.phoenix.query.BaseConnectionlessQueryTest;
+import org.apache.phoenix.schema.PName;
+import org.apache.phoenix.schema.PNameFactory;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTableType;
+import org.apache.phoenix.util.MetaDataUtil;
+import org.apache.phoenix.util.SchemaUtil;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.mockito.Mockito;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+@RunWith(Parameterized.class)
+public class IndexScrutinyMapperTest extends BaseConnectionlessQueryTest {
+ String schema, tableName, indexName;
+ boolean isNamespaceEnabled;
+ PTable inputTable;
+
+ @Before
+ public void setup() {
+ schema = "S_" + generateUniqueName();
+ tableName = "T_" + generateUniqueName();
+ indexName = "I_" + generateUniqueName();
+ inputTable = Mockito.mock(PTable.class);
+
+ }
+
+ @Parameterized.Parameters(name ="IndexUpgradeToolTest_isNamespaceEnabled={0}")
+ public static synchronized Collection<Boolean> data() {
+ return Arrays.asList( false, true);
+ }
+
+ public IndexScrutinyMapperTest(boolean isNamespaceEnabled) {
+ this.isNamespaceEnabled = isNamespaceEnabled;
+ }
+ @Test
+ public void testGetSourceTableName_table() {
+ String fullTableName = SchemaUtil.getQualifiedTableName(schema, tableName);
+ PName sourcePhysicalName = SchemaUtil.getPhysicalHBaseTableName(schema, tableName,
+ isNamespaceEnabled);
+ String expectedName = SchemaUtil.getPhysicalTableName(Bytes.toBytes(fullTableName),
+ isNamespaceEnabled).toString();
+ //setup
+ Mockito.when(inputTable.getType()).thenReturn(PTableType.TABLE);
+ Mockito.when(inputTable.getPhysicalName()).thenReturn(sourcePhysicalName);
+ Mockito.when(inputTable.getTableName()).thenReturn(PNameFactory.newName(tableName));
+ Mockito.when(inputTable.getSchemaName()).thenReturn(PNameFactory.newName(schema));
+ //test
+ String output = IndexScrutinyMapper.getSourceTableName(inputTable, isNamespaceEnabled);
+ //assert
+ Assert.assertEquals(expectedName, output);
+ }
+
+ @Test
+ public void testGetSourceTableName_view() {
+ String fullTableName = SchemaUtil.getQualifiedTableName(schema, tableName);
+ PName sourcePhysicalName = SchemaUtil.getPhysicalHBaseTableName(schema, tableName,
+ isNamespaceEnabled);
+ String expectedName = SchemaUtil.getPhysicalTableName(Bytes.toBytes(fullTableName),
+ isNamespaceEnabled).toString();
+ //setup
+ Mockito.when(inputTable.getType()).thenReturn(PTableType.VIEW);
+ Mockito.when(inputTable.getPhysicalName()).thenReturn(sourcePhysicalName);
+ //test
+ String output = IndexScrutinyMapper.getSourceTableName(inputTable, isNamespaceEnabled);
+ //assert
+ Assert.assertEquals(expectedName, output);
+ }
+
+ @Test
+ public void testGetSourceTableName_index() {
+ String fullTableName = SchemaUtil.getQualifiedTableName(schema, indexName);
+ PName sourcePhysicalName = SchemaUtil.getPhysicalHBaseTableName(schema, indexName,
+ isNamespaceEnabled);
+ String expectedName = SchemaUtil.getPhysicalTableName(Bytes.toBytes(fullTableName),
+ isNamespaceEnabled).toString();
+
+ //setup
+ Mockito.when(inputTable.getType()).thenReturn(PTableType.INDEX);
+ Mockito.when(inputTable.getPhysicalName()).thenReturn(sourcePhysicalName);
+ Mockito.when(inputTable.getTableName()).thenReturn(PNameFactory.newName(indexName));
+ Mockito.when(inputTable.getSchemaName()).thenReturn(PNameFactory.newName(schema));
+
+ //test
+ String output = IndexScrutinyMapper.getSourceTableName(inputTable, isNamespaceEnabled);
+ //assert
+ Assert.assertEquals(expectedName, output);
+ }
+
+ @Test
+ public void testGetSourceTableName_viewIndex() {
+ PName physicalTableName = SchemaUtil.getPhysicalHBaseTableName(schema, tableName,
+ isNamespaceEnabled);
+ String expectedName = MetaDataUtil.getViewIndexPhysicalName(physicalTableName.getString());
+ PName physicalIndexTableName = PNameFactory
+ .newName(MetaDataUtil.getViewIndexPhysicalName(physicalTableName.getString()));
+
+ PTable pSourceTable = Mockito.mock(PTable.class);
+ //setup
+ Mockito.when(pSourceTable.getPhysicalName()).thenReturn(physicalIndexTableName);
+ //test
+ String output = IndexScrutinyMapper.getSourceTableName(pSourceTable, isNamespaceEnabled);
+ //assert
+ Assert.assertEquals(expectedName, output);
+ }
+}