You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by pb...@apache.org on 2019/05/28 22:01:47 UTC
[phoenix] branch 4.x-HBase-1.2 updated (34ffbb9 -> 68d1a04)
This is an automated email from the ASF dual-hosted git repository.
pboado pushed a change to branch 4.x-HBase-1.2
in repository https://gitbox.apache.org/repos/asf/phoenix.git.
from 34ffbb9 PHOENIX-5231 Configurable Stats Cache
new c98c89f PHOENIX-4296: reverse scan in ChunkedResultIterator
new a5f1b5c PHOENIX-5291 Ensure that Phoenix coprocessor close all scanners.
new 42511fb PHOENIX-5297 POM cleanup and de-duplication
new 68d1a04 PHOENIX-5112 Simplify QueryPlan selection in Phoenix.
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
phoenix-core/pom.xml | 14 -----
.../UngroupedAggregateRegionObserver.java | 17 ++++-
.../phoenix/iterate/ChunkedResultIterator.java | 13 +++-
.../apache/phoenix/optimize/QueryOptimizer.java | 18 ------
.../phoenix/iterate/ChunkedResultIteratorTest.java | 73 ++++++++++++++++++++++
phoenix-pherf/pom.xml | 7 ---
pom.xml | 8 +--
7 files changed, 101 insertions(+), 49 deletions(-)
create mode 100644 phoenix-core/src/test/java/org/apache/phoenix/iterate/ChunkedResultIteratorTest.java
[phoenix] 02/04: PHOENIX-5291 Ensure that Phoenix coprocessor close
all scanners.
Posted by pb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
pboado pushed a commit to branch 4.x-HBase-1.2
in repository https://gitbox.apache.org/repos/asf/phoenix.git
commit a5f1b5c26b9d15c68244bc62faa57b70361d209b
Author: Lars Hofhansl <la...@apache.org>
AuthorDate: Thu May 23 06:40:34 2019 +0100
PHOENIX-5291 Ensure that Phoenix coprocessor close all scanners.
---
.../coprocessor/UngroupedAggregateRegionObserver.java | 17 ++++++++++++++---
1 file changed, 14 insertions(+), 3 deletions(-)
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
index dc7567b..dc61a98 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
@@ -1175,7 +1175,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
long rowCount = 0; // in case of async, we report 0 as number of rows updated
StatisticsCollectionRunTracker statsRunTracker =
StatisticsCollectionRunTracker.getInstance(config);
- boolean runUpdateStats = statsRunTracker.addUpdateStatsCommandRegion(region.getRegionInfo(),scan.getFamilyMap().keySet());
+ final boolean runUpdateStats = statsRunTracker.addUpdateStatsCommandRegion(region.getRegionInfo(),scan.getFamilyMap().keySet());
if (runUpdateStats) {
if (!async) {
rowCount = callable.call();
@@ -1204,8 +1204,11 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
@Override
public void close() throws IOException {
- // No-op because we want to manage closing of the inner scanner ourselves.
- // This happens inside StatsCollectionCallable.
+ // If we ran/scheduled StatsCollectionCallable the delegate
+ // scanner is closed there. Otherwise close it here.
+ if (!runUpdateStats) {
+ super.close();
+ }
}
@Override
@@ -1442,6 +1445,14 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
+ fullTableName);
Scan scan = new Scan();
scan.setMaxVersions();
+
+ // close the passed scanner since we are returning a brand-new one
+ try {
+ if (s != null) {
+ s.close();
+ }
+ } catch (IOException ignore) {}
+
return new StoreScanner(store, store.getScanInfo(), scan, scanners,
ScanType.COMPACT_RETAIN_DELETES, store.getSmallestReadPoint(),
HConstants.OLDEST_TIMESTAMP);
[phoenix] 01/04: PHOENIX-4296: reverse scan in ChunkedResultIterator
Posted by pb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
pboado pushed a commit to branch 4.x-HBase-1.2
in repository https://gitbox.apache.org/repos/asf/phoenix.git
commit c98c89f2fbbc34817cef1e07d030c65e84cc1d66
Author: chfeng <ch...@gmail.com>
AuthorDate: Thu May 16 11:41:41 2019 +0100
PHOENIX-4296: reverse scan in ChunkedResultIterator
---
.../phoenix/iterate/ChunkedResultIterator.java | 13 +++-
.../phoenix/iterate/ChunkedResultIteratorTest.java | 73 ++++++++++++++++++++++
2 files changed, 83 insertions(+), 3 deletions(-)
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ChunkedResultIterator.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ChunkedResultIterator.java
index acb6c04..1aab2d5 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ChunkedResultIterator.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ChunkedResultIterator.java
@@ -58,6 +58,7 @@ public class ChunkedResultIterator implements PeekingResultIterator {
private final ParallelIteratorFactory delegateIteratorFactory;
private ImmutableBytesWritable lastKey = new ImmutableBytesWritable();
+ private ImmutableBytesWritable prevLastKey = new ImmutableBytesWritable();
private final StatementContext context;
private final TableRef tableRef;
private final long chunkSize;
@@ -96,8 +97,9 @@ public class ChunkedResultIterator implements PeekingResultIterator {
}
}
- private ChunkedResultIterator(ParallelIteratorFactory delegateIteratorFactory, MutationState mutationState,
- StatementContext context, TableRef tableRef, Scan scan, long chunkSize, ResultIterator scanner, QueryPlan plan) throws SQLException {
+ private ChunkedResultIterator(ParallelIteratorFactory delegateIteratorFactory,
+ MutationState mutationState, StatementContext context, TableRef tableRef, Scan scan,
+ long chunkSize, ResultIterator scanner, QueryPlan plan) throws SQLException {
this.delegateIteratorFactory = delegateIteratorFactory;
this.context = context;
this.tableRef = tableRef;
@@ -138,8 +140,12 @@ public class ChunkedResultIterator implements PeekingResultIterator {
if (resultIterator.peek() == null && lastKey != null) {
resultIterator.close();
scan = ScanUtil.newScan(scan);
- if(ScanUtil.isLocalIndex(scan)) {
+ if (ScanUtil.isLocalIndex(scan)) {
scan.setAttribute(SCAN_START_ROW_SUFFIX, ByteUtil.copyKeyBytesIfNecessary(lastKey));
+ } else if (ScanUtil.isReversed(scan)) {
+ // lastKey is the last row the previous iterator meet but not returned.
+ // for reverse scan, use prevLastKey as the new stopRow.
+ scan.setStopRow(ByteUtil.copyKeyBytesIfNecessary(prevLastKey));
} else {
scan.setStartRow(ByteUtil.copyKeyBytesIfNecessary(lastKey));
}
@@ -212,6 +218,7 @@ public class ChunkedResultIterator implements PeekingResultIterator {
byte[] currentKey = lastKey.get();
int offset = lastKey.getOffset();
int length = lastKey.getLength();
+ prevLastKey.set(lastKey.copyBytes());
newTuple.getKey(lastKey);
return Bytes.compareTo(currentKey, offset, length, lastKey.get(), lastKey.getOffset(), lastKey.getLength()) != 0;
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/iterate/ChunkedResultIteratorTest.java b/phoenix-core/src/test/java/org/apache/phoenix/iterate/ChunkedResultIteratorTest.java
new file mode 100644
index 0000000..18402f0
--- /dev/null
+++ b/phoenix-core/src/test/java/org/apache/phoenix/iterate/ChunkedResultIteratorTest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.iterate;
+
+import static org.apache.phoenix.util.TestUtil.PHOENIX_JDBC_URL;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.util.List;
+import java.util.Properties;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.phoenix.compile.QueryPlan;
+import org.apache.phoenix.end2end.ParallelStatsDisabledIT;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixStatement;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTableKey;
+import org.apache.phoenix.schema.TableRef;
+import org.junit.Assert;
+import org.junit.Test;
+
+@SuppressWarnings("deprecated") public class ChunkedResultIteratorTest
+ extends ParallelStatsDisabledIT {
+
+ @Test
+ public void testChunked() throws Exception {
+ Properties props = new Properties();
+ props.setProperty(QueryServices.RENEW_LEASE_ENABLED, "false");
+ props.setProperty(QueryServices.SCAN_RESULT_CHUNK_SIZE, "2");
+ Connection conn = DriverManager.getConnection(PHOENIX_JDBC_URL, props);
+ String tableName = generateUniqueName();
+
+ conn.createStatement().execute("CREATE TABLE " + tableName
+ + " (A UNSIGNED_LONG NOT NULL PRIMARY KEY, B VARCHAR(10))");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES (1, 'A')");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES (2, 'B')");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES (3, 'C')");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES (4, 'D')");
+ conn.commit();
+
+
+ String sql = "SELECT A, B FROM " + tableName + " ORDER BY A DESC";
+ PhoenixStatement stmt = conn.createStatement().unwrap(PhoenixStatement.class);
+ ResultSet rs = stmt.executeQuery(sql);
+
+ int cnt = 0;
+ while ((rs.next())) {
+ cnt++;
+ assertTrue("too many results returned", cnt <= 4);
+ }
+ assertEquals(4, cnt);
+ }
+}
[phoenix] 03/04: PHOENIX-5297 POM cleanup and de-duplication
Posted by pb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
pboado pushed a commit to branch 4.x-HBase-1.2
in repository https://gitbox.apache.org/repos/asf/phoenix.git
commit 42511fb880151962beb0005d2f514ac5c48acf33
Author: Josh Elser <el...@apache.org>
AuthorDate: Fri May 24 17:02:11 2019 +0100
PHOENIX-5297 POM cleanup and de-duplication
Signed-off-by: Geoffrey Jacoby <gj...@apache.org>
---
phoenix-core/pom.xml | 14 --------------
phoenix-pherf/pom.xml | 7 -------
pom.xml | 8 ++++----
3 files changed, 4 insertions(+), 25 deletions(-)
diff --git a/phoenix-core/pom.xml b/phoenix-core/pom.xml
index 3aab0ed..99cab92 100644
--- a/phoenix-core/pom.xml
+++ b/phoenix-core/pom.xml
@@ -10,20 +10,6 @@
<name>Phoenix Core</name>
<description>Core Phoenix codebase</description>
- <licenses>
- <license>
- <name>The Apache Software License, Version 2.0</name>
- <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
- <distribution>repo</distribution>
- <comments />
- </license>
- </licenses>
-
- <organization>
- <name>Apache Software Foundation</name>
- <url>http://www.apache.org</url>
- </organization>
-
<properties>
<top.dir>${project.basedir}/..</top.dir>
<argparse4j.version>0.8.1</argparse4j.version>
diff --git a/phoenix-pherf/pom.xml b/phoenix-pherf/pom.xml
index 8640b3a..6463c8f 100644
--- a/phoenix-pherf/pom.xml
+++ b/phoenix-pherf/pom.xml
@@ -27,13 +27,6 @@
<shaded.package>org.apache.phoenix.shaded</shaded.package>
</properties>
- <repositories>
- <repository>
- <id>apache release</id>
- <url>https://repository.apache.org/content/repositories/releases/</url>
- </repository>
- </repositories>
-
<dependencies>
<dependency>
<groupId>org.apache.phoenix</groupId>
diff --git a/pom.xml b/pom.xml
index 83119ce..4ed9b89 100644
--- a/pom.xml
+++ b/pom.xml
@@ -11,7 +11,7 @@
<licenses>
<license>
<name>The Apache Software License, Version 2.0</name>
- <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+ <url>https://www.apache.org/licenses/LICENSE-2.0.txt</url>
<distribution>repo</distribution>
<comments/>
</license>
@@ -19,7 +19,7 @@
<organization>
<name>Apache Software Foundation</name>
- <url>http://www.apache.org</url>
+ <url>https://www.apache.org</url>
</organization>
<modules>
@@ -45,7 +45,7 @@
</parent>
<scm>
- <connection>scm:git:http://git-wip-us.apache.org/repos/asf/phoenix.git</connection>
+ <connection>scm:git:https://git-wip-us.apache.org/repos/asf/phoenix.git</connection>
<url>https://git-wip-us.apache.org/repos/asf/phoenix.git</url>
<developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/phoenix.git</developerConnection>
</scm>
@@ -409,7 +409,7 @@
<configuration>
<quiet>true</quiet>
<links>
- <link>http://hbase.apache.org/apidocs/</link>
+ <link>https://hbase.apache.org/apidocs/</link>
</links>
</configuration>
<executions>
[phoenix] 04/04: PHOENIX-5112 Simplify QueryPlan selection in
Phoenix.
Posted by pb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
pboado pushed a commit to branch 4.x-HBase-1.2
in repository https://gitbox.apache.org/repos/asf/phoenix.git
commit 68d1a04f1c80212a6fe6dd9574de9c34ad39b779
Author: Lars Hofhansl <la...@apache.org>
AuthorDate: Sat May 25 02:55:09 2019 +0100
PHOENIX-5112 Simplify QueryPlan selection in Phoenix.
---
.../org/apache/phoenix/optimize/QueryOptimizer.java | 18 ------------------
1 file changed, 18 deletions(-)
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java b/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java
index 43a5950..4f0dfeb 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java
@@ -324,24 +324,6 @@ public class QueryOptimizer {
QueryPlan plan = compiler.compile();
- boolean optimizedSort =
- plan.getOrderBy().getOrderByExpressions().isEmpty()
- && !dataPlan.getOrderBy().getOrderByExpressions().isEmpty()
- || plan.getGroupBy().isOrderPreserving()
- && !dataPlan.getGroupBy().isOrderPreserving();
-
- // If query doesn't have where clause, or the planner didn't add any (bound) scan ranges, and some of
- // columns to project/filter are missing in the index then we need to get missing columns from main table
- // for each row in local index. It's like full scan of both local index and data table which is inefficient.
- // Then we don't use the index. If all the columns to project are present in the index
- // then we can use the index even the query doesn't have where clause.
- // We'll use the index anyway if it allowed us to avoid a sort operation.
- if (index.getIndexType() == IndexType.LOCAL
- && (indexSelect.getWhere() == null
- || plan.getContext().getScanRanges().getBoundRanges().size() == 1)
- && !plan.getContext().getDataColumns().isEmpty() && !optimizedSort) {
- return null;
- }
indexTableRef = plan.getTableRef();
indexTable = indexTableRef.getTable();
indexState = indexTable.getIndexState();