You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by la...@apache.org on 2016/06/02 20:22:20 UTC
[01/16] phoenix git commit: Changed version to
4.4.0-HBase-1.x-SNAPSHOT
Repository: phoenix
Updated Branches:
refs/heads/4.0 [created] 46e99bb1b
refs/heads/4.x-HBase-0.98 0032a3d8e -> da39bd4e6
refs/heads/4.x-HBase-1.0 dfa3eec7a -> 65ee886fb
refs/heads/4.x-HBase-1.1 7b252763f -> cf551d536
refs/heads/4.x-HBase-1.x [created] 6cb6a3766
refs/heads/master eb275a5c6 -> 7dcf95a40
Changed version to 4.4.0-HBase-1.x-SNAPSHOT
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/03fce013
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/03fce013
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/03fce013
Branch: refs/heads/4.x-HBase-1.x
Commit: 03fce013c3a0c4883d3d1e9ad037d81c471ef74f
Parents: 174d0e6
Author: Enis Soztutar <en...@apache.org>
Authored: Thu Mar 19 13:34:46 2015 -0700
Committer: Enis Soztutar <en...@apache.org>
Committed: Thu Mar 19 13:34:46 2015 -0700
----------------------------------------------------------------------
phoenix-assembly/pom.xml | 2 +-
phoenix-core/pom.xml | 2 +-
phoenix-flume/pom.xml | 2 +-
phoenix-pherf/pom.xml | 2 +-
phoenix-pig/pom.xml | 2 +-
pom.xml | 2 +-
6 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/03fce013/phoenix-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-assembly/pom.xml b/phoenix-assembly/pom.xml
index a887dbf..96bb16f 100644
--- a/phoenix-assembly/pom.xml
+++ b/phoenix-assembly/pom.xml
@@ -26,7 +26,7 @@
<parent>
<groupId>org.apache.phoenix</groupId>
<artifactId>phoenix</artifactId>
- <version>4.4.0-HBase-0.98-SNAPSHOT</version>
+ <version>4.4.0-HBase-1.x-SNAPSHOT</version>
</parent>
<artifactId>phoenix-assembly</artifactId>
<name>Phoenix Assembly</name>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/03fce013/phoenix-core/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-core/pom.xml b/phoenix-core/pom.xml
index 31b7afd..a325b27 100644
--- a/phoenix-core/pom.xml
+++ b/phoenix-core/pom.xml
@@ -4,7 +4,7 @@
<parent>
<groupId>org.apache.phoenix</groupId>
<artifactId>phoenix</artifactId>
- <version>4.4.0-HBase-0.98-SNAPSHOT</version>
+ <version>4.4.0-HBase-1.x-SNAPSHOT</version>
</parent>
<artifactId>phoenix-core</artifactId>
<name>Phoenix Core</name>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/03fce013/phoenix-flume/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-flume/pom.xml b/phoenix-flume/pom.xml
index 6e29227..af01f6b 100644
--- a/phoenix-flume/pom.xml
+++ b/phoenix-flume/pom.xml
@@ -26,7 +26,7 @@
<parent>
<groupId>org.apache.phoenix</groupId>
<artifactId>phoenix</artifactId>
- <version>4.4.0-HBase-0.98-SNAPSHOT</version>
+ <version>4.4.0-HBase-1.x-SNAPSHOT</version>
</parent>
<artifactId>phoenix-flume</artifactId>
<name>Phoenix - Flume</name>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/03fce013/phoenix-pherf/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-pherf/pom.xml b/phoenix-pherf/pom.xml
index 26698b2..a0d521c 100644
--- a/phoenix-pherf/pom.xml
+++ b/phoenix-pherf/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.phoenix</groupId>
<artifactId>phoenix</artifactId>
- <version>4.4.0-HBase-0.98-SNAPSHOT</version>
+ <version>4.4.0-HBase-1.x-SNAPSHOT</version>
</parent>
<artifactId>pherf</artifactId>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/03fce013/phoenix-pig/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-pig/pom.xml b/phoenix-pig/pom.xml
index 7a2072c..0709657 100644
--- a/phoenix-pig/pom.xml
+++ b/phoenix-pig/pom.xml
@@ -26,7 +26,7 @@
<parent>
<groupId>org.apache.phoenix</groupId>
<artifactId>phoenix</artifactId>
- <version>4.4.0-HBase-0.98-SNAPSHOT</version>
+ <version>4.4.0-HBase-1.x-SNAPSHOT</version>
</parent>
<artifactId>phoenix-pig</artifactId>
<name>Phoenix - Pig</name>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/03fce013/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index f8f268d..92b0ed6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.phoenix</groupId>
<artifactId>phoenix</artifactId>
- <version>4.4.0-HBase-0.98-SNAPSHOT</version>
+ <version>4.4.0-HBase-1.x-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Apache Phoenix</name>
<description>A SQL layer over HBase</description>
[03/16] phoenix git commit: PHOENIX-1642 Make Phoenix Master Branch
pointing to HBase1.0.0
Posted by la...@apache.org.
PHOENIX-1642 Make Phoenix Master Branch pointing to HBase1.0.0
Conflicts:
phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/update/ColumnReference.java
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/a29e163f
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/a29e163f
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/a29e163f
Branch: refs/heads/4.x-HBase-1.x
Commit: a29e163fcdf3ec06c98de423bfd34861af227307
Parents: 03fce01
Author: Enis Soztutar <en...@apache.org>
Authored: Thu Mar 19 12:07:16 2015 -0700
Committer: Enis Soztutar <en...@apache.org>
Committed: Thu Mar 19 13:37:21 2015 -0700
----------------------------------------------------------------------
phoenix-assembly/src/build/client.xml | 4 +-
phoenix-core/pom.xml | 4 +-
...ReplayWithIndexWritesAndCompressedWALIT.java | 34 ++--
.../phoenix/end2end/index/LocalIndexIT.java | 32 ++--
.../end2end/index/MutableIndexFailureIT.java | 6 +-
.../index/balancer/IndexLoadBalancerIT.java | 6 +-
.../phoenix/trace/PhoenixTraceReaderIT.java | 2 +-
.../phoenix/trace/PhoenixTracingEndToEndIT.java | 20 +-
.../regionserver/IndexHalfStoreFileReader.java | 41 ++++-
.../IndexHalfStoreFileReaderGenerator.java | 14 +-
.../regionserver/IndexSplitTransaction.java | 28 +--
.../hbase/regionserver/KeyValueSkipListSet.java | 183 +++++++++++++++++++
.../hbase/regionserver/LocalIndexMerger.java | 4 +-
.../hbase/regionserver/LocalIndexSplitter.java | 29 +--
.../apache/phoenix/compile/TraceQueryPlan.java | 14 +-
.../coprocessor/BaseScannerRegionObserver.java | 4 +-
.../apache/phoenix/execute/BaseQueryPlan.java | 2 +-
.../apache/phoenix/execute/MutationState.java | 4 +-
.../org/apache/phoenix/hbase/index/Indexer.java | 10 +-
.../hbase/index/balancer/IndexLoadBalancer.java | 5 +
.../hbase/index/covered/data/IndexMemStore.java | 27 +--
.../index/covered/data/LazyValueGetter.java | 5 +-
.../example/CoveredColumnIndexCodec.java | 6 +-
.../filter/ApplyAndFilterDeletesFilter.java | 8 +-
.../index/covered/update/ColumnReference.java | 10 +-
.../ipc/PhoenixIndexRpcSchedulerFactory.java | 19 +-
.../index/scanner/FilteredKeyValueScanner.java | 17 +-
.../phoenix/hbase/index/scanner/Scanner.java | 5 +-
.../hbase/index/scanner/ScannerBuilder.java | 10 +-
.../hbase/index/wal/IndexedKeyValue.java | 17 --
.../apache/phoenix/jdbc/PhoenixConnection.java | 4 +-
.../apache/phoenix/trace/TraceMetricSource.java | 15 +-
.../org/apache/phoenix/trace/TraceReader.java | 2 +-
.../apache/phoenix/trace/TracingIterator.java | 2 +-
.../org/apache/phoenix/trace/TracingUtils.java | 2 +-
.../org/apache/phoenix/trace/util/NullSpan.java | 10 +-
.../org/apache/phoenix/trace/util/Tracing.java | 51 ++++--
.../java/org/apache/phoenix/util/IndexUtil.java | 12 +-
.../hbase/ipc/PhoenixIndexRpcSchedulerTest.java | 4 +-
.../PhoenixIndexRpcSchedulerFactoryTest.java | 6 +-
.../index/covered/TestLocalTableState.java | 8 +-
.../index/covered/data/TestIndexMemStore.java | 5 +-
.../index/write/TestWALRecoveryCaching.java | 14 +-
.../recovery/TestPerRegionIndexWriteCache.java | 15 +-
.../phoenix/trace/TraceMetricsSourceTest.java | 4 +-
phoenix-flume/pom.xml | 4 +-
pom.xml | 14 +-
47 files changed, 481 insertions(+), 261 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-assembly/src/build/client.xml
----------------------------------------------------------------------
diff --git a/phoenix-assembly/src/build/client.xml b/phoenix-assembly/src/build/client.xml
index f674331..101ccd6 100644
--- a/phoenix-assembly/src/build/client.xml
+++ b/phoenix-assembly/src/build/client.xml
@@ -46,8 +46,8 @@
<include>jline:jline</include>
<include>sqlline:sqlline</include>
<include>org.apache.hbase:hbase*</include>
- <include>org.cloudera.htrace:htrace-core</include>
- <include>io.netty:netty</include>
+ <include>org.apache.htrace:htrace-core</include>
+ <include>io.netty:netty-all</include>
<include>commons-codec:commons-codec</include>
</includes>
</dependencySet>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-core/pom.xml b/phoenix-core/pom.xml
index a325b27..d4dc2e2 100644
--- a/phoenix-core/pom.xml
+++ b/phoenix-core/pom.xml
@@ -318,12 +318,12 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.cloudera.htrace</groupId>
+ <groupId>org.apache.htrace</groupId>
<artifactId>htrace-core</artifactId>
</dependency>
<dependency>
<groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
+ <artifactId>netty-all</artifactId>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/it/java/org/apache/hadoop/hbase/regionserver/wal/WALReplayWithIndexWritesAndCompressedWALIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/hadoop/hbase/regionserver/wal/WALReplayWithIndexWritesAndCompressedWALIT.java b/phoenix-core/src/it/java/org/apache/hadoop/hbase/regionserver/wal/WALReplayWithIndexWritesAndCompressedWALIT.java
index 8cf8a8a..3b8ff29 100644
--- a/phoenix-core/src/it/java/org/apache/hadoop/hbase/regionserver/wal/WALReplayWithIndexWritesAndCompressedWALIT.java
+++ b/phoenix-core/src/it/java/org/apache/hadoop/hbase/regionserver/wal/WALReplayWithIndexWritesAndCompressedWALIT.java
@@ -48,6 +48,9 @@ import org.apache.hadoop.hbase.regionserver.RegionServerAccounting;
import org.apache.hadoop.hbase.regionserver.RegionServerServices;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.wal.WAL;
+import org.apache.hadoop.hbase.wal.WALFactory;
+import org.apache.hadoop.hbase.wal.WALSplitter;
import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
import org.apache.phoenix.hbase.index.IndexTestingUtils;
import org.apache.phoenix.hbase.index.TableName;
@@ -65,7 +68,7 @@ import org.mockito.Mockito;
/**
* For pre-0.94.9 instances, this class tests correctly deserializing WALEdits w/o compression. Post
- * 0.94.9 we can support a custom {@link WALEditCodec}, which handles reading/writing the compressed
+ * 0.94.9 we can support a custom {@link WALCellCodec} which handles reading/writing the compressed
* edits.
* <p>
* Most of the underlying work (creating/splitting the WAL, etc) is from
@@ -93,13 +96,12 @@ public class WALReplayWithIndexWritesAndCompressedWALIT {
@Before
public void setUp() throws Exception {
setupCluster();
+ Path hbaseRootDir = UTIL.getDataTestDir();
this.conf = HBaseConfiguration.create(UTIL.getConfiguration());
this.fs = UTIL.getDFSCluster().getFileSystem();
this.hbaseRootDir = new Path(this.conf.get(HConstants.HBASE_DIR));
this.oldLogDir = new Path(this.hbaseRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
this.logDir = new Path(this.hbaseRootDir, HConstants.HREGION_LOGDIR_NAME);
- // reset the log reader to ensure we pull the one from this config
- HLogFactory.resetLogReaderClass();
}
private void setupCluster() throws Exception {
@@ -133,11 +135,11 @@ public class WALReplayWithIndexWritesAndCompressedWALIT {
protected void startCluster() throws Exception {
UTIL.startMiniDFSCluster(3);
UTIL.startMiniZKCluster();
- UTIL.startMiniHBaseCluster(1, 1);
Path hbaseRootDir = UTIL.getDFSCluster().getFileSystem().makeQualified(new Path("/hbase"));
LOG.info("hbase.rootdir=" + hbaseRootDir);
UTIL.getConfiguration().set(HConstants.HBASE_DIR, hbaseRootDir.toString());
+ UTIL.startMiniHBaseCluster(1, 1);
}
@After
@@ -183,8 +185,11 @@ public class WALReplayWithIndexWritesAndCompressedWALIT {
// create the region + its WAL
HRegion region0 = HRegion.createHRegion(hri, hbaseRootDir, this.conf, htd);
region0.close();
- region0.getLog().closeAndDelete();
- HLog wal = createWAL(this.conf);
+ region0.getWAL().close();
+
+ WALFactory walFactory = new WALFactory(this.conf, null, "localhost,1234");
+
+ WAL wal = createWAL(this.conf, walFactory);
RegionServerServices mockRS = Mockito.mock(RegionServerServices.class);
// mock out some of the internals of the RSS, so we can run CPs
Mockito.when(mockRS.getWAL(null)).thenReturn(wal);
@@ -206,15 +211,13 @@ public class WALReplayWithIndexWritesAndCompressedWALIT {
// we should then see the server go down
Mockito.verify(mockRS, Mockito.times(1)).abort(Mockito.anyString(),
Mockito.any(Exception.class));
- region.close(true);
- wal.close();
// then create the index table so we are successful on WAL replay
CoveredColumnIndexer.createIndexTable(UTIL.getHBaseAdmin(), INDEX_TABLE_NAME);
// run the WAL split and setup the region
- runWALSplit(this.conf);
- HLog wal2 = createWAL(this.conf);
+ runWALSplit(this.conf, walFactory);
+ WAL wal2 = createWAL(this.conf, walFactory);
HRegion region1 = new HRegion(basedir, wal2, this.fs, this.conf, hri, htd, mockRS);
// initialize the region - this should replay the WALEdits from the WAL
@@ -257,8 +260,9 @@ public class WALReplayWithIndexWritesAndCompressedWALIT {
* @return WAL with retries set down from 5 to 1 only.
* @throws IOException
*/
- private HLog createWAL(final Configuration c) throws IOException {
- HLog wal = HLogFactory.createHLog(FileSystem.get(c), logDir, "localhost,1234", c);
+ private WAL createWAL(final Configuration c, WALFactory walFactory) throws IOException {
+ WAL wal = walFactory.getWAL(new byte[]{});
+
// Set down maximum recovery so we dfsclient doesn't linger retrying something
// long gone.
HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal).getOutputStream(), 1);
@@ -271,11 +275,11 @@ public class WALReplayWithIndexWritesAndCompressedWALIT {
* @return The single split file made
* @throws IOException
*/
- private Path runWALSplit(final Configuration c) throws IOException {
+ private Path runWALSplit(final Configuration c, WALFactory walFactory) throws IOException {
FileSystem fs = FileSystem.get(c);
- List<Path> splits = HLogSplitter.split(this.hbaseRootDir, new Path(this.logDir, "localhost,1234"),
- this.oldLogDir, fs, c);
+ List<Path> splits = WALSplitter.split(this.hbaseRootDir, new Path(this.logDir, "localhost,1234"),
+ this.oldLogDir, fs, c, walFactory);
// Split should generate only 1 file since there's only 1 region
assertEquals("splits=" + splits, 1, splits.size());
// Make sure the file exists
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/LocalIndexIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/LocalIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/LocalIndexIT.java
index a7b7655..5e01510 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/LocalIndexIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/LocalIndexIT.java
@@ -37,9 +37,8 @@ import java.util.concurrent.CountDownLatch;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.catalog.CatalogTracker;
-import org.apache.hadoop.hbase.catalog.MetaReader;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Result;
@@ -727,24 +726,27 @@ public class LocalIndexIT extends BaseHBaseManagedTimeIT {
HBaseAdmin admin = driver.getConnectionQueryServices(getUrl(), TestUtil.TEST_PROPERTIES).getAdmin();
for (int i = 1; i < 5; i++) {
- CatalogTracker ct = new CatalogTracker(admin.getConfiguration());
admin.split(Bytes.toBytes(TestUtil.DEFAULT_DATA_TABLE_NAME), ByteUtil.concat(Bytes.toBytes(strings[3*i])));
List<HRegionInfo> regionsOfUserTable =
- MetaReader.getTableRegions(ct, TableName.valueOf(TestUtil.DEFAULT_DATA_TABLE_NAME), false);
+ MetaTableAccessor.getTableRegions(getUtility().getZooKeeperWatcher(), admin.getConnection(),
+ TableName.valueOf(TestUtil.DEFAULT_DATA_TABLE_NAME), false);
while (regionsOfUserTable.size() != (4+i)) {
Thread.sleep(100);
- regionsOfUserTable = MetaReader.getTableRegions(ct, TableName.valueOf(TestUtil.DEFAULT_DATA_TABLE_NAME), false);
+ regionsOfUserTable = MetaTableAccessor.getTableRegions(getUtility().getZooKeeperWatcher(),
+ admin.getConnection(), TableName.valueOf(TestUtil.DEFAULT_DATA_TABLE_NAME), false);
}
assertEquals(4+i, regionsOfUserTable.size());
TableName indexTable =
TableName.valueOf(MetaDataUtil.getLocalIndexTableName(TestUtil.DEFAULT_DATA_TABLE_NAME));
List<HRegionInfo> regionsOfIndexTable =
- MetaReader.getTableRegions(ct, indexTable, false);
+ MetaTableAccessor.getTableRegions(getUtility().getZooKeeperWatcher(),
+ admin.getConnection(), indexTable, false);
while (regionsOfIndexTable.size() != (4 + i)) {
Thread.sleep(100);
- regionsOfIndexTable = MetaReader.getTableRegions(ct, indexTable, false);
+ regionsOfIndexTable = MetaTableAccessor.getTableRegions(getUtility().getZooKeeperWatcher(),
+ admin.getConnection(), indexTable, false);
}
assertEquals(4 + i, regionsOfIndexTable.size());
String query = "SELECT t_id,k1,v1 FROM " + TestUtil.DEFAULT_DATA_TABLE_NAME;
@@ -847,32 +849,32 @@ public class LocalIndexIT extends BaseHBaseManagedTimeIT {
assertTrue(rs.next());
HBaseAdmin admin = driver.getConnectionQueryServices(getUrl(), TestUtil.TEST_PROPERTIES).getAdmin();
- CatalogTracker ct = new CatalogTracker(admin.getConfiguration());
List<HRegionInfo> regionsOfUserTable =
- MetaReader.getTableRegions(ct,
+ MetaTableAccessor.getTableRegions(getUtility().getZooKeeperWatcher(), admin.getConnection(),
TableName.valueOf(TestUtil.DEFAULT_DATA_TABLE_NAME), false);
admin.mergeRegions(regionsOfUserTable.get(0).getEncodedNameAsBytes(),
regionsOfUserTable.get(1).getEncodedNameAsBytes(), false);
regionsOfUserTable =
- MetaReader.getTableRegions(ct,
+ MetaTableAccessor.getTableRegions(getUtility().getZooKeeperWatcher(), admin.getConnection(),
TableName.valueOf(TestUtil.DEFAULT_DATA_TABLE_NAME), false);
while (regionsOfUserTable.size() != 3) {
Thread.sleep(100);
- regionsOfUserTable =
- MetaReader.getTableRegions(ct,
- TableName.valueOf(TestUtil.DEFAULT_DATA_TABLE_NAME), false);
+ regionsOfUserTable = MetaTableAccessor.getTableRegions(getUtility().getZooKeeperWatcher(),
+ admin.getConnection(), TableName.valueOf(TestUtil.DEFAULT_DATA_TABLE_NAME), false);
}
assertEquals(3, regionsOfUserTable.size());
TableName indexTable =
TableName.valueOf(MetaDataUtil
.getLocalIndexTableName(TestUtil.DEFAULT_DATA_TABLE_NAME));
List<HRegionInfo> regionsOfIndexTable =
- MetaReader.getTableRegions(ct, indexTable, false);
+ MetaTableAccessor.getTableRegions(getUtility().getZooKeeperWatcher(),
+ admin.getConnection(), indexTable, false);
while (regionsOfIndexTable.size() != 3) {
Thread.sleep(100);
- regionsOfIndexTable = MetaReader.getTableRegions(ct, indexTable, false);
+ regionsOfIndexTable = MetaTableAccessor.getTableRegions(
+ getUtility().getZooKeeperWatcher(), admin.getConnection(), indexTable, false);
}
assertEquals(3, regionsOfIndexTable.size());
String query = "SELECT t_id,k1,v1 FROM " + TestUtil.DEFAULT_DATA_TABLE_NAME;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
index d11c059..dfc7ffb 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
@@ -304,7 +304,8 @@ public class MutableIndexFailureIT extends BaseTest {
Collection<ServerName> rss = cluster.getClusterStatus().getServers();
HBaseAdmin admin = this.util.getHBaseAdmin();
List<HRegionInfo> regions = admin.getTableRegions(catalogTable);
- ServerName catalogRS = cluster.getServerHoldingRegion(regions.get(0).getRegionName());
+ ServerName catalogRS = cluster.getServerHoldingRegion(regions.get(0).getTable(),
+ regions.get(0).getRegionName());
ServerName metaRS = cluster.getServerHoldingMeta();
ServerName rsToBeKilled = null;
@@ -324,7 +325,8 @@ public class MutableIndexFailureIT extends BaseTest {
this.util.waitFor(30000, 200, new Waiter.Predicate<Exception>() {
@Override
public boolean evaluate() throws Exception {
- ServerName sn = cluster.getServerHoldingRegion(indexRegion.getRegionName());
+ ServerName sn = cluster.getServerHoldingRegion(indexRegion.getTable(),
+ indexRegion.getRegionName());
return (sn != null && sn.equals(dstRS));
}
});
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/it/java/org/apache/phoenix/hbase/index/balancer/IndexLoadBalancerIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/hbase/index/balancer/IndexLoadBalancerIT.java b/phoenix-core/src/it/java/org/apache/phoenix/hbase/index/balancer/IndexLoadBalancerIT.java
index d534b6a..449dccf 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/hbase/index/balancer/IndexLoadBalancerIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/hbase/index/balancer/IndexLoadBalancerIT.java
@@ -29,10 +29,10 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.catalog.MetaReader;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
@@ -447,8 +447,8 @@ public class IndexLoadBalancerIT {
throws IOException, InterruptedException {
List<Pair<HRegionInfo, ServerName>> tableRegionsAndLocations =
- MetaReader.getTableRegionsAndLocations(master.getCatalogTracker(), TableName
- .valueOf(tableName));
+ MetaTableAccessor.getTableRegionsAndLocations(master.getZooKeeper(), master.getConnection(),
+ TableName.valueOf(tableName));
List<Pair<byte[], ServerName>> startKeyAndLocationPairs =
new ArrayList<Pair<byte[], ServerName>>(tableRegionsAndLocations.size());
Pair<byte[], ServerName> startKeyAndLocation = null;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/it/java/org/apache/phoenix/trace/PhoenixTraceReaderIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/trace/PhoenixTraceReaderIT.java b/phoenix-core/src/it/java/org/apache/phoenix/trace/PhoenixTraceReaderIT.java
index 1308c13..2315074 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/trace/PhoenixTraceReaderIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/trace/PhoenixTraceReaderIT.java
@@ -38,7 +38,7 @@ import org.apache.hadoop.metrics2.MetricsTag;
import org.apache.phoenix.metrics.MetricInfo;
import org.apache.phoenix.trace.TraceReader.SpanInfo;
import org.apache.phoenix.trace.TraceReader.TraceHolder;
-import org.cloudera.htrace.Span;
+import org.apache.htrace.Span;
import org.junit.Test;
/**
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/it/java/org/apache/phoenix/trace/PhoenixTracingEndToEndIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/trace/PhoenixTracingEndToEndIT.java b/phoenix-core/src/it/java/org/apache/phoenix/trace/PhoenixTracingEndToEndIT.java
index 05d9e41..8febfff 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/trace/PhoenixTracingEndToEndIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/trace/PhoenixTracingEndToEndIT.java
@@ -35,18 +35,18 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.metrics2.MetricsSource;
+import org.apache.htrace.Sampler;
+import org.apache.htrace.Span;
+import org.apache.htrace.SpanReceiver;
+import org.apache.htrace.Trace;
+import org.apache.htrace.TraceScope;
+import org.apache.htrace.impl.ProbabilitySampler;
import org.apache.phoenix.coprocessor.BaseScannerRegionObserver;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.metrics.Metrics;
import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.trace.TraceReader.SpanInfo;
import org.apache.phoenix.trace.TraceReader.TraceHolder;
-import org.cloudera.htrace.Sampler;
-import org.cloudera.htrace.Span;
-import org.cloudera.htrace.SpanReceiver;
-import org.cloudera.htrace.Trace;
-import org.cloudera.htrace.TraceScope;
-import org.cloudera.htrace.impl.ProbabilitySampler;
import org.junit.After;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -349,7 +349,7 @@ public class PhoenixTracingEndToEndIT extends BaseTracingTestIT {
});
assertTrue("Didn't find the parallel scanner in the tracing", found);
}
-
+
@Test
public void testCustomAnnotationTracing() throws Exception {
final String customAnnotationKey = "myannot";
@@ -375,7 +375,7 @@ public class PhoenixTracingEndToEndIT extends BaseTracingTestIT {
stmt.execute();
conn.commit();
conn.rollback();
-
+
// setup for next set of updates
stmt.setString(1, "key2");
stmt.setLong(2, 2);
@@ -456,10 +456,10 @@ public class PhoenixTracingEndToEndIT extends BaseTracingTestIT {
return currentTrace.toString().contains(annotationKey + " - " + annotationValue);
}
});
-
+
assertTrue("Didn't find the custom annotation in the tracing", tracingComplete);
}
-
+
private boolean checkStoredTraces(Connection conn, TraceChecker checker) throws Exception {
TraceReader reader = new TraceReader(conn);
int retries = 0;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReader.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReader.java b/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReader.java
index 172486d..654daf0 100644
--- a/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReader.java
+++ b/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReader.java
@@ -25,11 +25,13 @@ import java.util.Map.Entry;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.Type;
+import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
@@ -159,7 +161,7 @@ public class IndexHalfStoreFileReader extends StoreFile.Reader {
return getChangedKey(delegate.getKeyValue(), changeBottomKeys);
}
- private ByteBuffer getChangedKey(KeyValue kv, boolean changeBottomKeys) {
+ private ByteBuffer getChangedKey(Cell kv, boolean changeBottomKeys) {
// new KeyValue(row, family, qualifier, timestamp, type, value)
byte[] newRowkey = getNewRowkeyByRegionStartKeyReplacedWithSplitKey(kv, changeBottomKeys);
KeyValue newKv =
@@ -171,7 +173,7 @@ public class IndexHalfStoreFileReader extends StoreFile.Reader {
return keyBuffer;
}
- private byte[] getNewRowkeyByRegionStartKeyReplacedWithSplitKey(KeyValue kv, boolean changeBottomKeys) {
+ private byte[] getNewRowkeyByRegionStartKeyReplacedWithSplitKey(Cell kv, boolean changeBottomKeys) {
int lenOfRemainingKey = kv.getRowLength() - offset;
byte[] keyReplacedStartKey = new byte[lenOfRemainingKey + splitRow.length];
System.arraycopy(changeBottomKeys ? new byte[splitRow.length] : splitRow, 0,
@@ -202,11 +204,11 @@ public class IndexHalfStoreFileReader extends StoreFile.Reader {
return Bytes.toStringBinary(getValue());
}
- public KeyValue getKeyValue() {
+ public Cell getKeyValue() {
if (atEnd) {
return null;
}
- KeyValue kv = delegate.getKeyValue();
+ Cell kv = delegate.getKeyValue();
boolean changeBottomKeys =
regionInfo.getStartKey().length == 0 && splitRow.length != offset;
if (!top) {
@@ -221,7 +223,7 @@ public class IndexHalfStoreFileReader extends StoreFile.Reader {
kv.getQualifierOffset(), kv.getQualifierLength(),
kv.getTimestamp(), Type.codeToType(kv.getTypeByte()),
kv.getValueArray(), kv.getValueOffset(), kv.getValueLength(),
- kv.getTags());
+ kv.getTagsArray(), kv.getTagsOffset(), kv.getTagsLength());
return changedKv;
}
@@ -251,6 +253,7 @@ public class IndexHalfStoreFileReader extends StoreFile.Reader {
}
public boolean seekBefore(byte[] key, int offset, int length) throws IOException {
+
if (top) {
byte[] fk = getFirstKey();
// This will be null when the file is empty in which we can not seekBefore to
@@ -262,8 +265,7 @@ public class IndexHalfStoreFileReader extends StoreFile.Reader {
return false;
}
KeyValue replacedKey = getKeyPresentInHFiles(key);
- return this.delegate.seekBefore(replacedKey.getBuffer(),
- replacedKey.getKeyOffset(), replacedKey.getKeyLength());
+ return this.delegate.seekBefore(replacedKey);
} else {
// The equals sign isn't strictly necessary just here to be consistent with
// seekTo
@@ -274,6 +276,12 @@ public class IndexHalfStoreFileReader extends StoreFile.Reader {
return this.delegate.seekBefore(key, offset, length);
}
+ @Override
+ public boolean seekBefore(Cell cell) throws IOException {
+ KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
+ return seekBefore(kv.getBuffer(), kv.getKeyOffset(), kv.getKeyLength());
+ }
+
public boolean seekTo() throws IOException {
boolean b = delegate.seekTo();
if (!b) {
@@ -328,6 +336,12 @@ public class IndexHalfStoreFileReader extends StoreFile.Reader {
return delegate.seekTo(key, offset, length);
}
+ @Override
+ public int seekTo(Cell cell) throws IOException {
+ KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
+ return seekTo(kv.getBuffer(), kv.getKeyOffset(), kv.getKeyLength());
+ }
+
public int reseekTo(byte[] key) throws IOException {
return reseekTo(key, 0, key.length);
}
@@ -355,6 +369,12 @@ public class IndexHalfStoreFileReader extends StoreFile.Reader {
return delegate.reseekTo(key, offset, length);
}
+ @Override
+ public int reseekTo(Cell cell) throws IOException {
+ KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
+ return reseekTo(kv.getBuffer(), kv.getKeyOffset(), kv.getKeyLength());
+ }
+
public org.apache.hadoop.hbase.io.hfile.HFile.Reader getReader() {
return this.delegate.getReader();
}
@@ -373,7 +393,7 @@ public class IndexHalfStoreFileReader extends StoreFile.Reader {
};
}
- private boolean isSatisfiedMidKeyCondition(KeyValue kv) {
+ private boolean isSatisfiedMidKeyCondition(Cell kv) {
if (CellUtil.isDelete(kv) && kv.getValueLength() == 0) {
// In case of a Delete type KV, let it be going to both the daughter regions.
// No problems in doing so. In the correct daughter region where it belongs to, this delete
@@ -428,9 +448,10 @@ public class IndexHalfStoreFileReader extends StoreFile.Reader {
&& keyValue.getTimestamp() == HConstants.LATEST_TIMESTAMP
&& Bytes.compareTo(keyValue.getRowArray(), keyValue.getRowOffset(),
keyValue.getRowLength(), splitRow, 0, splitRow.length) == 0
- && keyValue.isDeleteFamily()) {
+ && CellUtil.isDeleteFamily(keyValue)) {
KeyValue createFirstDeleteFamilyOnRow =
- KeyValue.createFirstDeleteFamilyOnRow(regionStartKeyInHFile, keyValue.getFamily());
+ KeyValueUtil.createFirstDeleteFamilyOnRow(regionStartKeyInHFile,
+ keyValue.getFamily());
return createFirstDeleteFamilyOnRow;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReaderGenerator.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReaderGenerator.java b/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReaderGenerator.java
index 718f820..1284dcf 100644
--- a/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReaderGenerator.java
+++ b/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReaderGenerator.java
@@ -30,8 +30,8 @@ import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.catalog.MetaReader;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
@@ -80,7 +80,7 @@ public class IndexHalfStoreFileReaderGenerator extends BaseRegionObserver {
HRegionInfo childRegion = region.getRegionInfo();
byte[] splitKey = null;
if (reader == null && r != null) {
- Scan scan = MetaReader.getScanForTableName(tableName);
+ Scan scan = MetaTableAccessor.getScanForTableName(tableName);
SingleColumnValueFilter scvf = null;
if (Reference.isTopFileRegion(r.getFileRegion())) {
scvf = new SingleColumnValueFilter(HConstants.CATALOG_FAMILY,
@@ -107,8 +107,8 @@ public class IndexHalfStoreFileReaderGenerator extends BaseRegionObserver {
}
if (result == null || result.isEmpty()) {
Pair<HRegionInfo, HRegionInfo> mergeRegions =
- MetaReader.getRegionsFromMergeQualifier(ctx.getEnvironment()
- .getRegionServerServices().getCatalogTracker(),
+ MetaTableAccessor.getRegionsFromMergeQualifier(ctx.getEnvironment()
+ .getRegionServerServices().getConnection(),
region.getRegionName());
if (mergeRegions == null || mergeRegions.getFirst() == null) return reader;
byte[] splitRow =
@@ -121,10 +121,8 @@ public class IndexHalfStoreFileReaderGenerator extends BaseRegionObserver {
childRegion = mergeRegions.getSecond();
regionStartKeyInHFile = mergeRegions.getSecond().getStartKey();
}
- splitKey =
- KeyValue.createFirstOnRow(
- region.getStartKey().length == 0 ? new byte[region.getEndKey().length] : region
- .getStartKey()).getKey();
+ splitKey = KeyValue.createFirstOnRow(region.getStartKey().length == 0 ?
+ new byte[region.getEndKey().length] : region.getStartKey()).getKey();
} else {
HRegionInfo parentRegion = HRegionInfo.getHRegionInfo(result);
regionStartKeyInHFile =
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexSplitTransaction.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexSplitTransaction.java b/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexSplitTransaction.java
index 048506d..920380b 100644
--- a/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexSplitTransaction.java
+++ b/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexSplitTransaction.java
@@ -42,11 +42,11 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.RegionTransition;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.catalog.CatalogTracker;
-import org.apache.hadoop.hbase.catalog.MetaEditor;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.executor.EventType;
@@ -286,11 +286,11 @@ public class IndexSplitTransaction extends SplitTransaction {
// and assign the parent region.
if (!testing) {
if (metaEntries == null || metaEntries.isEmpty()) {
- MetaEditor.splitRegion(server.getCatalogTracker(),
- parent.getRegionInfo(), daughterRegions.getFirst().getRegionInfo(),
- daughterRegions.getSecond().getRegionInfo(), server.getServerName());
+ MetaTableAccessor.splitRegion(server.getConnection(), parent.getRegionInfo(),
+ daughterRegions.getFirst().getRegionInfo(),
+ daughterRegions.getSecond().getRegionInfo(), server.getServerName());
} else {
- offlineParentInMetaAndputMetaEntries(server.getCatalogTracker(),
+ offlineParentInMetaAndputMetaEntries(server.getConnection(),
parent.getRegionInfo(), daughterRegions.getFirst().getRegionInfo(), daughterRegions
.getSecond().getRegionInfo(), server.getServerName(), metaEntries);
}
@@ -415,10 +415,10 @@ public class IndexSplitTransaction extends SplitTransaction {
if (services != null) {
try {
// add 2nd daughter first (see HBASE-4335)
- services.postOpenDeployTasks(b, server.getCatalogTracker());
+ services.postOpenDeployTasks(b);
// Should add it to OnlineRegions
services.addToOnlineRegions(b);
- services.postOpenDeployTasks(a, server.getCatalogTracker());
+ services.postOpenDeployTasks(a);
services.addToOnlineRegions(a);
} catch (KeeperException ke) {
throw new IOException(ke);
@@ -583,7 +583,7 @@ public class IndexSplitTransaction extends SplitTransaction {
return regions;
}
- private void offlineParentInMetaAndputMetaEntries(CatalogTracker catalogTracker,
+ private void offlineParentInMetaAndputMetaEntries(Connection conn,
HRegionInfo parent, HRegionInfo splitA, HRegionInfo splitB,
ServerName serverName, List<Mutation> metaEntries) throws IOException {
List<Mutation> mutations = metaEntries;
@@ -592,19 +592,19 @@ public class IndexSplitTransaction extends SplitTransaction {
copyOfParent.setSplit(true);
//Put for parent
- Put putParent = MetaEditor.makePutFromRegionInfo(copyOfParent);
- MetaEditor.addDaughtersToPut(putParent, splitA, splitB);
+ Put putParent = MetaTableAccessor.makePutFromRegionInfo(copyOfParent);
+ MetaTableAccessor.addDaughtersToPut(putParent, splitA, splitB);
mutations.add(putParent);
//Puts for daughters
- Put putA = MetaEditor.makePutFromRegionInfo(splitA);
- Put putB = MetaEditor.makePutFromRegionInfo(splitB);
+ Put putA = MetaTableAccessor.makePutFromRegionInfo(splitA);
+ Put putB = MetaTableAccessor.makePutFromRegionInfo(splitB);
addLocation(putA, serverName, 1); //these are new regions, openSeqNum = 1 is fine.
addLocation(putB, serverName, 1);
mutations.add(putA);
mutations.add(putB);
- MetaEditor.mutateMetaTable(catalogTracker, mutations);
+ MetaTableAccessor.mutateMetaTable(conn, mutations);
}
public Put addLocation(final Put p, final ServerName sn, long openSeqNum) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueSkipListSet.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueSkipListSet.java b/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueSkipListSet.java
new file mode 100644
index 0000000..211aa10
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueSkipListSet.java
@@ -0,0 +1,183 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.KeyValue;
+
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.NavigableSet;
+import java.util.SortedSet;
+import java.util.concurrent.ConcurrentNavigableMap;
+import java.util.concurrent.ConcurrentSkipListMap;
+
+/**
+ * A {@link java.util.Set} of {@link KeyValue}s implemented on top of a
+ * {@link java.util.concurrent.ConcurrentSkipListMap}. Works like a
+ * {@link java.util.concurrent.ConcurrentSkipListSet} in all but one regard:
+ * An add will overwrite if already an entry for the added key. In other words,
+ * where CSLS does "Adds the specified element to this set if it is not already
+ * present.", this implementation "Adds the specified element to this set EVEN
+ * if it is already present overwriting what was there previous". The call to
+ * add returns true if no value in the backing map or false if there was an
+ * entry with same key (though value may be different).
+ * <p>Otherwise,
+ * has same attributes as ConcurrentSkipListSet: e.g. tolerant of concurrent
+ * get and set and won't throw ConcurrentModificationException when iterating.
+ */
+public class KeyValueSkipListSet implements NavigableSet<KeyValue> {
+ private final ConcurrentNavigableMap<KeyValue, KeyValue> delegatee;
+
+ KeyValueSkipListSet(final KeyValue.KVComparator c) {
+ this.delegatee = new ConcurrentSkipListMap<KeyValue, KeyValue>(c);
+ }
+
+ KeyValueSkipListSet(final ConcurrentNavigableMap<KeyValue, KeyValue> m) {
+ this.delegatee = m;
+ }
+
+ public KeyValue ceiling(KeyValue e) {
+ throw new UnsupportedOperationException("Not implemented");
+ }
+
+ public Iterator<KeyValue> descendingIterator() {
+ return this.delegatee.descendingMap().values().iterator();
+ }
+
+ public NavigableSet<KeyValue> descendingSet() {
+ throw new UnsupportedOperationException("Not implemented");
+ }
+
+ public KeyValue floor(KeyValue e) {
+ throw new UnsupportedOperationException("Not implemented");
+ }
+
+ public SortedSet<KeyValue> headSet(final KeyValue toElement) {
+ return headSet(toElement, false);
+ }
+
+ public NavigableSet<KeyValue> headSet(final KeyValue toElement,
+ boolean inclusive) {
+ return new KeyValueSkipListSet(this.delegatee.headMap(toElement, inclusive));
+ }
+
+ public KeyValue higher(KeyValue e) {
+ throw new UnsupportedOperationException("Not implemented");
+ }
+
+ public Iterator<KeyValue> iterator() {
+ return this.delegatee.values().iterator();
+ }
+
+ public KeyValue lower(KeyValue e) {
+ throw new UnsupportedOperationException("Not implemented");
+ }
+
+ public KeyValue pollFirst() {
+ throw new UnsupportedOperationException("Not implemented");
+ }
+
+ public KeyValue pollLast() {
+ throw new UnsupportedOperationException("Not implemented");
+ }
+
+ public SortedSet<KeyValue> subSet(KeyValue fromElement, KeyValue toElement) {
+ throw new UnsupportedOperationException("Not implemented");
+ }
+
+ public NavigableSet<KeyValue> subSet(KeyValue fromElement,
+ boolean fromInclusive, KeyValue toElement, boolean toInclusive) {
+ throw new UnsupportedOperationException("Not implemented");
+ }
+
+ public SortedSet<KeyValue> tailSet(KeyValue fromElement) {
+ return tailSet(fromElement, true);
+ }
+
+ public NavigableSet<KeyValue> tailSet(KeyValue fromElement, boolean inclusive) {
+ return new KeyValueSkipListSet(this.delegatee.tailMap(fromElement, inclusive));
+ }
+
+ public Comparator<? super KeyValue> comparator() {
+ throw new UnsupportedOperationException("Not implemented");
+ }
+
+ public KeyValue first() {
+ return this.delegatee.get(this.delegatee.firstKey());
+ }
+
+ public KeyValue last() {
+ return this.delegatee.get(this.delegatee.lastKey());
+ }
+
+ public boolean add(KeyValue e) {
+ return this.delegatee.put(e, e) == null;
+ }
+
+ public boolean addAll(Collection<? extends KeyValue> c) {
+ throw new UnsupportedOperationException("Not implemented");
+ }
+
+ public void clear() {
+ this.delegatee.clear();
+ }
+
+ public boolean contains(Object o) {
+ //noinspection SuspiciousMethodCalls
+ return this.delegatee.containsKey(o);
+ }
+
+ public boolean containsAll(Collection<?> c) {
+ throw new UnsupportedOperationException("Not implemented");
+ }
+
+ public boolean isEmpty() {
+ return this.delegatee.isEmpty();
+ }
+
+ public boolean remove(Object o) {
+ return this.delegatee.remove(o) != null;
+ }
+
+ public boolean removeAll(Collection<?> c) {
+ throw new UnsupportedOperationException("Not implemented");
+ }
+
+ public boolean retainAll(Collection<?> c) {
+ throw new UnsupportedOperationException("Not implemented");
+ }
+
+ public KeyValue get(KeyValue kv) {
+ return this.delegatee.get(kv);
+ }
+
+ public int size() {
+ return this.delegatee.size();
+ }
+
+ public Object[] toArray() {
+ throw new UnsupportedOperationException("Not implemented");
+ }
+
+ public <T> T[] toArray(T[] a) {
+ throw new UnsupportedOperationException("Not implemented");
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/LocalIndexMerger.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/LocalIndexMerger.java b/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/LocalIndexMerger.java
index 6f8dd79..f074df7 100644
--- a/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/LocalIndexMerger.java
+++ b/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/LocalIndexMerger.java
@@ -23,8 +23,8 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.catalog.MetaReader;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.coprocessor.BaseRegionServerObserver;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
@@ -55,7 +55,7 @@ public class LocalIndexMerger extends BaseRegionServerObserver {
.getValue(MetaDataUtil.IS_LOCAL_INDEX_TABLE_PROP_BYTES)))) {
TableName indexTable =
TableName.valueOf(MetaDataUtil.getLocalIndexPhysicalName(tableDesc.getName()));
- if (!MetaReader.tableExists(rs.getCatalogTracker(), indexTable)) return;
+ if (!MetaTableAccessor.tableExists(rs.getConnection(), indexTable)) return;
HRegion indexRegionA = IndexUtil.getIndexRegion(regionA, ctx.getEnvironment());
if (indexRegionA == null) {
LOG.warn("Index region corresponindg to data region " + regionA
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/LocalIndexSplitter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/LocalIndexSplitter.java b/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/LocalIndexSplitter.java
index 2ac61cb..9af8251 100644
--- a/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/LocalIndexSplitter.java
+++ b/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/LocalIndexSplitter.java
@@ -17,17 +17,12 @@
*/
package org.apache.hadoop.hbase.regionserver;
-import java.io.IOException;
-import java.sql.SQLException;
-import java.util.List;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.catalog.MetaEditor;
-import org.apache.hadoop.hbase.catalog.MetaReader;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
@@ -39,16 +34,20 @@ import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.parse.AlterIndexStatement;
import org.apache.phoenix.parse.ParseNodeFactory;
import org.apache.phoenix.schema.MetaDataClient;
-import org.apache.phoenix.schema.types.PBoolean;
import org.apache.phoenix.schema.PIndexState;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTable.IndexType;
+import org.apache.phoenix.schema.types.PBoolean;
import org.apache.phoenix.util.IndexUtil;
import org.apache.phoenix.util.MetaDataUtil;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.SchemaUtil;
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.List;
+
public class LocalIndexSplitter extends BaseRegionObserver {
private static final Log LOG = LogFactory.getLog(LocalIndexSplitter.class);
@@ -73,7 +72,7 @@ public class LocalIndexSplitter extends BaseRegionObserver {
.getValue(MetaDataUtil.IS_LOCAL_INDEX_TABLE_PROP_BYTES)))) {
TableName indexTable =
TableName.valueOf(MetaDataUtil.getLocalIndexPhysicalName(tableDesc.getName()));
- if (!MetaReader.tableExists(rss.getCatalogTracker(), indexTable)) return;
+ if (!MetaTableAccessor.tableExists(rss.getConnection(), indexTable)) return;
HRegion indexRegion = IndexUtil.getIndexRegion(environment);
if (indexRegion == null) {
@@ -105,14 +104,16 @@ public class LocalIndexSplitter extends BaseRegionObserver {
copyOfParent.setOffline(true);
copyOfParent.setSplit(true);
// Put for parent
- Put putParent = MetaEditor.makePutFromRegionInfo(copyOfParent);
- MetaEditor.addDaughtersToPut(putParent, daughterRegions.getFirst().getRegionInfo(),
- daughterRegions.getSecond().getRegionInfo());
+ Put putParent = MetaTableAccessor.makePutFromRegionInfo(copyOfParent);
+ MetaTableAccessor.addDaughtersToPut(putParent,
+ daughterRegions.getFirst().getRegionInfo(),
+ daughterRegions.getSecond().getRegionInfo());
metaEntries.add(putParent);
// Puts for daughters
- Put putA = MetaEditor.makePutFromRegionInfo(daughterRegions.getFirst().getRegionInfo());
- Put putB =
- MetaEditor.makePutFromRegionInfo(daughterRegions.getSecond().getRegionInfo());
+ Put putA = MetaTableAccessor.makePutFromRegionInfo(
+ daughterRegions.getFirst().getRegionInfo());
+ Put putB = MetaTableAccessor.makePutFromRegionInfo(
+ daughterRegions.getSecond().getRegionInfo());
st.addLocation(putA, rss.getServerName(), 1);
st.addLocation(putB, rss.getServerName(), 1);
metaEntries.add(putA);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/main/java/org/apache/phoenix/compile/TraceQueryPlan.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/TraceQueryPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/TraceQueryPlan.java
index 815ac1e..3b601b0 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/TraceQueryPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/TraceQueryPlan.java
@@ -30,6 +30,8 @@ import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.htrace.Sampler;
+import org.apache.htrace.TraceScope;
import org.apache.phoenix.compile.GroupByCompiler.GroupBy;
import org.apache.phoenix.compile.OrderByCompiler.OrderBy;
import org.apache.phoenix.expression.Determinism;
@@ -58,9 +60,6 @@ import org.apache.phoenix.schema.types.PLong;
import org.apache.phoenix.trace.util.Tracing;
import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.SizedUtil;
-import org.cloudera.htrace.Sampler;
-import org.cloudera.htrace.TraceScope;
-import org.cloudera.htrace.impl.ProbabilitySampler;
public class TraceQueryPlan implements QueryPlan {
@@ -124,14 +123,9 @@ public class TraceQueryPlan implements QueryPlan {
if(!first) return null;
TraceScope traceScope = conn.getTraceScope();
if (traceStatement.isTraceOn()) {
- double samplingRate = traceStatement.getSamplingRate();
- if (samplingRate >= 1.0) {
- conn.setSampler(Sampler.ALWAYS);
- } else if (samplingRate < 1.0 && samplingRate > 0.0) {
- conn.setSampler(new ProbabilitySampler(samplingRate));
- } else {
+ conn.setSampler(Tracing.getConfiguredSampler(traceStatement));
+ if (conn.getSampler() == Sampler.NEVER) {
closeTraceScope(conn);
- conn.setSampler(Sampler.NEVER);
}
if (traceScope == null && !conn.getSampler().equals(Sampler.NEVER)) {
traceScope = Tracing.startNewSpan(conn, "Enabling trace");
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
index 25ac408..c3988a0 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
@@ -53,8 +53,8 @@ import org.apache.phoenix.schema.tuple.Tuple;
import org.apache.phoenix.util.IndexUtil;
import org.apache.phoenix.util.ScanUtil;
import org.apache.phoenix.util.ServerUtil;
-import org.cloudera.htrace.Span;
-import org.cloudera.htrace.Trace;
+import org.apache.htrace.Span;
+import org.apache.htrace.Trace;
import com.google.common.collect.ImmutableList;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
index 94233c8..4ca2dee 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
@@ -66,7 +66,7 @@ import org.apache.phoenix.util.LogUtil;
import org.apache.phoenix.util.SQLCloseable;
import org.apache.phoenix.util.SQLCloseables;
import org.apache.phoenix.util.ScanUtil;
-import org.cloudera.htrace.TraceScope;
+import org.apache.htrace.TraceScope;
import com.google.common.collect.Lists;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
index b98d705..467746b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
@@ -54,8 +54,8 @@ import org.apache.phoenix.util.LogUtil;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.SQLCloseable;
import org.apache.phoenix.util.ServerUtil;
-import org.cloudera.htrace.Span;
-import org.cloudera.htrace.TraceScope;
+import org.apache.htrace.Span;
+import org.apache.htrace.TraceScope;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
index a4fc96b..1c3d1e2 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
@@ -65,9 +65,9 @@ import org.apache.phoenix.hbase.index.write.recovery.StoreFailuresInCachePolicy;
import org.apache.phoenix.hbase.index.write.recovery.TrackingParallelWriterIndexCommitter;
import org.apache.phoenix.trace.TracingUtils;
import org.apache.phoenix.trace.util.NullSpan;
-import org.cloudera.htrace.Span;
-import org.cloudera.htrace.Trace;
-import org.cloudera.htrace.TraceScope;
+import org.apache.htrace.Span;
+import org.apache.htrace.Trace;
+import org.apache.htrace.TraceScope;
import com.google.common.collect.Multimap;
@@ -475,7 +475,7 @@ public class Indexer extends BaseRegionObserver {
* present
*/
private IndexedKeyValue getFirstIndexedKeyValue(WALEdit edit) {
- for (KeyValue kv : edit.getKeyValues()) {
+ for (Cell kv : edit.getCells()) {
if (kv instanceof IndexedKeyValue) {
return (IndexedKeyValue) kv;
}
@@ -490,7 +490,7 @@ public class Indexer extends BaseRegionObserver {
*/
private Collection<Pair<Mutation, byte[]>> extractIndexUpdate(WALEdit edit) {
Collection<Pair<Mutation, byte[]>> indexUpdates = new ArrayList<Pair<Mutation, byte[]>>();
- for (KeyValue kv : edit.getKeyValues()) {
+ for (Cell kv : edit.getCells()) {
if (kv instanceof IndexedKeyValue) {
IndexedKeyValue ikv = (IndexedKeyValue) kv;
indexUpdates.add(new Pair<Mutation, byte[]>(ikv.getMutation(), ikv.getIndexTable()));
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/balancer/IndexLoadBalancer.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/balancer/IndexLoadBalancer.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/balancer/IndexLoadBalancer.java
index 296ff95..146028e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/balancer/IndexLoadBalancer.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/balancer/IndexLoadBalancer.java
@@ -124,6 +124,11 @@ public class IndexLoadBalancer implements LoadBalancer {
}
@Override
+ public void onConfigurationChange(Configuration conf) {
+ setConf(conf);
+ }
+
+ @Override
public void setClusterStatus(ClusterStatus st) {
this.clusterStatus = st;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/data/IndexMemStore.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/data/IndexMemStore.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/data/IndexMemStore.java
index 89489ec..7ae54ad 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/data/IndexMemStore.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/data/IndexMemStore.java
@@ -24,8 +24,10 @@ import java.util.SortedSet;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.KVComparator;
+import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.regionserver.IndexKeyValueSkipListSet;
import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
@@ -213,7 +215,7 @@ public class IndexMemStore implements KeyValueStore {
* @return false if the key is null or if there is no data
*/
@Override
- public synchronized boolean seek(KeyValue key) {
+ public synchronized boolean seek(Cell key) {
if (key == null) {
close();
return false;
@@ -221,16 +223,16 @@ public class IndexMemStore implements KeyValueStore {
// kvset and snapshot will never be null.
// if tailSet can't find anything, SortedSet is empty (not null).
- kvsetIt = kvsetAtCreation.tailSet(key).iterator();
+ kvsetIt = kvsetAtCreation.tailSet(KeyValueUtil.ensureKeyValue(key)).iterator();
kvsetItRow = null;
- return seekInSubLists(key);
+ return seekInSubLists();
}
/**
* (Re)initialize the iterators after a seek or a reseek.
*/
- private synchronized boolean seekInSubLists(KeyValue key) {
+ private synchronized boolean seekInSubLists() {
nextRow = getNext(kvsetIt);
return nextRow != null;
}
@@ -241,7 +243,7 @@ public class IndexMemStore implements KeyValueStore {
* @return true if there is at least one KV to read, false otherwise
*/
@Override
- public synchronized boolean reseek(KeyValue key) {
+ public synchronized boolean reseek(Cell key) {
/*
* See HBASE-4195 & HBASE-3855 & HBASE-6591 for the background on this implementation. This
* code is executed concurrently with flush and puts, without locks. Two points must be known
@@ -252,8 +254,9 @@ public class IndexMemStore implements KeyValueStore {
* we iterated to and restore the reseeked set to at least that point.
*/
- kvsetIt = kvsetAtCreation.tailSet(getHighest(key, kvsetItRow)).iterator();
- return seekInSubLists(key);
+ KeyValue kv = KeyValueUtil.ensureKeyValue(key);
+ kvsetIt = kvsetAtCreation.tailSet(getHighest(kv, kvsetItRow)).iterator();
+ return seekInSubLists();
}
/*
@@ -272,18 +275,18 @@ public class IndexMemStore implements KeyValueStore {
}
@Override
- public synchronized KeyValue peek() {
+ public synchronized Cell peek() {
// DebugPrint.println(" MS@" + hashCode() + " peek = " + getLowest());
return nextRow;
}
@Override
- public synchronized KeyValue next() {
+ public synchronized Cell next() {
if (nextRow == null) {
return null;
}
- final KeyValue ret = nextRow;
+ final Cell ret = nextRow;
// Advance the iterators
nextRow = getNext(kvsetIt);
@@ -314,7 +317,7 @@ public class IndexMemStore implements KeyValueStore {
}
@Override
- public boolean backwardSeek(KeyValue arg0) throws IOException {
+ public boolean backwardSeek(Cell arg0) throws IOException {
throw new UnsupportedOperationException();
}
@@ -324,7 +327,7 @@ public class IndexMemStore implements KeyValueStore {
}
@Override
- public boolean seekToPreviousRow(KeyValue arg0) throws IOException {
+ public boolean seekToPreviousRow(Cell arg0) throws IOException {
throw new UnsupportedOperationException();
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/data/LazyValueGetter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/data/LazyValueGetter.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/data/LazyValueGetter.java
index 21eb5cf..554b394 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/data/LazyValueGetter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/data/LazyValueGetter.java
@@ -22,6 +22,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
+import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.phoenix.hbase.index.ValueGetter;
import org.apache.phoenix.hbase.index.covered.update.ColumnReference;
@@ -78,9 +79,9 @@ public class LazyValueGetter implements ValueGetter {
return null;
}
// there is a next value - we only care about the current value, so we can just snag that
- KeyValue next = scan.next();
+ Cell next = scan.next();
if (ref.matches(next)) {
- return new ImmutableBytesPtr(next.getBuffer(), next.getValueOffset(), next.getValueLength());
+ return new ImmutableBytesPtr(next.getValueArray(), next.getValueOffset(), next.getValueLength());
}
return null;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/example/CoveredColumnIndexCodec.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/example/CoveredColumnIndexCodec.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/example/CoveredColumnIndexCodec.java
index 6750be2..658ce91 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/example/CoveredColumnIndexCodec.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/example/CoveredColumnIndexCodec.java
@@ -24,6 +24,7 @@ import java.util.List;
import java.util.Map.Entry;
import org.apache.commons.lang.ArrayUtils;
+import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Mutation;
@@ -169,7 +170,8 @@ public class CoveredColumnIndexCodec extends BaseIndexCodec {
/**
* Get the next batch of primary table values for the given columns
* @param refs columns to match against
- * @param state
+ * @param kvs
+ * @param currentRow
* @return the total length of all values found and the entries to add for the index
*/
@SuppressWarnings("deprecation")
@@ -187,7 +189,7 @@ private Pair<Integer, List<ColumnEntry>> getNextEntries(List<CoveredColumn> refs
continue;
}
// there is a next value - we only care about the current value, so we can just snag that
- KeyValue next = kvs.next();
+ Cell next = kvs.next();
if (ref.matchesFamily(next.getFamily()) && ref.matchesQualifier(next.getQualifier())) {
byte[] v = next.getValue();
totalValueLength += v.length;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/filter/ApplyAndFilterDeletesFilter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/filter/ApplyAndFilterDeletesFilter.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/filter/ApplyAndFilterDeletesFilter.java
index de21d56..03ff760 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/filter/ApplyAndFilterDeletesFilter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/filter/ApplyAndFilterDeletesFilter.java
@@ -205,8 +205,8 @@ public class ApplyAndFilterDeletesFilter extends FilterBase {
@SuppressWarnings("deprecation")
@Override
public KeyValue getHint(KeyValue kv) {
- return KeyValue.createLastOnRow(kv.getBuffer(), kv.getRowOffset(), kv.getRowLength(),
- kv.getBuffer(), kv.getFamilyOffset(), kv.getFamilyLength(), kv.getBuffer(),
+ return KeyValueUtil.createLastOnRow(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(),
+ kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(), kv.getQualifierArray(),
kv.getQualifierOffset(), kv.getQualifierLength());
}
}
@@ -259,7 +259,7 @@ public class ApplyAndFilterDeletesFilter extends FilterBase {
if (deleteColumn == null) {
return false;
}
- if (CellUtil.matchingFamily(deleteColumn, next) && deleteColumn.matchingQualifier(next)) {
+ if (CellUtil.matchingFamily(deleteColumn, next) && CellUtil.matchingQualifier(deleteColumn, next)) {
// falls within the timestamp range
if (deleteColumn.getTimestamp() >= next.getTimestamp()) {
return true;
@@ -280,7 +280,7 @@ public class ApplyAndFilterDeletesFilter extends FilterBase {
// keyvalue has the exact timestamp or is an older (smaller) timestamp, and we can allow that
// one.
if (pointDelete != null && CellUtil.matchingFamily(pointDelete, next)
- && pointDelete.matchingQualifier(next)) {
+ && CellUtil.matchingQualifier(pointDelete, next)) {
if (pointDelete.getTimestamp() == next.getTimestamp()) {
return true;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/update/ColumnReference.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/update/ColumnReference.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/update/ColumnReference.java
index 4ea7295..ddb5850 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/update/ColumnReference.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/update/ColumnReference.java
@@ -17,6 +17,7 @@
*/
package org.apache.phoenix.hbase.index.covered.update;
+import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.util.Bytes;
@@ -88,10 +89,9 @@ public class ColumnReference implements Comparable<ColumnReference> {
return this.qualifierPtr;
}
- @SuppressWarnings("deprecation")
- public boolean matches(KeyValue kv) {
- if (matchesFamily(kv.getRowArray(), kv.getFamilyOffset(), kv.getFamilyLength())) {
- return matchesQualifier(kv.getRowArray(), kv.getQualifierOffset(), kv.getQualifierLength());
+ public boolean matches(Cell kv) {
+ if (matchesFamily(kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength())) {
+ return matchesQualifier(kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength());
}
return false;
}
@@ -175,4 +175,4 @@ public class ColumnReference implements Comparable<ColumnReference> {
public String toString() {
return "ColumnReference - " + Bytes.toString(getFamily()) + ":" + Bytes.toString(getQualifier());
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/ipc/PhoenixIndexRpcSchedulerFactory.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/ipc/PhoenixIndexRpcSchedulerFactory.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/ipc/PhoenixIndexRpcSchedulerFactory.java
index 8e0b86f..1789b0e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/ipc/PhoenixIndexRpcSchedulerFactory.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/ipc/PhoenixIndexRpcSchedulerFactory.java
@@ -20,8 +20,10 @@ package org.apache.phoenix.hbase.index.ipc;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.ipc.PhoenixIndexRpcScheduler;
+import org.apache.hadoop.hbase.ipc.PriorityFunction;
import org.apache.hadoop.hbase.ipc.RpcScheduler;
import org.apache.hadoop.hbase.regionserver.RegionServerServices;
import org.apache.hadoop.hbase.regionserver.RpcSchedulerFactory;
@@ -43,24 +45,16 @@ public class PhoenixIndexRpcSchedulerFactory implements RpcSchedulerFactory {
"Running an older version of HBase (less than 0.98.4), Phoenix index RPC handling cannot be enabled.";
@Override
- public RpcScheduler create(Configuration conf, RegionServerServices services) {
+ public RpcScheduler create(Configuration conf, PriorityFunction priorityFunction, Abortable abortable) {
// create the delegate scheduler
RpcScheduler delegate;
try {
// happens in <=0.98.4 where the scheduler factory is not visible
- delegate = new SimpleRpcSchedulerFactory().create(conf, services);
+ delegate = new SimpleRpcSchedulerFactory().create(conf, priorityFunction, abortable);
} catch (IllegalAccessError e) {
LOG.fatal(VERSION_TOO_OLD_FOR_INDEX_RPC);
throw e;
}
- try {
- // make sure we are on a version that phoenix can support
- Class.forName("org.apache.hadoop.hbase.ipc.RpcExecutor");
- } catch (ClassNotFoundException e) {
- LOG.error(VERSION_TOO_OLD_FOR_INDEX_RPC
- + " Instead, using falling back to Simple RPC scheduling.");
- return delegate;
- }
int indexHandlerCount = conf.getInt(QueryServices.INDEX_HANDLER_COUNT_ATTRIB, QueryServicesOptions.DEFAULT_INDEX_HANDLER_COUNT);
int minPriority = getMinPriority(conf);
@@ -85,6 +79,11 @@ public class PhoenixIndexRpcSchedulerFactory implements RpcSchedulerFactory {
return scheduler;
}
+ @Override
+ public RpcScheduler create(Configuration configuration, PriorityFunction priorityFunction) {
+ return create(configuration, priorityFunction, null);
+ }
+
public static int getMinPriority(Configuration conf) {
return conf.getInt(QueryServices.MIN_INDEX_PRIOIRTY_ATTRIB, QueryServicesOptions.DEFAULT_INDEX_MIN_PRIORITY);
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/FilteredKeyValueScanner.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/FilteredKeyValueScanner.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/FilteredKeyValueScanner.java
index 1f16bef..bdf7126 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/FilteredKeyValueScanner.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/FilteredKeyValueScanner.java
@@ -21,6 +21,7 @@ package org.apache.phoenix.hbase.index.scanner;
import java.io.IOException;
import java.util.SortedSet;
+import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.client.Scan;
@@ -49,7 +50,7 @@ public class FilteredKeyValueScanner implements KeyValueScanner {
}
@Override
- public KeyValue peek() {
+ public Cell peek() {
return delegate.peek();
}
@@ -61,13 +62,13 @@ public class FilteredKeyValueScanner implements KeyValueScanner {
* filters.
*/
@Override
- public KeyValue next() throws IOException {
+ public Cell next() throws IOException {
seekToNextUnfilteredKeyValue();
return delegate.next();
}
@Override
- public boolean seek(KeyValue key) throws IOException {
+ public boolean seek(Cell key) throws IOException {
if (filter.filterAllRemaining()) { return false; }
// see if we can seek to the next key
if (!delegate.seek(key)) { return false; }
@@ -78,7 +79,7 @@ public class FilteredKeyValueScanner implements KeyValueScanner {
@SuppressWarnings("deprecation")
private boolean seekToNextUnfilteredKeyValue() throws IOException {
while (true) {
- KeyValue peeked = delegate.peek();
+ Cell peeked = delegate.peek();
// no more key values, so we are done
if (peeked == null) { return false; }
@@ -103,13 +104,13 @@ public class FilteredKeyValueScanner implements KeyValueScanner {
}
@Override
- public boolean reseek(KeyValue key) throws IOException {
+ public boolean reseek(Cell key) throws IOException {
this.delegate.reseek(key);
return this.seekToNextUnfilteredKeyValue();
}
@Override
- public boolean requestSeek(KeyValue kv, boolean forward, boolean useBloom) throws IOException {
+ public boolean requestSeek(Cell kv, boolean forward, boolean useBloom) throws IOException {
return this.reseek(kv);
}
@@ -145,7 +146,7 @@ public class FilteredKeyValueScanner implements KeyValueScanner {
}
@Override
- public boolean backwardSeek(KeyValue arg0) throws IOException {
+ public boolean backwardSeek(Cell arg0) throws IOException {
return this.delegate.backwardSeek(arg0);
}
@@ -155,7 +156,7 @@ public class FilteredKeyValueScanner implements KeyValueScanner {
}
@Override
- public boolean seekToPreviousRow(KeyValue arg0) throws IOException {
+ public boolean seekToPreviousRow(Cell arg0) throws IOException {
return this.delegate.seekToPreviousRow(arg0);
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/Scanner.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/Scanner.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/Scanner.java
index 868e892..43ddc45 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/Scanner.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/Scanner.java
@@ -21,6 +21,7 @@ package org.apache.phoenix.hbase.index.scanner;
import java.io.Closeable;
import java.io.IOException;
+import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.KeyValue;
/**
@@ -33,7 +34,7 @@ public interface Scanner extends Closeable {
* @return the next keyvalue in the scanner or <tt>null</tt> if there is no next {@link KeyValue}
* @throws IOException if there is an underlying error reading the data
*/
- public KeyValue next() throws IOException;
+ public Cell next() throws IOException;
/**
* Seek to immediately before the given {@link KeyValue}. If that exact {@link KeyValue} is
@@ -51,5 +52,5 @@ public interface Scanner extends Closeable {
* @return the next {@link KeyValue} or <tt>null</tt> if there are no more values in <tt>this</tt>
* @throws IOException if there is an error reading the underlying data.
*/
- public KeyValue peek() throws IOException;
+ public Cell peek() throws IOException;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/ScannerBuilder.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/ScannerBuilder.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/ScannerBuilder.java
index 575779a..ff33ec2 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/ScannerBuilder.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/ScannerBuilder.java
@@ -23,7 +23,9 @@ import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
+import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.filter.BinaryComparator;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
@@ -110,7 +112,7 @@ public class ScannerBuilder {
// create a scanner and wrap it as an iterator, meaning you can only go forward
final FilteredKeyValueScanner kvScanner = new FilteredKeyValueScanner(filters, memstore);
// seek the scanner to initialize it
- KeyValue start = KeyValue.createFirstOnRow(update.getRow());
+ KeyValue start = KeyValueUtil.createFirstOnRow(update.getRow());
try {
if (!kvScanner.seek(start)) {
return new EmptyScanner();
@@ -125,7 +127,7 @@ public class ScannerBuilder {
return new Scanner() {
@Override
- public KeyValue next() {
+ public Cell next() {
try {
return kvScanner.next();
} catch (IOException e) {
@@ -137,7 +139,7 @@ public class ScannerBuilder {
public boolean seek(KeyValue next) throws IOException {
// check to see if the next kv is after the current key, in which case we can use reseek,
// which will be more efficient
- KeyValue peek = kvScanner.peek();
+ Cell peek = kvScanner.peek();
// there is another value and its before the requested one - we can do a reseek!
if (peek != null) {
int compare = KeyValue.COMPARATOR.compare(peek, next);
@@ -152,7 +154,7 @@ public class ScannerBuilder {
}
@Override
- public KeyValue peek() throws IOException {
+ public Cell peek() throws IOException {
return kvScanner.peek();
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/wal/IndexedKeyValue.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/wal/IndexedKeyValue.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/wal/IndexedKeyValue.java
index 0270de5..b04cf0a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/wal/IndexedKeyValue.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/wal/IndexedKeyValue.java
@@ -116,23 +116,6 @@ public class IndexedKeyValue extends KeyValue {
return COLUMN_QUALIFIER.length;
}
- /**
- * This is a KeyValue that shouldn't actually be replayed/replicated, so we always mark it as
- * an {@link WALEdit#METAFAMILY} so it isn't replayed/replicated via the normal replay mechanism
- */
- @Override
- public boolean matchingFamily(final byte[] family) {
- return Bytes.equals(family, WALEdit.METAFAMILY);
- }
-
- /**
- * Not a real KeyValue
- */
- @Override
- public boolean matchingRow(final byte [] row) {
- return false;
- }
-
@Override
public String toString() {
return "IndexWrite:\n\ttable: " + indexTableName + "\n\tmutation:" + mutation;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
index 630c8f5..732dd8b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
@@ -92,8 +92,8 @@ import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.SQLCloseable;
import org.apache.phoenix.util.SQLCloseables;
-import org.cloudera.htrace.Sampler;
-import org.cloudera.htrace.TraceScope;
+import org.apache.htrace.Sampler;
+import org.apache.htrace.TraceScope;
import com.google.common.base.Objects;
import com.google.common.base.Strings;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/main/java/org/apache/phoenix/trace/TraceMetricSource.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/trace/TraceMetricSource.java b/phoenix-core/src/main/java/org/apache/phoenix/trace/TraceMetricSource.java
index 1b9e31a..e92dd6a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/trace/TraceMetricSource.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/trace/TraceMetricSource.java
@@ -22,11 +22,11 @@ import org.apache.hadoop.metrics2.*;
import org.apache.hadoop.metrics2.lib.Interns;
import org.apache.phoenix.metrics.MetricInfo;
import org.apache.phoenix.metrics.Metrics;
-import org.cloudera.htrace.HTraceConfiguration;
-import org.cloudera.htrace.Span;
-import org.cloudera.htrace.SpanReceiver;
-import org.cloudera.htrace.TimelineAnnotation;
-import org.cloudera.htrace.impl.MilliSpan;
+import org.apache.htrace.HTraceConfiguration;
+import org.apache.htrace.Span;
+import org.apache.htrace.SpanReceiver;
+import org.apache.htrace.TimelineAnnotation;
+import org.apache.htrace.impl.MilliSpan;
import java.io.IOException;
import java.util.ArrayList;
@@ -153,11 +153,6 @@ public class TraceMetricSource implements SpanReceiver, MetricsSource {
// noop
}
- @Override
- public void configure(HTraceConfiguration conf) {
- // noop
- }
-
private static class Metric {
List<Pair<MetricsInfo, Long>> counters = new ArrayList<Pair<MetricsInfo, Long>>();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/main/java/org/apache/phoenix/trace/TraceReader.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/trace/TraceReader.java b/phoenix-core/src/main/java/org/apache/phoenix/trace/TraceReader.java
index f3fc81d..ccb9064 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/trace/TraceReader.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/trace/TraceReader.java
@@ -34,7 +34,7 @@ import org.apache.phoenix.metrics.MetricInfo;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.util.LogUtil;
-import org.cloudera.htrace.Span;
+import org.apache.htrace.Span;
import com.google.common.base.Joiner;
import com.google.common.primitives.Longs;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/main/java/org/apache/phoenix/trace/TracingIterator.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/trace/TracingIterator.java b/phoenix-core/src/main/java/org/apache/phoenix/trace/TracingIterator.java
index bee5a1c..4808f25 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/trace/TracingIterator.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/trace/TracingIterator.java
@@ -22,7 +22,7 @@ import java.sql.SQLException;
import org.apache.phoenix.iterate.DelegateResultIterator;
import org.apache.phoenix.iterate.ResultIterator;
import org.apache.phoenix.schema.tuple.Tuple;
-import org.cloudera.htrace.TraceScope;
+import org.apache.htrace.TraceScope;
/**
* A simple iterator that closes the trace scope when the iterator is closed.
[07/16] phoenix git commit: PHOENIX-1676 Set priority of Index
Updates correctly
Posted by la...@apache.org.
PHOENIX-1676 Set priority of Index Updates correctly
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/8b0591ec
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/8b0591ec
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/8b0591ec
Branch: refs/heads/4.x-HBase-1.x
Commit: 8b0591ecd38ffa6a110f9fd5d9c8ce086d537e2c
Parents: 096586e
Author: Thomas <td...@salesforce.com>
Authored: Mon Mar 23 22:17:16 2015 -0700
Committer: Thomas <td...@salesforce.com>
Committed: Mon Mar 23 22:40:55 2015 -0700
----------------------------------------------------------------------
.../phoenix/end2end/index/IndexQosIT.java | 240 +++++++++++++++++++
.../hbase/ipc/PhoenixIndexRpcScheduler.java | 3 +
.../phoenix/hbase/index/IndexQosCompat.java | 98 --------
.../index/IndexQosRpcControllerFactory.java | 12 +-
.../index/table/CoprocessorHTableFactory.java | 20 --
.../phoenix/jdbc/PhoenixDatabaseMetaData.java | 7 +-
.../org/apache/phoenix/util/SchemaUtil.java | 7 +
7 files changed, 260 insertions(+), 127 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/8b0591ec/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexQosIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexQosIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexQosIT.java
new file mode 100644
index 0000000..7338b40
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexQosIT.java
@@ -0,0 +1,240 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE
+ * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the
+ * License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by
+ * applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+package org.apache.phoenix.end2end.index;
+
+import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL;
+import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR;
+import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_TERMINATOR;
+import static org.apache.phoenix.util.PhoenixRuntime.PHOENIX_TEST_DRIVER_URL_PARAM;
+import static org.apache.phoenix.util.TestUtil.LOCALHOST;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.ipc.BalancedQueueRpcExecutor;
+import org.apache.hadoop.hbase.ipc.CallRunner;
+import org.apache.hadoop.hbase.ipc.PhoenixIndexRpcScheduler;
+import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
+import org.apache.hadoop.hbase.ipc.RpcExecutor;
+import org.apache.hadoop.hbase.ipc.RpcScheduler;
+import org.apache.hadoop.hbase.master.AssignmentManager;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.regionserver.RegionServerServices;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
+import org.apache.phoenix.hbase.index.IndexQosRpcControllerFactory;
+import org.apache.phoenix.hbase.index.ipc.PhoenixIndexRpcSchedulerFactory;
+import org.apache.phoenix.jdbc.PhoenixTestDriver;
+import org.apache.phoenix.query.BaseTest;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.QueryUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.phoenix.util.SchemaUtil;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
+
+
+@Category(NeedsOwnMiniClusterTest.class)
+public class IndexQosIT extends BaseTest {
+
+ private static final String SCHEMA_NAME = "S";
+ private static final String INDEX_TABLE_NAME = "I";
+ private static final String DATA_TABLE_FULL_NAME = SchemaUtil.getTableName(SCHEMA_NAME, "T");
+ private static final String INDEX_TABLE_FULL_NAME = SchemaUtil.getTableName(SCHEMA_NAME, "I");
+ private static final int NUM_SLAVES = 2;
+
+ private static String url;
+ private static PhoenixTestDriver driver;
+ private HBaseTestingUtility util;
+ private HBaseAdmin admin;
+ private Configuration conf;
+ private static RpcExecutor spyRpcExecutor = Mockito.spy(new BalancedQueueRpcExecutor("test-queue", 30, 1, 300));
+
+ /**
+ * Factory that uses a spyed RpcExecutor
+ */
+ public static class TestPhoenixIndexRpcSchedulerFactory extends PhoenixIndexRpcSchedulerFactory {
+ @Override
+ public RpcScheduler create(Configuration conf, RegionServerServices services) {
+ PhoenixIndexRpcScheduler phoenixIndexRpcScheduler = (PhoenixIndexRpcScheduler)super.create(conf, services);
+ phoenixIndexRpcScheduler.setExecutorForTesting(spyRpcExecutor);
+ return phoenixIndexRpcScheduler;
+ }
+ }
+
+ @Before
+ public void doSetup() throws Exception {
+ conf = HBaseConfiguration.create();
+ setUpConfigForMiniCluster(conf);
+ conf.set(HRegionServer.REGION_SERVER_RPC_SCHEDULER_FACTORY_CLASS,
+ TestPhoenixIndexRpcSchedulerFactory.class.getName());
+ conf.set(RpcControllerFactory.CUSTOM_CONTROLLER_CONF_KEY, IndexQosRpcControllerFactory.class.getName());
+ util = new HBaseTestingUtility(conf);
+ // start cluster with 2 region servers
+ util.startMiniCluster(NUM_SLAVES);
+ admin = util.getHBaseAdmin();
+ String clientPort = util.getConfiguration().get(QueryServices.ZOOKEEPER_PORT_ATTRIB);
+ url = JDBC_PROTOCOL + JDBC_PROTOCOL_SEPARATOR + LOCALHOST + JDBC_PROTOCOL_SEPARATOR + clientPort
+ + JDBC_PROTOCOL_TERMINATOR + PHOENIX_TEST_DRIVER_URL_PARAM;
+ driver = initAndRegisterDriver(url, ReadOnlyProps.EMPTY_PROPS);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ try {
+ destroyDriver(driver);
+ if (admin!=null) {
+ admin.close();
+ }
+ } finally {
+ util.shutdownMiniCluster();
+ }
+ }
+
+ @Test
+ public void testIndexWriteQos() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = driver.connect(url, props);
+
+ // create the table
+ conn.createStatement().execute(
+ "CREATE TABLE " + DATA_TABLE_FULL_NAME + " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)");
+
+ // create the index
+ conn.createStatement().execute(
+ "CREATE INDEX " + INDEX_TABLE_NAME + " ON " + DATA_TABLE_FULL_NAME + " (v1) INCLUDE (v2)");
+
+ byte[] dataTableName = Bytes.toBytes(DATA_TABLE_FULL_NAME);
+ byte[] indexTableName = Bytes.toBytes(INDEX_TABLE_FULL_NAME);
+ MiniHBaseCluster cluster = util.getHBaseCluster();
+ HMaster master = cluster.getMaster();
+ AssignmentManager am = master.getAssignmentManager();
+
+ // verify there is only a single region for data table
+ List<HRegionInfo> tableRegions = admin.getTableRegions(dataTableName);
+ assertEquals("Expected single region for " + dataTableName, tableRegions.size(), 1);
+ HRegionInfo dataHri = tableRegions.get(0);
+
+ // verify there is only a single region for index table
+ tableRegions = admin.getTableRegions(indexTableName);
+ HRegionInfo indexHri = tableRegions.get(0);
+ assertEquals("Expected single region for " + indexTableName, tableRegions.size(), 1);
+
+ ServerName dataServerName = am.getRegionStates().getRegionServerOfRegion(dataHri);
+ ServerName indexServerName = am.getRegionStates().getRegionServerOfRegion(indexHri);
+
+ // if data table and index table are on same region server, move the index table to the other region server
+ if (dataServerName.equals(indexServerName)) {
+ HRegionServer server1 = util.getHBaseCluster().getRegionServer(0);
+ HRegionServer server2 = util.getHBaseCluster().getRegionServer(1);
+ HRegionServer dstServer = null;
+ HRegionServer srcServer = null;
+ if (server1.getServerName().equals(indexServerName)) {
+ dstServer = server2;
+ srcServer = server1;
+ } else {
+ dstServer = server1;
+ srcServer = server2;
+ }
+ byte[] encodedRegionNameInBytes = indexHri.getEncodedNameAsBytes();
+ admin.move(encodedRegionNameInBytes, Bytes.toBytes(dstServer.getServerName().getServerName()));
+ while (dstServer.getOnlineRegion(indexHri.getRegionName()) == null
+ || dstServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameInBytes)
+ || srcServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameInBytes)
+ || master.getAssignmentManager().getRegionStates().isRegionsInTransition()) {
+ // wait for the move to be finished
+ Thread.sleep(1);
+ }
+ }
+
+ dataHri = admin.getTableRegions(dataTableName).get(0);
+ dataServerName = am.getRegionStates().getRegionServerOfRegion(dataHri);
+ indexHri = admin.getTableRegions(indexTableName).get(0);
+ indexServerName = am.getRegionStates().getRegionServerOfRegion(indexHri);
+
+ // verify index and data tables are on different servers
+ assertNotEquals("Index and Data table should be on different region servers dataServer " + dataServerName
+ + " indexServer " + indexServerName, dataServerName, indexServerName);
+
+ PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + DATA_TABLE_FULL_NAME + " VALUES(?,?,?)");
+ stmt.setString(1, "k1");
+ stmt.setString(2, "v1");
+ stmt.setString(3, "v2");
+ stmt.execute();
+ conn.commit();
+
+ // run select query that should use the index
+ String selectSql = "SELECT k, v2 from " + DATA_TABLE_FULL_NAME + " WHERE v1=?";
+ stmt = conn.prepareStatement(selectSql);
+ stmt.setString(1, "v1");
+
+ // verify that the query does a range scan on the index table
+ ResultSet rs = stmt.executeQuery("EXPLAIN " + selectSql);
+ assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER S.I ['v1']", QueryUtil.getExplainPlan(rs));
+
+ // verify that the correct results are returned
+ rs = stmt.executeQuery();
+ assertTrue(rs.next());
+ assertEquals("k1", rs.getString(1));
+ assertEquals("v2", rs.getString(2));
+ assertFalse(rs.next());
+
+ // drop index table
+ conn.createStatement().execute(
+ "DROP INDEX " + INDEX_TABLE_NAME + " ON " + DATA_TABLE_FULL_NAME );
+ // create a data table with the same name as the index table
+ conn.createStatement().execute(
+ "CREATE TABLE " + INDEX_TABLE_FULL_NAME + " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)");
+
+ // upsert one row to the table (which has the same table name as the previous index table)
+ stmt = conn.prepareStatement("UPSERT INTO " + INDEX_TABLE_FULL_NAME + " VALUES(?,?,?)");
+ stmt.setString(1, "k1");
+ stmt.setString(2, "v1");
+ stmt.setString(3, "v2");
+ stmt.execute();
+ conn.commit();
+
+ // run select query on the new table
+ selectSql = "SELECT k, v2 from " + INDEX_TABLE_FULL_NAME + " WHERE v1=?";
+ stmt = conn.prepareStatement(selectSql);
+ stmt.setString(1, "v1");
+
+ // verify that the correct results are returned
+ rs = stmt.executeQuery();
+ assertTrue(rs.next());
+ assertEquals("k1", rs.getString(1));
+ assertEquals("v2", rs.getString(2));
+ assertFalse(rs.next());
+
+ // verify that that index queue is used only once (for the first upsert)
+ Mockito.verify(spyRpcExecutor).dispatch(Mockito.any(CallRunner.class));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/8b0591ec/phoenix-core/src/main/java/org/apache/hadoop/hbase/ipc/PhoenixIndexRpcScheduler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/hadoop/hbase/ipc/PhoenixIndexRpcScheduler.java b/phoenix-core/src/main/java/org/apache/hadoop/hbase/ipc/PhoenixIndexRpcScheduler.java
index 98002a7..4709304 100644
--- a/phoenix-core/src/main/java/org/apache/hadoop/hbase/ipc/PhoenixIndexRpcScheduler.java
+++ b/phoenix-core/src/main/java/org/apache/hadoop/hbase/ipc/PhoenixIndexRpcScheduler.java
@@ -43,6 +43,7 @@ public class PhoenixIndexRpcScheduler extends RpcScheduler {
private int minPriority;
private int maxPriority;
private RpcExecutor callExecutor;
+ private int port;
public PhoenixIndexRpcScheduler(int indexHandlerCount, Configuration conf,
RpcScheduler delegate, int minPriority, int maxPriority) {
@@ -67,11 +68,13 @@ public class PhoenixIndexRpcScheduler extends RpcScheduler {
@Override
public void init(Context context) {
delegate.init(context);
+ this.port = context.getListenerAddress().getPort();
}
@Override
public void start() {
delegate.start();
+ callExecutor.start(port);
}
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/8b0591ec/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/IndexQosCompat.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/IndexQosCompat.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/IndexQosCompat.java
deleted file mode 100644
index 5681d71..0000000
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/IndexQosCompat.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.phoenix.hbase.index;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
-import org.apache.phoenix.hbase.index.ipc.PhoenixIndexRpcSchedulerFactory;
-
-/**
- * Helper class to avoid loading HBase 0.98.3+ classes in older HBase installations
- */
-public class IndexQosCompat {
-
- private static final Log LOG = LogFactory.getLog(IndexQosCompat.class);
-
- /**
- * Full class name of the RpcControllerFactory. This is copied here so we don't need the static reference, so we can work with older versions of HBase 0.98, which don't have this class
- */
- private static final String HBASE_RPC_CONTROLLER_CLASS_NAME =
- "org.apache.hadoop.hbase.ipc.RpcControllerFactory";
- private static volatile boolean checked = false;
- private static boolean rpcControllerExists = false;
-
- private IndexQosCompat() {
- // private ctor for util class
- }
-
- /**
- * @param tableName name of the index table
- * @return configuration key for if a table should have Index QOS writes (its a target index
- * table)
- */
- public static String getTableIndexQosConfKey(String tableName) {
- return "phoenix.index.table.qos._" + tableName;
- }
-
- /**
- * Set the index rpc controller, if the rpc controller exists. No-op if there the RpcController
- * is not on the classpath.
- * @param conf to update
- */
- public static void setPhoenixIndexRpcController(Configuration conf) {
- if (rpcControllerExists()) {
- // then we can load the class just fine
- conf.set(RpcControllerFactory.CUSTOM_CONTROLLER_CONF_KEY,
- PhoenixIndexRpcSchedulerFactory.class.getName());
- }
- }
-
- private static boolean rpcControllerExists() {
- if (checked) {
- synchronized (IndexQosCompat.class) {
- if (!checked) {
- // try loading the class
- try {
- Class.forName(HBASE_RPC_CONTROLLER_CLASS_NAME);
- rpcControllerExists = true;
- } catch (ClassNotFoundException e) {
- LOG.warn("RpcControllerFactory doesn't exist, not setting custom index handler properties.");
- rpcControllerExists = false;
- }
-
- checked = true;
- }
- }
- }
- return rpcControllerExists;
- }
-
- /**
- * Ensure that the given table is enabled for index QOS handling
- * @param conf configuration to read/update
- * @param tableName name of the table to configure for index handlers
- */
- public static void enableIndexQosForTable(Configuration conf, String tableName) {
- String confKey = IndexQosCompat.getTableIndexQosConfKey(tableName);
- if (conf.get(confKey) == null) {
- conf.setBoolean(confKey, true);
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/8b0591ec/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/IndexQosRpcControllerFactory.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/IndexQosRpcControllerFactory.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/IndexQosRpcControllerFactory.java
index aa8b8d1..a192feb 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/IndexQosRpcControllerFactory.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/IndexQosRpcControllerFactory.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hbase.ipc.DelegatingPayloadCarryingRpcController;
import org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController;
import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
import org.apache.phoenix.hbase.index.ipc.PhoenixIndexRpcSchedulerFactory;
+import org.apache.phoenix.util.SchemaUtil;
/**
* {@link RpcControllerFactory} that overrides the standard {@link PayloadCarryingRpcController} to
@@ -60,27 +61,22 @@ public class IndexQosRpcControllerFactory extends RpcControllerFactory {
private class IndexQosRpcController extends DelegatingPayloadCarryingRpcController {
- private Configuration conf;
private int priority;
public IndexQosRpcController(PayloadCarryingRpcController delegate, Configuration conf) {
super(delegate);
- this.conf = conf;
this.priority = PhoenixIndexRpcSchedulerFactory.getMinPriority(conf);
}
-
@Override
public void setPriority(final TableName tn) {
// if its an index table, then we override to the index priority
- if (isIndexTable(tn)) {
+ if (!tn.isSystemTable() && !SchemaUtil.isSystemDataTable(tn.getNameAsString())) {
setPriority(this.priority);
- } else {
+ }
+ else {
super.setPriority(tn);
}
}
- private boolean isIndexTable(TableName tn) {
- return conf.get(IndexQosCompat.getTableIndexQosConfKey(tn.getNameAsString())) == null;
- }
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/8b0591ec/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CoprocessorHTableFactory.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CoprocessorHTableFactory.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CoprocessorHTableFactory.java
index 72a28be..ded618d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CoprocessorHTableFactory.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CoprocessorHTableFactory.java
@@ -19,21 +19,13 @@ package org.apache.phoenix.hbase.index.table;
import java.io.IOException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.CoprocessorEnvironment;
-import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.phoenix.hbase.index.IndexQosCompat;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
-import org.apache.phoenix.hbase.index.util.IndexManagementUtil;
public class CoprocessorHTableFactory implements HTableFactory {
- private static final Log LOG = LogFactory.getLog(CoprocessorHTableFactory.class);
private CoprocessorEnvironment e;
public CoprocessorHTableFactory(CoprocessorEnvironment e) {
@@ -42,18 +34,6 @@ public class CoprocessorHTableFactory implements HTableFactory {
@Override
public HTableInterface getTable(ImmutableBytesPtr tablename) throws IOException {
- Configuration conf = e.getConfiguration();
-
- // make sure we use the index priority writer for our rpcs
- IndexQosCompat.setPhoenixIndexRpcController(conf);
-
- // make sure we include the index table in the tables we need to track
- String tableName = Bytes.toString(tablename.copyBytesIfNecessary());
- IndexQosCompat.enableIndexQosForTable(conf, tableName);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Creating new HTable: " + tableName);
- }
return this.e.getTable(TableName.valueOf(tablename.copyBytesIfNecessary()));
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/8b0591ec/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
index 154fef7..15bcfd0 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
@@ -67,6 +67,7 @@ import org.apache.phoenix.util.KeyValueUtil;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.StringUtil;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
@@ -277,7 +278,11 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData, org.apache.pho
public static final int MUTABLE_SI_VERSION_THRESHOLD = VersionUtil.encodeVersion("0", "94", "10");
/** Version below which we fall back on the generic KeyValueBuilder */
public static final int CLIENT_KEY_VALUE_BUILDER_THRESHOLD = VersionUtil.encodeVersion("0", "94", "14");
-
+
+ // list of system tables
+ public static final List<String> SYSTEM_TABLE_NAMES = new ImmutableList.Builder<String>().add(SYSTEM_CATALOG_NAME)
+ .add(SYSTEM_STATS_NAME).add(SEQUENCE_FULLNAME).build();
+
PhoenixDatabaseMetaData(PhoenixConnection connection) throws SQLException {
this.emptyResultSet = new PhoenixResultSet(ResultIterator.EMPTY_ITERATOR, RowProjector.EMPTY_PROJECTOR, new PhoenixStatement(connection));
this.connection = connection;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/8b0591ec/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
index 2a1d3ff..4a8341d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
@@ -403,6 +403,13 @@ public class SchemaUtil {
if (QueryConstants.SYSTEM_SCHEMA_NAME.equals(schemaName)) return true;
return false;
}
+
+ /**
+ * Returns true if the given table is a system table (does not include future system indexes)
+ */
+ public static boolean isSystemDataTable(String fullTableName) {
+ return PhoenixDatabaseMetaData.SYSTEM_TABLE_NAMES.contains(fullTableName);
+ }
// Given the splits and the rowKeySchema, find out the keys that
public static byte[][] processSplits(byte[][] splits, LinkedHashSet<PColumn> pkColumns, Integer saltBucketNum, boolean defaultRowKeyOrder) throws SQLException {
[05/16] phoenix git commit: PHOENIX-1753 Query with RVC that doesn't
lead with the row key can return incorrect results
Posted by la...@apache.org.
PHOENIX-1753 Query with RVC that doesn't lead with the row key can return incorrect results
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/e06ceaf4
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/e06ceaf4
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/e06ceaf4
Branch: refs/heads/4.x-HBase-1.x
Commit: e06ceaf455642d92b500d0e4edc343a3342a0d93
Parents: 4bc162d
Author: James Taylor <jt...@salesforce.com>
Authored: Thu Mar 19 18:54:57 2015 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Sat Mar 21 11:13:24 2015 -0700
----------------------------------------------------------------------
.../phoenix/end2end/RowValueConstructorIT.java | 33 +++++++++++++++++++-
.../apache/phoenix/compile/WhereOptimizer.java | 4 +++
.../phoenix/compile/WhereOptimizerTest.java | 16 ++++++++++
3 files changed, 52 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/e06ceaf4/phoenix-core/src/it/java/org/apache/phoenix/end2end/RowValueConstructorIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/RowValueConstructorIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/RowValueConstructorIT.java
index 8d67fa4..3859785 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/RowValueConstructorIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/RowValueConstructorIT.java
@@ -1362,6 +1362,37 @@ public class RowValueConstructorIT extends BaseClientManagedTimeIT {
conn.close();
}
-
+ @Test
+ public void testRVCWithRowKeyNotLeading() throws Exception {
+ String ddl = "CREATE TABLE sorttest4 (rownum BIGINT primary key, name varchar(16), age integer)";
+ Connection conn = nextConnection(getUrl());
+ conn.createStatement().execute(ddl);
+ conn.close();
+ conn = nextConnection(getUrl());
+ String dml = "UPSERT INTO sorttest4 (rownum, name, age) values (?, ?, ?)";
+ PreparedStatement stmt = conn.prepareStatement(dml);
+ stmt.setInt(1, 1);
+ stmt.setString(2, "A");
+ stmt.setInt(3, 1);
+ stmt.executeUpdate();
+ stmt.setInt(1, 2);
+ stmt.setString(2, "B");
+ stmt.setInt(3, 2);
+ stmt.executeUpdate();
+ conn.commit();
+ conn.close();
+ // the below query should only return one record -> (1, "A", 1)
+ String query = "SELECT rownum, name, age FROM sorttest4 where (age, rownum) < (2, 2)";
+ conn = nextConnection(getUrl());
+ ResultSet rs = conn.createStatement().executeQuery(query);
+ int numRecords = 0;
+ while (rs.next()) {
+ assertEquals(1, rs.getInt(1));
+ assertEquals("A", rs.getString(2));
+ assertEquals(1, rs.getInt(3));
+ numRecords++;
+ }
+ assertEquals(1, numRecords);
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/e06ceaf4/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereOptimizer.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereOptimizer.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereOptimizer.java
index 713076e..b03793d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereOptimizer.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereOptimizer.java
@@ -542,6 +542,10 @@ public class WhereOptimizer {
int span = position - initialPosition;
return new SingleKeySlot(new RowValueConstructorKeyPart(table.getPKColumns().get(initialPosition), rvc, span, childSlots), initialPosition, span, EVERYTHING_RANGES);
}
+ // If we don't clear the child list, we end up passing some of
+ // the child expressions of previous matches up the tree, causing
+ // those expressions to form the scan start/stop key. PHOENIX-1753
+ childSlots.clear();
return null;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/e06ceaf4/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereOptimizerTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereOptimizerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereOptimizerTest.java
index 0ec6b45..94b25d0 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereOptimizerTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereOptimizerTest.java
@@ -1275,6 +1275,22 @@ public class WhereOptimizerTest extends BaseConnectionlessQueryTest {
}
@Test
+ public void testRVCExpressionWithNonFirstLeadingColOfRowKey() throws SQLException {
+ String old_value = "value";
+ String orgId = getOrganizationId();
+
+ String query = "select * from entity_history where (old_value, organization_id) >= (?,?)";
+ List<Object> binds = Arrays.<Object>asList(old_value, orgId);
+ StatementContext context = compileStatement(query, binds);
+ Scan scan = context.getScan();
+ Filter filter = scan.getFilter();
+ assertNotNull(filter);
+ assertTrue(filter instanceof SingleKeyValueComparisonFilter);
+ assertArrayEquals(HConstants.EMPTY_START_ROW, scan.getStartRow());
+ assertArrayEquals(HConstants.EMPTY_END_ROW, scan.getStopRow());
+ }
+
+ @Test
public void testMultiRVCExpressionsCombinedWithAnd() throws SQLException {
String lowerTenantId = "000000000000001";
String lowerParentId = "000000000000002";
[12/16] phoenix git commit: PHOENIX-1756 Add Month() and Second()
buildin functions(Alicia Ying Shu)
Posted by la...@apache.org.
PHOENIX-1756 Add Month() and Second() buildin functions(Alicia Ying Shu)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/6cb6a376
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/6cb6a376
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/6cb6a376
Branch: refs/heads/4.x-HBase-1.x
Commit: 6cb6a3766ca3b37ea6d410979d97d6daf7bdd10a
Parents: a8b27e3
Author: Rajeshbabu Chintaguntla <ra...@apache.org>
Authored: Thu Mar 26 00:40:50 2015 +0530
Committer: Rajeshbabu Chintaguntla <ra...@apache.org>
Committed: Thu Mar 26 00:40:50 2015 +0530
----------------------------------------------------------------------
.../end2end/YearMonthSecondFunctionIT.java | 48 ++++++++++-
.../phoenix/expression/ExpressionType.java | 8 +-
.../expression/function/MonthFunction.java | 83 ++++++++++++++++++++
.../expression/function/SecondFunction.java | 81 +++++++++++++++++++
4 files changed, 217 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/6cb6a376/phoenix-core/src/it/java/org/apache/phoenix/end2end/YearMonthSecondFunctionIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/YearMonthSecondFunctionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/YearMonthSecondFunctionIT.java
index d19314a..da745fe 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/YearMonthSecondFunctionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/YearMonthSecondFunctionIT.java
@@ -107,7 +107,7 @@ public class YearMonthSecondFunctionIT extends BaseHBaseManagedTimeIT {
"unsignedDates UNSIGNED_DATE, unsignedTimestamps UNSIGNED_TIMESTAMP, unsignedTimes UNSIGNED_TIME CONSTRAINT pk PRIMARY KEY (k1))";
conn.createStatement().execute(ddl);
String dml = "UPSERT INTO T1 VALUES (1, TO_DATE('2004-03-01 00:00:00'), TO_TIMESTAMP('2006-02-01 00:00:00'), TO_TIME('2008-02-01 00:00:00'), " +
- "TO_DATE('2010-03-01 00:00:00'), TO_TIMESTAMP('2012-02-01'), TO_TIME('2015-02-01 00:00:00'))";
+ "TO_DATE('2010-03-01 00:00:00:896', 'yyyy-MM-dd HH:mm:ss:SSS'), TO_TIMESTAMP('2012-02-01'), TO_TIME('2015-02-01 00:00:00'))";
conn.createStatement().execute(dml);
conn.commit();
@@ -122,4 +122,50 @@ public class YearMonthSecondFunctionIT extends BaseHBaseManagedTimeIT {
assertEquals(2015, rs.getInt(6));
assertFalse(rs.next());
}
+
+ @Test
+ public void testMonthFuncAgainstColumns() throws Exception {
+ String ddl =
+ "CREATE TABLE IF NOT EXISTS T1 (k1 INTEGER NOT NULL, dates DATE, timestamps TIMESTAMP, times TIME, " +
+ "unsignedDates UNSIGNED_DATE, unsignedTimestamps UNSIGNED_TIMESTAMP, unsignedTimes UNSIGNED_TIME CONSTRAINT pk PRIMARY KEY (k1))";
+ conn.createStatement().execute(ddl);
+ String dml = "UPSERT INTO T1 VALUES (1, TO_DATE('2004-03-10 00:00:00'), TO_TIMESTAMP('2006-04-12 00:00:00'), TO_TIME('2008-05-16 00:00:00'), " +
+ "TO_DATE('2010-06-20 00:00:00:789', 'yyyy-MM-dd HH:mm:ss:SSS'), TO_TIMESTAMP('2012-07-28'), TO_TIME('2015-12-25 00:00:00'))";
+ conn.createStatement().execute(dml);
+ conn.commit();
+
+ ResultSet rs = conn.createStatement().executeQuery("SELECT k1, MONTH(timestamps), MONTH(times), MONTH(unsignedDates), MONTH(unsignedTimestamps), " +
+ "MONTH(unsignedTimes) FROM T1 where MONTH(dates) = 3");
+ assertTrue(rs.next());
+ assertEquals(1, rs.getInt(1));
+ assertEquals(4, rs.getInt(2));
+ assertEquals(5, rs.getInt(3));
+ assertEquals(6, rs.getInt(4));
+ assertEquals(7, rs.getInt(5));
+ assertEquals(12, rs.getInt(6));
+ assertFalse(rs.next());
+ }
+
+ @Test
+ public void testSecondFuncAgainstColumns() throws Exception {
+ String ddl =
+ "CREATE TABLE IF NOT EXISTS T1 (k1 INTEGER NOT NULL, dates DATE, timestamps TIMESTAMP, times TIME, " +
+ "unsignedDates UNSIGNED_DATE, unsignedTimestamps UNSIGNED_TIMESTAMP, unsignedTimes UNSIGNED_TIME CONSTRAINT pk PRIMARY KEY (k1))";
+ conn.createStatement().execute(ddl);
+ String dml = "UPSERT INTO T1 VALUES (1, TO_DATE('2004-03-01 00:00:10'), TO_TIMESTAMP('2006-04-12 00:00:20'), TO_TIME('2008-05-16 10:00:30'), " +
+ "TO_DATE('2010-06-20 00:00:40:789', 'yyyy-MM-dd HH:mm:ss:SSS'), TO_TIMESTAMP('2012-07-28'), TO_TIME('2015-12-25 00:00:50'))";
+ conn.createStatement().execute(dml);
+ conn.commit();
+
+ ResultSet rs = conn.createStatement().executeQuery("SELECT k1, SECOND(dates), SECOND(times), SECOND(unsignedDates), SECOND(unsignedTimestamps), " +
+ "SECOND(unsignedTimes) FROM T1 where SECOND(timestamps)=20");
+ assertTrue(rs.next());
+ assertEquals(1, rs.getInt(1));
+ assertEquals(10, rs.getInt(2));
+ assertEquals(30, rs.getInt(3));
+ assertEquals(40, rs.getInt(4));
+ assertEquals(0, rs.getInt(5));
+ assertEquals(50, rs.getInt(6));
+ assertFalse(rs.next());
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/6cb6a376/phoenix-core/src/main/java/org/apache/phoenix/expression/ExpressionType.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/ExpressionType.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/ExpressionType.java
index c871bc5..3f4fea7 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/ExpressionType.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/ExpressionType.java
@@ -50,6 +50,7 @@ import org.apache.phoenix.expression.function.LpadFunction;
import org.apache.phoenix.expression.function.MD5Function;
import org.apache.phoenix.expression.function.MaxAggregateFunction;
import org.apache.phoenix.expression.function.MinAggregateFunction;
+import org.apache.phoenix.expression.function.MonthFunction;
import org.apache.phoenix.expression.function.NthValueFunction;
import org.apache.phoenix.expression.function.PercentRankAggregateFunction;
import org.apache.phoenix.expression.function.PercentileContAggregateFunction;
@@ -67,6 +68,7 @@ import org.apache.phoenix.expression.function.RoundTimestampExpression;
import org.apache.phoenix.expression.function.SQLIndexTypeFunction;
import org.apache.phoenix.expression.function.SQLTableTypeFunction;
import org.apache.phoenix.expression.function.SQLViewTypeFunction;
+import org.apache.phoenix.expression.function.SecondFunction;
import org.apache.phoenix.expression.function.SignFunction;
import org.apache.phoenix.expression.function.SqlTypeNameFunction;
import org.apache.phoenix.expression.function.StddevPopFunction;
@@ -194,9 +196,11 @@ public enum ExpressionType {
ToTimeFunction(ToTimeFunction.class),
ToTimestampFunction(ToTimestampFunction.class),
SignFunction(SignFunction.class),
- YearFunction(YearFunction.class)
+ YearFunction(YearFunction.class),
+ MonthFunction(MonthFunction.class),
+ SecondFunction(SecondFunction.class)
;
-
+
ExpressionType(Class<? extends Expression> clazz) {
this.clazz = clazz;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/6cb6a376/phoenix-core/src/main/java/org/apache/phoenix/expression/function/MonthFunction.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/MonthFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/MonthFunction.java
new file mode 100644
index 0000000..5ad6c34
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/MonthFunction.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.expression.function;
+
+import java.sql.SQLException;
+import java.util.List;
+
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.phoenix.expression.Expression;
+import org.apache.phoenix.parse.FunctionParseNode.Argument;
+import org.apache.phoenix.parse.FunctionParseNode.BuiltInFunction;
+import org.apache.phoenix.schema.tuple.Tuple;
+import org.apache.phoenix.schema.types.PDataType;
+import org.apache.phoenix.schema.types.PInteger;
+import org.apache.phoenix.schema.types.PTimestamp;
+import org.joda.time.DateTime;
+
+/**
+ *
+ * Implementation of the Month() buildin. Input Date/Timestamp/Time.
+ * Returns an integer from 1 to 12 representing the month omponent of date
+ *
+ */
+@BuiltInFunction(name=MonthFunction.NAME,
+args={@Argument(allowedTypes={PTimestamp.class})})
+public class MonthFunction extends ScalarFunction {
+ public static final String NAME = "MONTH";
+
+ public MonthFunction() {
+ }
+
+ public MonthFunction(List<Expression> children) throws SQLException {
+ super(children);
+ }
+
+ @Override
+ public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) {
+ Expression expression = getChildExpression();
+ if (!expression.evaluate(tuple, ptr)) {
+ return false;
+ }
+ if ( ptr.getLength() == 0) {
+ return true; //means null
+ }
+ long dateTime = expression.getDataType().getCodec().decodeLong(ptr, expression.getSortOrder());
+ DateTime dt = new DateTime(dateTime);
+ int month = dt.getMonthOfYear();
+ PDataType returnType = getDataType();
+ byte[] byteValue = new byte[returnType.getByteSize()];
+ returnType.getCodec().encodeInt(month, byteValue, 0);
+ ptr.set(byteValue);
+ return true;
+ }
+
+ @Override
+ public PDataType getDataType() {
+ return PInteger.INSTANCE;
+ }
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ private Expression getChildExpression() {
+ return children.get(0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/6cb6a376/phoenix-core/src/main/java/org/apache/phoenix/expression/function/SecondFunction.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/SecondFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/SecondFunction.java
new file mode 100644
index 0000000..5f39786
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/SecondFunction.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.expression.function;
+
+import java.sql.SQLException;
+import java.util.List;
+
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.phoenix.expression.Expression;
+import org.apache.phoenix.parse.FunctionParseNode.Argument;
+import org.apache.phoenix.parse.FunctionParseNode.BuiltInFunction;
+import org.apache.phoenix.schema.tuple.Tuple;
+import org.apache.phoenix.schema.types.PDataType;
+import org.apache.phoenix.schema.types.PInteger;
+import org.apache.phoenix.schema.types.PTimestamp;
+
+/**
+ *
+ * Implementation of the Second() buildin. Input Date/Timestamp/Time.
+ * Returns an integer from 0 to 59 representing the second component of time
+ *
+ */
+@BuiltInFunction(name=SecondFunction.NAME,
+args={@Argument(allowedTypes={PTimestamp.class})})
+public class SecondFunction extends ScalarFunction {
+ public static final String NAME = "SECOND";
+
+ public SecondFunction() {
+ }
+
+ public SecondFunction(List<Expression> children) throws SQLException {
+ super(children);
+ }
+
+ @Override
+ public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) {
+ Expression expression = getChildExpression();
+ if (!expression.evaluate(tuple, ptr)) {
+ return false;
+ }
+ if ( ptr.getLength() == 0) {
+ return true; //means null
+ }
+ long dateTime = expression.getDataType().getCodec().decodeLong(ptr, expression.getSortOrder());
+ int sec = (int)((dateTime/1000) % 60);
+ PDataType returnType = getDataType();
+ byte[] byteValue = new byte[returnType.getByteSize()];
+ returnType.getCodec().encodeInt(sec, byteValue, 0);
+ ptr.set(byteValue);
+ return true;
+ }
+
+ @Override
+ public PDataType getDataType() {
+ return PInteger.INSTANCE;
+ }
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ private Expression getChildExpression() {
+ return children.get(0);
+ }
+}
[08/16] phoenix git commit: PHOENIX-1676 Set priority of Index
Updates correctly, fix IndexQosIT
Posted by la...@apache.org.
PHOENIX-1676 Set priority of Index Updates correctly, fix IndexQosIT
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/f4180fa4
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/f4180fa4
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/f4180fa4
Branch: refs/heads/4.x-HBase-1.x
Commit: f4180fa40e26c685bfbf1b59cf4385f9b0e713e9
Parents: 8b0591e
Author: Thomas <td...@salesforce.com>
Authored: Mon Mar 23 22:51:53 2015 -0700
Committer: Thomas <td...@salesforce.com>
Committed: Mon Mar 23 22:54:56 2015 -0700
----------------------------------------------------------------------
.../java/org/apache/phoenix/end2end/index/IndexQosIT.java | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/f4180fa4/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexQosIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexQosIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexQosIT.java
index 7338b40..9558bcb 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexQosIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexQosIT.java
@@ -27,6 +27,7 @@ import java.util.List;
import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HRegionInfo;
@@ -36,13 +37,14 @@ import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.ipc.BalancedQueueRpcExecutor;
import org.apache.hadoop.hbase.ipc.CallRunner;
import org.apache.hadoop.hbase.ipc.PhoenixIndexRpcScheduler;
+import org.apache.hadoop.hbase.ipc.PriorityFunction;
import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
import org.apache.hadoop.hbase.ipc.RpcExecutor;
import org.apache.hadoop.hbase.ipc.RpcScheduler;
import org.apache.hadoop.hbase.master.AssignmentManager;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
-import org.apache.hadoop.hbase.regionserver.RegionServerServices;
+import org.apache.hadoop.hbase.regionserver.RSRpcServices;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
import org.apache.phoenix.hbase.index.IndexQosRpcControllerFactory;
@@ -82,8 +84,8 @@ public class IndexQosIT extends BaseTest {
*/
public static class TestPhoenixIndexRpcSchedulerFactory extends PhoenixIndexRpcSchedulerFactory {
@Override
- public RpcScheduler create(Configuration conf, RegionServerServices services) {
- PhoenixIndexRpcScheduler phoenixIndexRpcScheduler = (PhoenixIndexRpcScheduler)super.create(conf, services);
+ public RpcScheduler create(Configuration conf, PriorityFunction priorityFunction, Abortable abortable) {
+ PhoenixIndexRpcScheduler phoenixIndexRpcScheduler = (PhoenixIndexRpcScheduler)super.create(conf, priorityFunction, abortable);
phoenixIndexRpcScheduler.setExecutorForTesting(spyRpcExecutor);
return phoenixIndexRpcScheduler;
}
@@ -93,7 +95,7 @@ public class IndexQosIT extends BaseTest {
public void doSetup() throws Exception {
conf = HBaseConfiguration.create();
setUpConfigForMiniCluster(conf);
- conf.set(HRegionServer.REGION_SERVER_RPC_SCHEDULER_FACTORY_CLASS,
+ conf.set(RSRpcServices.REGION_SERVER_RPC_SCHEDULER_FACTORY_CLASS,
TestPhoenixIndexRpcSchedulerFactory.class.getName());
conf.set(RpcControllerFactory.CUSTOM_CONTROLLER_CONF_KEY, IndexQosRpcControllerFactory.class.getName());
util = new HBaseTestingUtility(conf);
[16/16] phoenix git commit: PHOENIX-258 minor test fixes.
Posted by la...@apache.org.
PHOENIX-258 minor test fixes.
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/7dcf95a4
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/7dcf95a4
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/7dcf95a4
Branch: refs/heads/master
Commit: 7dcf95a40063a25917a68c56c68fe61a11a4ef8b
Parents: eb275a5
Author: Lars Hofhansl <la...@apache.org>
Authored: Thu Jun 2 13:20:48 2016 -0700
Committer: Lars Hofhansl <la...@apache.org>
Committed: Thu Jun 2 13:22:07 2016 -0700
----------------------------------------------------------------------
.../it/java/org/apache/phoenix/end2end/SortMergeJoinMoreIT.java | 4 ++++
.../java/org/apache/phoenix/end2end/index/IndexExpressionIT.java | 2 +-
2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/7dcf95a4/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortMergeJoinMoreIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortMergeJoinMoreIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortMergeJoinMoreIT.java
index c36b24d..8ede6ae 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortMergeJoinMoreIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortMergeJoinMoreIT.java
@@ -444,12 +444,14 @@ public class SortMergeJoinMoreIT extends BaseHBaseManagedTimeIT {
"SORT-MERGE-JOIN (INNER) TABLES\n" +
" CLIENT PARALLEL 2-WAY SKIP SCAN ON 2 RANGES OVER EVENT_COUNT [0,'5SEC',~1462993520000000000,'Tr/Bal'] - [1,'5SEC',~1462993420000000000,'Tr/Bal']\n" +
" SERVER FILTER BY FIRST KEY ONLY\n" +
+ " SERVER DISTINCT PREFIX FILTER OVER [BUCKET, TIMESTAMP, LOCATION]\n" +
" SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [BUCKET, TIMESTAMP, LOCATION]\n" +
" CLIENT MERGE SORT\n" +
" CLIENT SORTED BY [BUCKET, TIMESTAMP]\n" +
"AND (SKIP MERGE)\n" +
" CLIENT PARALLEL 2-WAY SKIP SCAN ON 2 RANGES OVER " + t[i] + " [0,'5SEC',~1462993520000000000,'Tr/Bal'] - [1,'5SEC',~1462993420000000000,'Tr/Bal']\n" +
" SERVER FILTER BY FIRST KEY ONLY AND SRC_LOCATION = DST_LOCATION\n" +
+ " SERVER DISTINCT PREFIX FILTER OVER [BUCKET, TIMESTAMP, SRC_LOCATION, DST_LOCATION]\n" +
" SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [BUCKET, TIMESTAMP, SRC_LOCATION, DST_LOCATION]\n" +
" CLIENT MERGE SORT\n" +
" CLIENT SORTED BY [BUCKET, TIMESTAMP]\n" +
@@ -459,12 +461,14 @@ public class SortMergeJoinMoreIT extends BaseHBaseManagedTimeIT {
"SORT-MERGE-JOIN (INNER) TABLES\n" +
" CLIENT PARALLEL 2-WAY SKIP SCAN ON 2 RANGES OVER EVENT_COUNT [0,'5SEC',~1462993520000000000,'Tr/Bal'] - [1,'5SEC',~1462993420000000000,'Tr/Bal']\n" +
" SERVER FILTER BY FIRST KEY ONLY\n" +
+ " SERVER DISTINCT PREFIX FILTER OVER [BUCKET, TIMESTAMP, LOCATION]\n" +
" SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [BUCKET, TIMESTAMP, LOCATION]\n" +
" CLIENT MERGE SORT\n" +
" CLIENT SORTED BY [BUCKET, TIMESTAMP]\n" +
"AND (SKIP MERGE)\n" +
" CLIENT PARALLEL 2-WAY SKIP SCAN ON 2 RANGES OVER " + t[i] + " [0,'5SEC',1462993420000000001,'Tr/Bal'] - [1,'5SEC',1462993520000000000,'Tr/Bal']\n" +
" SERVER FILTER BY FIRST KEY ONLY AND SRC_LOCATION = DST_LOCATION\n" +
+ " SERVER DISTINCT PREFIX FILTER OVER [BUCKET, TIMESTAMP, SRC_LOCATION, DST_LOCATION]\n" +
" SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [BUCKET, TIMESTAMP, SRC_LOCATION, DST_LOCATION]\n" +
" CLIENT MERGE SORT\n" +
"CLIENT SORTED BY [E.BUCKET, E.TIMESTAMP]\n" +
http://git-wip-us.apache.org/repos/asf/phoenix/blob/7dcf95a4/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java
index 9cfd9ab..2e55594 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java
@@ -531,7 +531,7 @@ public class IndexExpressionIT extends BaseHBaseManagedTimeIT {
String expectedPlan = "CLIENT PARALLEL 1-WAY RANGE SCAN OVER "
+ (localIndex ? fullDataTableName + " [1,0] - [1,*]"
: "INDEX_TEST.IDX [0] - [*]")
- + "\n SERVER FILTER BY FIRST KEY ONLY\n SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [TO_BIGINT(\"(A.INT_COL1 + 1)\")]"
+ + "\n SERVER FILTER BY FIRST KEY ONLY\n SERVER DISTINCT PREFIX FILTER OVER [TO_BIGINT(\"(A.INT_COL1 + 1)\")]\n SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [TO_BIGINT(\"(A.INT_COL1 + 1)\")]"
+ (localIndex ? "\nCLIENT MERGE SORT" : "");
assertEquals(expectedPlan, QueryUtil.getExplainPlan(rs));
rs = conn.createStatement().executeQuery(sql);
[02/16] phoenix git commit: PHOENIX-1642 Make Phoenix Master Branch
pointing to HBase1.0.0
Posted by la...@apache.org.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/main/java/org/apache/phoenix/trace/TracingUtils.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/trace/TracingUtils.java b/phoenix-core/src/main/java/org/apache/phoenix/trace/TracingUtils.java
index cee3b95..8bd918e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/trace/TracingUtils.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/trace/TracingUtils.java
@@ -19,7 +19,7 @@ package org.apache.phoenix.trace;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
-import org.cloudera.htrace.Span;
+import org.apache.htrace.Span;
/**
* Utilities for tracing
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/main/java/org/apache/phoenix/trace/util/NullSpan.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/trace/util/NullSpan.java b/phoenix-core/src/main/java/org/apache/phoenix/trace/util/NullSpan.java
index 3799fdb..b4f70b9 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/trace/util/NullSpan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/trace/util/NullSpan.java
@@ -21,8 +21,9 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
-import org.cloudera.htrace.Span;
-import org.cloudera.htrace.TimelineAnnotation;
+import org.apache.htrace.Span;
+import org.apache.htrace.TimelineAnnotation;
+import org.apache.phoenix.util.StringUtil;
/**
* Fake {@link Span} that doesn't save any state, in place of <tt>null</tt> return values, to avoid
@@ -109,4 +110,9 @@ public class NullSpan implements Span {
public String getProcessId() {
return null;
}
+
+ @Override
+ public String toJson() {
+ return StringUtil.EMPTY_STRING;
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/main/java/org/apache/phoenix/trace/util/Tracing.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/trace/util/Tracing.java b/phoenix-core/src/main/java/org/apache/phoenix/trace/util/Tracing.java
index 7cd55e8..c9add01 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/trace/util/Tracing.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/trace/util/Tracing.java
@@ -19,6 +19,7 @@ package org.apache.phoenix.trace.util;
import static org.apache.phoenix.util.StringUtil.toBytes;
+import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.Callable;
@@ -28,20 +29,22 @@ import javax.annotation.Nullable;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.htrace.HTraceConfiguration;
import org.apache.phoenix.call.CallRunner;
import org.apache.phoenix.call.CallWrapper;
import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.parse.TraceStatement;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.trace.TraceMetricSource;
-import org.cloudera.htrace.Sampler;
-import org.cloudera.htrace.Span;
-import org.cloudera.htrace.Trace;
-import org.cloudera.htrace.TraceScope;
-import org.cloudera.htrace.Tracer;
-import org.cloudera.htrace.impl.ProbabilitySampler;
-import org.cloudera.htrace.wrappers.TraceCallable;
-import org.cloudera.htrace.wrappers.TraceRunnable;
+import org.apache.htrace.Sampler;
+import org.apache.htrace.Span;
+import org.apache.htrace.Trace;
+import org.apache.htrace.TraceScope;
+import org.apache.htrace.Tracer;
+import org.apache.htrace.impl.ProbabilitySampler;
+import org.apache.htrace.wrappers.TraceCallable;
+import org.apache.htrace.wrappers.TraceRunnable;
import com.google.common.base.Function;
import com.google.common.base.Preconditions;
@@ -58,10 +61,10 @@ public class Tracing {
// Constants for tracing across the wire
public static final String TRACE_ID_ATTRIBUTE_KEY = "phoenix.trace.traceid";
public static final String SPAN_ID_ATTRIBUTE_KEY = "phoenix.trace.spanid";
-
+
// Constants for passing into the metrics system
private static final String TRACE_METRIC_PREFIX = "phoenix.trace.instance";
-
+
/**
* Manage the types of frequencies that we support. By default, we never turn on tracing.
*/
@@ -110,11 +113,12 @@ public class Tracing {
private static Function<ConfigurationAdapter, Sampler<?>> CREATE_PROBABILITY =
new Function<ConfigurationAdapter, Sampler<?>>() {
@Override
- public Sampler<?> apply(ConfigurationAdapter conn) {
+ public Sampler<?> apply(ConfigurationAdapter conf) {
// get the connection properties for the probability information
- String probThresholdStr = conn.get(QueryServices.TRACING_PROBABILITY_THRESHOLD_ATTRIB, null);
- double threshold = probThresholdStr == null ? QueryServicesOptions.DEFAULT_TRACING_PROBABILITY_THRESHOLD : Double.parseDouble(probThresholdStr);
- return new ProbabilitySampler(threshold);
+ Map<String, String> items = new HashMap<String, String>();
+ items.put(ProbabilitySampler.SAMPLER_FRACTION_CONF_KEY,
+ conf.get(QueryServices.TRACING_PROBABILITY_THRESHOLD_ATTRIB, Double.toString(QueryServicesOptions.DEFAULT_TRACING_PROBABILITY_THRESHOLD)));
+ return new ProbabilitySampler(HTraceConfiguration.fromMap(items));
}
};
@@ -130,6 +134,19 @@ public class Tracing {
conf));
}
+ public static Sampler<?> getConfiguredSampler(TraceStatement traceStatement) {
+ double samplingRate = traceStatement.getSamplingRate();
+ if (samplingRate >= 1.0) {
+ return Sampler.ALWAYS;
+ } else if (samplingRate < 1.0 && samplingRate > 0.0) {
+ Map<String, String> items = new HashMap<String, String>();
+ items.put(ProbabilitySampler.SAMPLER_FRACTION_CONF_KEY, Double.toString(samplingRate));
+ return new ProbabilitySampler(HTraceConfiguration.fromMap(items));
+ } else {
+ return Sampler.NEVER;
+ }
+ }
+
private static Sampler<?> getSampler(String traceLevel, ConfigurationAdapter conf) {
return Frequency.getSampler(traceLevel).builder.apply(conf);
}
@@ -202,13 +219,13 @@ public class Tracing {
public static CallWrapper withTracing(PhoenixConnection conn, String desc) {
return new TracingWrapper(conn, desc);
}
-
+
private static void addCustomAnnotationsToSpan(@Nullable Span span, @NotNull PhoenixConnection conn) {
Preconditions.checkNotNull(conn);
-
+
if (span == null) {
return;
- }
+ }
Map<String, String> annotations = conn.getCustomTracingAnnotations();
// copy over the annotations as bytes
for (Map.Entry<String, String> annotation : annotations.entrySet()) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
index c147f91..0ab9791 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
@@ -569,6 +569,10 @@ public class IndexUtil {
return cell.getMvccVersion();
}
+ @Override public long getSequenceId() {
+ return cell.getSequenceId();
+ }
+
@Override
public byte[] getValueArray() {
return cell.getValueArray();
@@ -595,7 +599,7 @@ public class IndexUtil {
}
@Override
- public short getTagsLength() {
+ public int getTagsLength() {
return cell.getTagsLength();
}
@@ -618,12 +622,6 @@ public class IndexUtil {
public byte[] getRow() {
return cell.getRow();
}
-
- @Override
- @Deprecated
- public int getTagsLengthUnsigned() {
- return cell.getTagsLengthUnsigned();
- }
};
// Wrap cell in cell that offsets row key
result.set(i, newCell);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/test/java/org/apache/hadoop/hbase/ipc/PhoenixIndexRpcSchedulerTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/hadoop/hbase/ipc/PhoenixIndexRpcSchedulerTest.java b/phoenix-core/src/test/java/org/apache/hadoop/hbase/ipc/PhoenixIndexRpcSchedulerTest.java
index ec18d9b..8bd8c11 100644
--- a/phoenix-core/src/test/java/org/apache/hadoop/hbase/ipc/PhoenixIndexRpcSchedulerTest.java
+++ b/phoenix-core/src/test/java/org/apache/hadoop/hbase/ipc/PhoenixIndexRpcSchedulerTest.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.ipc;
import static org.junit.Assert.assertEquals;
+import java.net.InetSocketAddress;
import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.TimeUnit;
@@ -37,6 +38,7 @@ import org.mockito.Mockito;
public class PhoenixIndexRpcSchedulerTest {
private static final Configuration conf = HBaseConfiguration.create();
+ private static final InetSocketAddress isa = new InetSocketAddress("localhost", 0);
@Test
public void testIndexPriorityWritesToIndexHandler() throws Exception {
@@ -86,7 +88,7 @@ public class PhoenixIndexRpcSchedulerTest {
private void dispatchCallWithPriority(RpcScheduler scheduler, int priority) throws Exception {
CallRunner task = Mockito.mock(CallRunner.class);
RequestHeader header = RequestHeader.newBuilder().setPriority(priority).build();
- RpcServer server = new RpcServer(null, "test-rpcserver", null, null, conf, scheduler);
+ RpcServer server = new RpcServer(null, "test-rpcserver", null, isa, conf, scheduler);
RpcServer.Call call =
server.new Call(0, null, null, header, null, null, null, null, 10, null);
Mockito.when(task.getCall()).thenReturn(call);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/test/java/org/apache/hadoop/hbase/regionserver/PhoenixIndexRpcSchedulerFactoryTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/hadoop/hbase/regionserver/PhoenixIndexRpcSchedulerFactoryTest.java b/phoenix-core/src/test/java/org/apache/hadoop/hbase/regionserver/PhoenixIndexRpcSchedulerFactoryTest.java
index 4918bba..7d08c0d 100644
--- a/phoenix-core/src/test/java/org/apache/hadoop/hbase/regionserver/PhoenixIndexRpcSchedulerFactoryTest.java
+++ b/phoenix-core/src/test/java/org/apache/hadoop/hbase/regionserver/PhoenixIndexRpcSchedulerFactoryTest.java
@@ -30,13 +30,13 @@ public class PhoenixIndexRpcSchedulerFactoryTest {
@Test
public void ensureInstantiation() throws Exception {
Configuration conf = new Configuration(false);
- conf.setClass(HRegionServer.REGION_SERVER_RPC_SCHEDULER_FACTORY_CLASS,
+ conf.setClass(RSRpcServices.REGION_SERVER_RPC_SCHEDULER_FACTORY_CLASS,
PhoenixIndexRpcSchedulerFactory.class, RpcSchedulerFactory.class);
// kinda lame that we copy the copy from the regionserver to do this and can't use a static
// method, but meh
try {
Class<?> rpcSchedulerFactoryClass =
- conf.getClass(HRegionServer.REGION_SERVER_RPC_SCHEDULER_FACTORY_CLASS,
+ conf.getClass(RSRpcServices.REGION_SERVER_RPC_SCHEDULER_FACTORY_CLASS,
SimpleRpcSchedulerFactory.class);
Object o = rpcSchedulerFactoryClass.newInstance();
assertTrue(o instanceof PhoenixIndexRpcSchedulerFactory);
@@ -63,7 +63,7 @@ public class PhoenixIndexRpcSchedulerFactoryTest {
setMinMax(conf, 0, 4);
factory.create(conf, null);
- setMinMax(conf, 101, 102);
+ setMinMax(conf, 201, 202);
factory.create(conf, null);
setMinMax(conf, 102, 101);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/TestLocalTableState.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/TestLocalTableState.java b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/TestLocalTableState.java
index 8c15551..54db5d8 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/TestLocalTableState.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/TestLocalTableState.java
@@ -76,7 +76,7 @@ public class TestLocalTableState {
public Boolean answer(InvocationOnMock invocation) throws Throwable {
List<KeyValue> list = (List<KeyValue>) invocation.getArguments()[0];
KeyValue kv = new KeyValue(row, fam, qual, ts, Type.Put, stored);
- kv.setMvccVersion(0);
+ kv.setSequenceId(0);
list.add(kv);
return false;
}
@@ -115,7 +115,7 @@ public class TestLocalTableState {
Mockito.when(region.getScanner(Mockito.any(Scan.class))).thenReturn(scanner);
final byte[] stored = Bytes.toBytes("stored-value");
final KeyValue storedKv = new KeyValue(row, fam, qual, ts, Type.Put, stored);
- storedKv.setMvccVersion(2);
+ storedKv.setSequenceId(2);
Mockito.when(scanner.next(Mockito.any(List.class))).thenAnswer(new Answer<Boolean>() {
@Override
public Boolean answer(InvocationOnMock invocation) throws Throwable {
@@ -129,7 +129,7 @@ public class TestLocalTableState {
LocalTableState table = new LocalTableState(env, state, m);
// add the kvs from the mutation
KeyValue kv = KeyValueUtil.ensureKeyValue(m.get(fam, qual).get(0));
- kv.setMvccVersion(0);
+ kv.setSequenceId(0);
table.addPendingUpdates(kv);
// setup the lookup
@@ -161,7 +161,7 @@ public class TestLocalTableState {
Mockito.when(region.getScanner(Mockito.any(Scan.class))).thenReturn(scanner);
final KeyValue storedKv =
new KeyValue(row, fam, qual, ts, Type.Put, Bytes.toBytes("stored-value"));
- storedKv.setMvccVersion(2);
+ storedKv.setSequenceId(2);
Mockito.when(scanner.next(Mockito.any(List.class))).thenAnswer(new Answer<Boolean>() {
@Override
public Boolean answer(InvocationOnMock invocation) throws Throwable {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/data/TestIndexMemStore.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/data/TestIndexMemStore.java b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/data/TestIndexMemStore.java
index 41e7e65..42e0b03 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/data/TestIndexMemStore.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/data/TestIndexMemStore.java
@@ -24,7 +24,6 @@ import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.phoenix.hbase.index.covered.data.IndexMemStore;
import org.junit.Test;
public class TestIndexMemStore {
@@ -40,9 +39,9 @@ public class TestIndexMemStore {
IndexMemStore store = new IndexMemStore(IndexMemStore.COMPARATOR);
long ts = 10;
KeyValue kv = new KeyValue(row, family, qual, ts, Type.Put, val);
- kv.setMvccVersion(2);
+ kv.setSequenceId(2);
KeyValue kv2 = new KeyValue(row, family, qual, ts, Type.Put, val2);
- kv2.setMvccVersion(0);
+ kv2.setSequenceId(0);
store.add(kv, true);
// adding the exact same kv shouldn't change anything stored if not overwritting
store.add(kv2, false);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestWALRecoveryCaching.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestWALRecoveryCaching.java b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestWALRecoveryCaching.java
index 375b754..60c11d7 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestWALRecoveryCaching.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestWALRecoveryCaching.java
@@ -216,7 +216,7 @@ public class TestWALRecoveryCaching {
LOG.info("\t== Offline: " + server.getServerName());
continue;
}
- List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(server);
+ List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(server.getRSRpcServices());
LOG.info("\t" + server.getServerName() + " regions: " + regions);
}
@@ -262,9 +262,9 @@ public class TestWALRecoveryCaching {
}
/**
- * @param miniHBaseCluster
+ * @param cluster
* @param server
- * @param bs
+ * @param table
* @return
*/
private List<HRegion> getRegionsFromServerForTable(MiniHBaseCluster cluster, ServerName server,
@@ -281,9 +281,9 @@ public class TestWALRecoveryCaching {
}
/**
- * @param miniHBaseCluster
- * @param indexedTableName
- * @param tableNameString
+ * @param cluster
+ * @param indexTable
+ * @param primaryTable
*/
private ServerName ensureTablesLiveOnSameServer(MiniHBaseCluster cluster, byte[] indexTable,
byte[] primaryTable) throws Exception {
@@ -366,7 +366,7 @@ public class TestWALRecoveryCaching {
List<HRegion> indexRegions = cluster.getRegions(table);
Set<ServerName> indexServers = new HashSet<ServerName>();
for (HRegion region : indexRegions) {
- indexServers.add(cluster.getServerHoldingRegion(region.getRegionName()));
+ indexServers.add(cluster.getServerHoldingRegion(null, region.getRegionName()));
}
return indexServers;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/recovery/TestPerRegionIndexWriteCache.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/recovery/TestPerRegionIndexWriteCache.java b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/recovery/TestPerRegionIndexWriteCache.java
index a3a02ce..f42dbd7 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/recovery/TestPerRegionIndexWriteCache.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/recovery/TestPerRegionIndexWriteCache.java
@@ -36,10 +36,10 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.wal.HLog;
-import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.wal.WAL;
+import org.apache.hadoop.hbase.wal.WALFactory;
import org.apache.phoenix.hbase.index.table.HTableInterfaceReference;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.junit.After;
@@ -70,13 +70,14 @@ public class TestPerRegionIndexWriteCache {
@SuppressWarnings("deprecation")
@Before
public void setUp() throws Exception {
- FileSystem newFS = FileSystem.get(TEST_UTIL.getConfiguration());
Path hbaseRootDir = TEST_UTIL.getDataTestDir();
-
+ TEST_UTIL.getConfiguration().set("hbase.rootdir", hbaseRootDir.toString());
+
+ FileSystem newFS = FileSystem.newInstance(TEST_UTIL.getConfiguration());
HRegionInfo hri = new HRegionInfo(tableName, null, null, false);
- Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
- HLog wal = HLogFactory.createHLog(newFS,
- hbaseRootDir, "logs", TEST_UTIL.getConfiguration());
+ Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
+ WALFactory walFactory = new WALFactory(TEST_UTIL.getConfiguration(), null, "TestPerRegionIndexWriteCache");
+ WAL wal = walFactory.getWAL(Bytes.toBytes("logs"));
HTableDescriptor htd = new HTableDescriptor(tableName);
HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
htd.addFamily(a);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-core/src/test/java/org/apache/phoenix/trace/TraceMetricsSourceTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/trace/TraceMetricsSourceTest.java b/phoenix-core/src/test/java/org/apache/phoenix/trace/TraceMetricsSourceTest.java
index 4622959..eabcaca 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/trace/TraceMetricsSourceTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/trace/TraceMetricsSourceTest.java
@@ -24,8 +24,8 @@ import org.apache.hadoop.metrics2.MetricsCollector;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.metrics2.MetricsTag;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.cloudera.htrace.Span;
-import org.cloudera.htrace.impl.MilliSpan;
+import org.apache.htrace.Span;
+import org.apache.htrace.impl.MilliSpan;
import org.junit.BeforeClass;
import org.junit.Test;
import org.mockito.Mockito;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/phoenix-flume/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-flume/pom.xml b/phoenix-flume/pom.xml
index af01f6b..1777aa8 100644
--- a/phoenix-flume/pom.xml
+++ b/phoenix-flume/pom.xml
@@ -71,12 +71,12 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.cloudera.htrace</groupId>
+ <groupId>org.apache.htrace</groupId>
<artifactId>htrace-core</artifactId>
</dependency>
<dependency>
<groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
+ <artifactId>netty-all</artifactId>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a29e163f/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 92b0ed6..0e656e7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -75,12 +75,12 @@
<test.output.tofile>true</test.output.tofile>
<!-- Hadoop Versions -->
- <hbase.version>0.98.9-hadoop2</hbase.version>
- <hadoop-two.version>2.2.0</hadoop-two.version>
+ <hbase.version>1.0.1-SNAPSHOT</hbase.version>
+ <hadoop-two.version>2.5.1</hadoop-two.version>
<!-- Dependency versions -->
<commons-cli.version>1.2</commons-cli.version>
- <hadoop.version>1.0.4</hadoop.version>
+ <hadoop.version>2.5.1</hadoop.version>
<pig.version>0.12.0</pig.version>
<jackson.version>1.8.8</jackson.version>
<antlr.version>3.5</antlr.version>
@@ -99,9 +99,9 @@
<findbugs.version>1.3.2</findbugs.version>
<jline.version>2.11</jline.version>
<snappy.version>0.3</snappy.version>
- <netty.version>3.6.6.Final</netty.version>
+ <netty.version>4.0.23.Final</netty.version>
<commons-codec.version>1.7</commons-codec.version>
- <htrace.version>2.04</htrace.version>
+ <htrace.version>3.1.0-incubating</htrace.version>
<collections.version>3.2.1</collections.version>
<jodatime.version>2.3</jodatime.version>
@@ -626,13 +626,13 @@
<version>${slf4j.version}</version>
</dependency>
<dependency>
- <groupId>org.cloudera.htrace</groupId>
+ <groupId>org.apache.htrace</groupId>
<artifactId>htrace-core</artifactId>
<version>${htrace.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
+ <artifactId>netty-all</artifactId>
<version>${netty.version}</version>
</dependency>
<dependency>
[06/16] phoenix git commit: PHOENIX-1746 Pass through guidepost
config params on UPDATE STATISTICS call
Posted by la...@apache.org.
PHOENIX-1746 Pass through guidepost config params on UPDATE STATISTICS call
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/096586e6
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/096586e6
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/096586e6
Branch: refs/heads/4.x-HBase-1.x
Commit: 096586e65e2779433bf30c30e97f78ae2316365e
Parents: e06ceaf
Author: James Taylor <jt...@salesforce.com>
Authored: Mon Mar 23 10:28:23 2015 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Mon Mar 23 10:29:29 2015 -0700
----------------------------------------------------------------------
.../StatsCollectorWithSplitsAndMultiCFIT.java | 6 ++++++
phoenix-core/src/main/antlr3/PhoenixSQL.g | 4 ++--
.../coprocessor/BaseScannerRegionObserver.java | 6 ++++--
.../UngroupedAggregateRegionObserver.java | 4 +++-
.../apache/phoenix/jdbc/PhoenixStatement.java | 9 +++++----
.../apache/phoenix/parse/ParseNodeFactory.java | 4 ++--
.../parse/UpdateStatisticsStatement.java | 11 +++++++++-
.../apache/phoenix/schema/MetaDataClient.java | 19 ++++++++++++++----
.../schema/stats/StatisticsCollector.java | 21 +++++++++++++++-----
9 files changed, 63 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/096586e6/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorWithSplitsAndMultiCFIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorWithSplitsAndMultiCFIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorWithSplitsAndMultiCFIT.java
index c34d598..bcb3a0a 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorWithSplitsAndMultiCFIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorWithSplitsAndMultiCFIT.java
@@ -136,6 +136,12 @@ public class StatsCollectorWithSplitsAndMultiCFIT extends StatsCollectorAbstract
assertRowCountAndByteCount(info, rowCountArr[i], byteCountArr[i]);
i++;
}
+
+ TestUtil.analyzeTable(conn, STATS_TEST_TABLE_NAME_NEW);
+ String query = "UPDATE STATISTICS " + STATS_TEST_TABLE_NAME_NEW + " SET \"" + QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB + "\"=" + Long.toString(2000);
+ conn.createStatement().execute(query);
+ keyRanges = getAllSplits(conn, STATS_TEST_TABLE_NAME_NEW);
+ assertEquals(6, keyRanges.size());
}
protected void assertRowCountAndByteCount(GuidePostsInfo info, long rowCount, long byteCount) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/096586e6/phoenix-core/src/main/antlr3/PhoenixSQL.g
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/antlr3/PhoenixSQL.g b/phoenix-core/src/main/antlr3/PhoenixSQL.g
index 6a2e3b9..0330a39 100644
--- a/phoenix-core/src/main/antlr3/PhoenixSQL.g
+++ b/phoenix-core/src/main/antlr3/PhoenixSQL.g
@@ -520,8 +520,8 @@ alter_table_node returns [AlterTableStatement ret]
;
update_statistics_node returns [UpdateStatisticsStatement ret]
- : UPDATE STATISTICS t=from_table_name (s=INDEX | s=ALL | s=COLUMNS)?
- {ret = factory.updateStatistics(factory.namedTable(null, t), s == null ? StatisticsCollectionScope.getDefault() : StatisticsCollectionScope.valueOf(SchemaUtil.normalizeIdentifier(s.getText())));}
+ : UPDATE STATISTICS t=from_table_name (s=INDEX | s=ALL | s=COLUMNS)? (SET (p=properties))?
+ {ret = factory.updateStatistics(factory.namedTable(null, t), s == null ? StatisticsCollectionScope.getDefault() : StatisticsCollectionScope.valueOf(SchemaUtil.normalizeIdentifier(s.getText())), p);}
;
prop_name returns [String ret]
http://git-wip-us.apache.org/repos/asf/phoenix/blob/096586e6/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
index c3988a0..a2269b4 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
@@ -38,6 +38,8 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.htrace.Span;
+import org.apache.htrace.Trace;
import org.apache.phoenix.execute.TupleProjector;
import org.apache.phoenix.expression.Expression;
import org.apache.phoenix.expression.KeyValueColumnExpression;
@@ -53,8 +55,6 @@ import org.apache.phoenix.schema.tuple.Tuple;
import org.apache.phoenix.util.IndexUtil;
import org.apache.phoenix.util.ScanUtil;
import org.apache.phoenix.util.ServerUtil;
-import org.apache.htrace.Span;
-import org.apache.htrace.Trace;
import com.google.common.collect.ImmutableList;
@@ -85,6 +85,8 @@ abstract public class BaseScannerRegionObserver extends BaseRegionObserver {
public static final String EXPECTED_UPPER_REGION_KEY = "_ExpectedUpperRegionKey";
public static final String REVERSE_SCAN = "_ReverseScan";
public static final String ANALYZE_TABLE = "_ANALYZETABLE";
+ public static final String GUIDEPOST_WIDTH_BYTES = "_GUIDEPOST_WIDTH_BYTES";
+ public static final String GUIDEPOST_PER_REGION = "_GUIDEPOST_PER_REGION";
/**
* Attribute name used to pass custom annotations in Scans and Mutations (later). Custom annotations
* are used to augment log lines emitted by Phoenix. See https://issues.apache.org/jira/browse/PHOENIX-1198.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/096586e6/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
index fc37a84..e43e5e5 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
@@ -162,8 +162,10 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver{
long ts = scan.getTimeRange().getMax();
StatisticsCollector stats = null;
if(ScanUtil.isAnalyzeTable(scan)) {
+ byte[] gp_width_bytes = scan.getAttribute(BaseScannerRegionObserver.GUIDEPOST_WIDTH_BYTES);
+ byte[] gp_per_region_bytes = scan.getAttribute(BaseScannerRegionObserver.GUIDEPOST_PER_REGION);
// Let this throw, as this scan is being done for the sole purpose of collecting stats
- stats = new StatisticsCollector(c.getEnvironment(), region.getRegionInfo().getTable().getNameAsString(), ts);
+ stats = new StatisticsCollector(c.getEnvironment(), region.getRegionInfo().getTable().getNameAsString(), ts, gp_width_bytes, gp_per_region_bytes);
}
if (ScanUtil.isLocalIndex(scan)) {
/*
http://git-wip-us.apache.org/repos/asf/phoenix/blob/096586e6/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
index 996d243..f802ff4 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
@@ -34,6 +34,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import java.util.Map;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.util.Pair;
@@ -717,8 +718,8 @@ public class PhoenixStatement implements Statement, SQLCloseable, org.apache.pho
private static class ExecutableUpdateStatisticsStatement extends UpdateStatisticsStatement implements
CompilableStatement {
- public ExecutableUpdateStatisticsStatement(NamedTableNode table, StatisticsCollectionScope scope) {
- super(table, scope);
+ public ExecutableUpdateStatisticsStatement(NamedTableNode table, StatisticsCollectionScope scope, Map<String,Object> props) {
+ super(table, scope, props);
}
@SuppressWarnings("unchecked")
@@ -919,8 +920,8 @@ public class PhoenixStatement implements Statement, SQLCloseable, org.apache.pho
}
@Override
- public UpdateStatisticsStatement updateStatistics(NamedTableNode table, StatisticsCollectionScope scope) {
- return new ExecutableUpdateStatisticsStatement(table, scope);
+ public UpdateStatisticsStatement updateStatistics(NamedTableNode table, StatisticsCollectionScope scope, Map<String,Object> props) {
+ return new ExecutableUpdateStatisticsStatement(table, scope, props);
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/096586e6/phoenix-core/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java b/phoenix-core/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java
index 82ae821..931f327 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java
@@ -357,8 +357,8 @@ public class ParseNodeFactory {
return new DivideParseNode(children);
}
- public UpdateStatisticsStatement updateStatistics(NamedTableNode table, StatisticsCollectionScope scope) {
- return new UpdateStatisticsStatement(table, scope);
+ public UpdateStatisticsStatement updateStatistics(NamedTableNode table, StatisticsCollectionScope scope, Map<String,Object> props) {
+ return new UpdateStatisticsStatement(table, scope, props);
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/096586e6/phoenix-core/src/main/java/org/apache/phoenix/parse/UpdateStatisticsStatement.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/parse/UpdateStatisticsStatement.java b/phoenix-core/src/main/java/org/apache/phoenix/parse/UpdateStatisticsStatement.java
index dff9f06..6f7b736 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/parse/UpdateStatisticsStatement.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/parse/UpdateStatisticsStatement.java
@@ -21,6 +21,8 @@ import static org.apache.phoenix.schema.stats.StatisticsCollectionScope.ALL;
import static org.apache.phoenix.schema.stats.StatisticsCollectionScope.COLUMNS;
import static org.apache.phoenix.schema.stats.StatisticsCollectionScope.INDEX;
+import java.util.Map;
+
import org.apache.phoenix.schema.stats.StatisticsCollectionScope;
import com.sun.istack.NotNull;
@@ -28,9 +30,12 @@ import com.sun.istack.NotNull;
public class UpdateStatisticsStatement extends SingleTableStatement {
private final StatisticsCollectionScope scope;
- public UpdateStatisticsStatement(NamedTableNode table, @NotNull StatisticsCollectionScope scope) {
+ private final Map<String,Object> props;
+
+ public UpdateStatisticsStatement(NamedTableNode table, @NotNull StatisticsCollectionScope scope, Map<String,Object> props) {
super(table, 0);
this.scope = scope;
+ this.props = props;
}
public boolean updateColumns() {
@@ -43,5 +48,9 @@ public class UpdateStatisticsStatement extends SingleTableStatement {
public boolean updateAll() {
return scope == ALL;
+ }
+
+ public Map<String,Object> getProps() {
+ return props;
};
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/096586e6/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index 7688531..2ba0cde 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -153,6 +153,7 @@ import org.apache.phoenix.schema.PTable.LinkType;
import org.apache.phoenix.schema.PTable.ViewType;
import org.apache.phoenix.schema.stats.PTableStats;
import org.apache.phoenix.schema.types.PDataType;
+import org.apache.phoenix.schema.types.PInteger;
import org.apache.phoenix.schema.types.PLong;
import org.apache.phoenix.schema.types.PVarbinary;
import org.apache.phoenix.schema.types.PVarchar;
@@ -618,7 +619,7 @@ public class MetaDataClient {
PTable table = resolver.getTables().get(0).getTable();
long rowCount = 0;
if (updateStatisticsStmt.updateColumns()) {
- rowCount += updateStatisticsInternal(table.getPhysicalName(), table);
+ rowCount += updateStatisticsInternal(table.getPhysicalName(), table, updateStatisticsStmt.getProps());
}
if (updateStatisticsStmt.updateIndex()) {
// TODO: If our table is a VIEW with multiple indexes or a TABLE with local indexes,
@@ -626,7 +627,7 @@ public class MetaDataClient {
// across all indexes in that case so that we don't re-calculate the same stats
// multiple times.
for (PTable index : table.getIndexes()) {
- rowCount += updateStatisticsInternal(index.getPhysicalName(), index);
+ rowCount += updateStatisticsInternal(index.getPhysicalName(), index, updateStatisticsStmt.getProps());
}
// If analyzing the indexes of a multi-tenant table or a table with view indexes
// then analyze all of those indexes too.
@@ -654,14 +655,14 @@ public class MetaDataClient {
return PTableStats.EMPTY_STATS;
}
};
- rowCount += updateStatisticsInternal(name, indexLogicalTable);
+ rowCount += updateStatisticsInternal(name, indexLogicalTable, updateStatisticsStmt.getProps());
}
}
}
return new MutationState((int)rowCount, connection);
}
- private long updateStatisticsInternal(PName physicalName, PTable logicalTable) throws SQLException {
+ private long updateStatisticsInternal(PName physicalName, PTable logicalTable, Map<String, Object> statsProps) throws SQLException {
ReadOnlyProps props = connection.getQueryServices().getProps();
final long msMinBetweenUpdates = props
.getLong(QueryServices.MIN_STATS_UPDATE_FREQ_MS_ATTRIB,
@@ -691,6 +692,16 @@ public class MetaDataClient {
Scan scan = plan.getContext().getScan();
scan.setCacheBlocks(false);
scan.setAttribute(BaseScannerRegionObserver.ANALYZE_TABLE, PDataType.TRUE_BYTES);
+ if (statsProps != null) {
+ Object gp_width = statsProps.get(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB);
+ if (gp_width != null) {
+ scan.setAttribute(BaseScannerRegionObserver.GUIDEPOST_WIDTH_BYTES, PLong.INSTANCE.toBytes(gp_width));
+ }
+ Object gp_per_region = statsProps.get(QueryServices.STATS_GUIDEPOST_PER_REGION_ATTRIB);
+ if (gp_per_region != null) {
+ scan.setAttribute(BaseScannerRegionObserver.GUIDEPOST_PER_REGION, PInteger.INSTANCE.toBytes(gp_per_region));
+ }
+ }
MutationState mutationState = plan.execute();
rowCount = mutationState.getUpdateCount();
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/096586e6/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
index ade0fba..d6f25c4 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
@@ -38,6 +38,9 @@ import org.apache.phoenix.coprocessor.MetaDataProtocol;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
+import org.apache.phoenix.schema.SortOrder;
+import org.apache.phoenix.schema.types.PInteger;
+import org.apache.phoenix.schema.types.PLong;
import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.TimeKeeper;
import org.slf4j.Logger;
@@ -65,15 +68,23 @@ public class StatisticsCollector {
private Pair<Long,GuidePostsInfo> cachedGps = null;
public StatisticsCollector(RegionCoprocessorEnvironment env, String tableName, long clientTimeStamp) throws IOException {
- this(env, tableName, clientTimeStamp, null);
+ this(env, tableName, clientTimeStamp, null, null, null);
+ }
+
+ public StatisticsCollector(RegionCoprocessorEnvironment env, String tableName, long clientTimeStamp, byte[] gp_width_bytes, byte[] gp_per_region_bytes) throws IOException {
+ this(env, tableName, clientTimeStamp, null, gp_width_bytes, gp_per_region_bytes);
}
public StatisticsCollector(RegionCoprocessorEnvironment env, String tableName, long clientTimeStamp, byte[] family) throws IOException {
+ this(env, tableName, clientTimeStamp, family, null, null);
+ }
+
+ public StatisticsCollector(RegionCoprocessorEnvironment env, String tableName, long clientTimeStamp, byte[] family, byte[] gp_width_bytes, byte[] gp_per_region_bytes) throws IOException {
Configuration config = env.getConfiguration();
- int guidepostPerRegion = config.getInt(QueryServices.STATS_GUIDEPOST_PER_REGION_ATTRIB,
- QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_PER_REGION);
- long guidepostWidth = config.getLong(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB,
- QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_WIDTH_BYTES);
+ int guidepostPerRegion = gp_per_region_bytes == null ? config.getInt(QueryServices.STATS_GUIDEPOST_PER_REGION_ATTRIB,
+ QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_PER_REGION) : PInteger.INSTANCE.getCodec().decodeInt(gp_per_region_bytes, 0, SortOrder.getDefault());
+ long guidepostWidth = gp_width_bytes == null ? config.getLong(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB,
+ QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_WIDTH_BYTES) : PLong.INSTANCE.getCodec().decodeInt(gp_width_bytes, 0, SortOrder.getDefault());
this.guidepostDepth = StatisticsUtil.getGuidePostDepth(guidepostPerRegion, guidepostWidth, env.getRegion().getTableDesc());
// Get the stats table associated with the current table on which the CP is
// triggered
[14/16] phoenix git commit: PHOENIX-258 minor test fixes.
Posted by la...@apache.org.
PHOENIX-258 minor test fixes.
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/65ee886f
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/65ee886f
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/65ee886f
Branch: refs/heads/4.x-HBase-1.0
Commit: 65ee886fb33c91332dcbe342f144a8dafd4afa27
Parents: dfa3eec
Author: Lars Hofhansl <la...@apache.org>
Authored: Thu Jun 2 13:20:48 2016 -0700
Committer: Lars Hofhansl <la...@apache.org>
Committed: Thu Jun 2 13:21:36 2016 -0700
----------------------------------------------------------------------
.../it/java/org/apache/phoenix/end2end/SortMergeJoinMoreIT.java | 4 ++++
.../java/org/apache/phoenix/end2end/index/IndexExpressionIT.java | 2 +-
2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/65ee886f/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortMergeJoinMoreIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortMergeJoinMoreIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortMergeJoinMoreIT.java
index c36b24d..8ede6ae 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortMergeJoinMoreIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortMergeJoinMoreIT.java
@@ -444,12 +444,14 @@ public class SortMergeJoinMoreIT extends BaseHBaseManagedTimeIT {
"SORT-MERGE-JOIN (INNER) TABLES\n" +
" CLIENT PARALLEL 2-WAY SKIP SCAN ON 2 RANGES OVER EVENT_COUNT [0,'5SEC',~1462993520000000000,'Tr/Bal'] - [1,'5SEC',~1462993420000000000,'Tr/Bal']\n" +
" SERVER FILTER BY FIRST KEY ONLY\n" +
+ " SERVER DISTINCT PREFIX FILTER OVER [BUCKET, TIMESTAMP, LOCATION]\n" +
" SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [BUCKET, TIMESTAMP, LOCATION]\n" +
" CLIENT MERGE SORT\n" +
" CLIENT SORTED BY [BUCKET, TIMESTAMP]\n" +
"AND (SKIP MERGE)\n" +
" CLIENT PARALLEL 2-WAY SKIP SCAN ON 2 RANGES OVER " + t[i] + " [0,'5SEC',~1462993520000000000,'Tr/Bal'] - [1,'5SEC',~1462993420000000000,'Tr/Bal']\n" +
" SERVER FILTER BY FIRST KEY ONLY AND SRC_LOCATION = DST_LOCATION\n" +
+ " SERVER DISTINCT PREFIX FILTER OVER [BUCKET, TIMESTAMP, SRC_LOCATION, DST_LOCATION]\n" +
" SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [BUCKET, TIMESTAMP, SRC_LOCATION, DST_LOCATION]\n" +
" CLIENT MERGE SORT\n" +
" CLIENT SORTED BY [BUCKET, TIMESTAMP]\n" +
@@ -459,12 +461,14 @@ public class SortMergeJoinMoreIT extends BaseHBaseManagedTimeIT {
"SORT-MERGE-JOIN (INNER) TABLES\n" +
" CLIENT PARALLEL 2-WAY SKIP SCAN ON 2 RANGES OVER EVENT_COUNT [0,'5SEC',~1462993520000000000,'Tr/Bal'] - [1,'5SEC',~1462993420000000000,'Tr/Bal']\n" +
" SERVER FILTER BY FIRST KEY ONLY\n" +
+ " SERVER DISTINCT PREFIX FILTER OVER [BUCKET, TIMESTAMP, LOCATION]\n" +
" SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [BUCKET, TIMESTAMP, LOCATION]\n" +
" CLIENT MERGE SORT\n" +
" CLIENT SORTED BY [BUCKET, TIMESTAMP]\n" +
"AND (SKIP MERGE)\n" +
" CLIENT PARALLEL 2-WAY SKIP SCAN ON 2 RANGES OVER " + t[i] + " [0,'5SEC',1462993420000000001,'Tr/Bal'] - [1,'5SEC',1462993520000000000,'Tr/Bal']\n" +
" SERVER FILTER BY FIRST KEY ONLY AND SRC_LOCATION = DST_LOCATION\n" +
+ " SERVER DISTINCT PREFIX FILTER OVER [BUCKET, TIMESTAMP, SRC_LOCATION, DST_LOCATION]\n" +
" SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [BUCKET, TIMESTAMP, SRC_LOCATION, DST_LOCATION]\n" +
" CLIENT MERGE SORT\n" +
"CLIENT SORTED BY [E.BUCKET, E.TIMESTAMP]\n" +
http://git-wip-us.apache.org/repos/asf/phoenix/blob/65ee886f/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java
index 5b513d0..cecaf50 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java
@@ -531,7 +531,7 @@ public class IndexExpressionIT extends BaseHBaseManagedTimeIT {
String expectedPlan = "CLIENT PARALLEL 1-WAY RANGE SCAN OVER "
+ (localIndex ? "_LOCAL_IDX_" + fullDataTableName + " [-32768,0] - [-32768,*]"
: "INDEX_TEST.IDX [0] - [*]")
- + "\n SERVER FILTER BY FIRST KEY ONLY\n SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [TO_BIGINT(\"(A.INT_COL1 + 1)\")]"
+ + "\n SERVER FILTER BY FIRST KEY ONLY\n SERVER DISTINCT PREFIX FILTER OVER [TO_BIGINT(\"(A.INT_COL1 + 1)\")]\n SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [TO_BIGINT(\"(A.INT_COL1 + 1)\")]"
+ (localIndex ? "\nCLIENT MERGE SORT" : "");
assertEquals(expectedPlan, QueryUtil.getExplainPlan(rs));
rs = conn.createStatement().executeQuery(sql);
[09/16] phoenix git commit: PHOENIX-1653 Support separate clusters
for MR jobs
Posted by la...@apache.org.
PHOENIX-1653 Support separate clusters for MR jobs
Add support for the input and output formats of a Phoenix MapReduce job to
point to separate clusters using override configuration settings. Defaults to
existing behavior (HConstants.ZOOKEEPER_QUORUM)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/7de8ee1e
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/7de8ee1e
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/7de8ee1e
Branch: refs/heads/4.x-HBase-1.x
Commit: 7de8ee1e914f5e0008ca9d983869757e4ca92b78
Parents: f4180fa
Author: gjacoby <gj...@salesforce.com>
Authored: Fri Feb 27 16:49:14 2015 -0800
Committer: Gabriel Reid <ga...@ngdata.com>
Committed: Tue Mar 24 20:07:52 2015 +0100
----------------------------------------------------------------------
.../phoenix/mapreduce/PhoenixInputFormat.java | 15 ++--
.../phoenix/mapreduce/PhoenixRecordWriter.java | 2 +-
.../phoenix/mapreduce/index/IndexTool.java | 2 +-
.../index/PhoenixIndexImportMapper.java | 2 +-
.../phoenix/mapreduce/util/ConnectionUtil.java | 88 ++++++++++++++------
.../util/PhoenixConfigurationUtil.java | 72 ++++++++++++++--
.../mapreduce/util/PhoenixMapReduceUtil.java | 22 ++++-
.../util/PhoenixConfigurationUtilTest.java | 60 ++++++++++++-
.../pig/util/QuerySchemaParserFunction.java | 2 +-
.../pig/util/SqlQueryToColumnInfoFunction.java | 2 +-
10 files changed, 219 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/7de8ee1e/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixInputFormat.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixInputFormat.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixInputFormat.java
index a83b9ae..31759b4 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixInputFormat.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixInputFormat.java
@@ -98,15 +98,16 @@ public class PhoenixInputFormat<T extends DBWritable> extends InputFormat<NullWr
* @throws IOException
* @throws SQLException
*/
- private QueryPlan getQueryPlan(final JobContext context,final Configuration configuration) throws IOException {
+ private QueryPlan getQueryPlan(final JobContext context, final Configuration configuration)
+ throws IOException {
Preconditions.checkNotNull(context);
- try{
+ try {
final String currentScnValue = configuration.get(PhoenixConfigurationUtil.CURRENT_SCN_VALUE);
final Properties overridingProps = new Properties();
if(currentScnValue != null) {
overridingProps.put(PhoenixRuntime.CURRENT_SCN_ATTRIB, currentScnValue);
}
- final Connection connection = ConnectionUtil.getConnection(configuration,overridingProps);
+ final Connection connection = ConnectionUtil.getInputConnection(configuration, overridingProps);
final String selectStatement = PhoenixConfigurationUtil.getSelectStatement(configuration);
Preconditions.checkNotNull(selectStatement);
final Statement statement = connection.createStatement();
@@ -116,9 +117,11 @@ public class PhoenixInputFormat<T extends DBWritable> extends InputFormat<NullWr
// Initialize the query plan so it sets up the parallel scans
queryPlan.iterator();
return queryPlan;
- } catch(Exception exception) {
- LOG.error(String.format("Failed to get the query plan with error [%s]",exception.getMessage()));
+ } catch (Exception exception) {
+ LOG.error(String.format("Failed to get the query plan with error [%s]",
+ exception.getMessage()));
throw new RuntimeException(exception);
}
- }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/7de8ee1e/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordWriter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordWriter.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordWriter.java
index 4d26bf4..5843076 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordWriter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordWriter.java
@@ -46,7 +46,7 @@ public class PhoenixRecordWriter<T extends DBWritable> extends RecordWriter<Nul
private long numRecords = 0;
public PhoenixRecordWriter(final Configuration configuration) throws SQLException {
- this.conn = ConnectionUtil.getConnection(configuration);
+ this.conn = ConnectionUtil.getOutputConnection(configuration);
this.batchSize = PhoenixConfigurationUtil.getBatchSize(configuration);
final String upsertQuery = PhoenixConfigurationUtil.getUpsertStatement(configuration);
this.statement = this.conn.prepareStatement(upsertQuery);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/7de8ee1e/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java
index d93ef9c..300f575 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java
@@ -163,7 +163,7 @@ public class IndexTool extends Configured implements Tool {
final String qDataTable = SchemaUtil.getTableName(schemaName, dataTable);
final String qIndexTable = SchemaUtil.getTableName(schemaName, indexTable);
- connection = ConnectionUtil.getConnection(configuration);
+ connection = ConnectionUtil.getInputConnection(configuration);
if(!isValidIndexTable(connection, dataTable, indexTable)) {
throw new IllegalArgumentException(String.format(" %s is not an index table for %s ",qIndexTable,qDataTable));
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/7de8ee1e/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixIndexImportMapper.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixIndexImportMapper.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixIndexImportMapper.java
index 7bf4bfc..30f6dc0 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixIndexImportMapper.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixIndexImportMapper.java
@@ -73,7 +73,7 @@ public class PhoenixIndexImportMapper extends Mapper<NullWritable, PhoenixIndexD
indexTableName = PhoenixConfigurationUtil.getOutputTableName(configuration);
final Properties overrideProps = new Properties ();
overrideProps.put(PhoenixRuntime.CURRENT_SCN_ATTRIB, configuration.get(PhoenixConfigurationUtil.CURRENT_SCN_VALUE));
- connection = ConnectionUtil.getConnection(configuration,overrideProps);
+ connection = ConnectionUtil.getOutputConnection(configuration,overrideProps);
connection.setAutoCommit(false);
final String upsertQuery = PhoenixConfigurationUtil.getUpsertStatement(configuration);
this.pStatement = connection.prepareStatement(upsertQuery);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/7de8ee1e/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/ConnectionUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/ConnectionUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/ConnectionUtil.java
index 3234967..e677104 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/ConnectionUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/ConnectionUtil.java
@@ -24,49 +24,89 @@ import java.util.Iterator;
import java.util.Map;
import java.util.Properties;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HConstants;
import org.apache.phoenix.util.QueryUtil;
-import com.google.common.base.Preconditions;
-
/**
* Utility class to return a {@link Connection} .
*/
public class ConnectionUtil {
+
+
+ /**
+ * Retrieve the configured input Connection.
+ *
+ * @param conf configuration containing connection information
+ * @return the configured input connection
+ */
+ public static Connection getInputConnection(final Configuration conf) throws SQLException {
+ return getInputConnection(conf, new Properties());
+ }
/**
- * Returns the {@link Connection} from Configuration
- * @param configuration
- * @return
- * @throws SQLException
+ * Retrieve the configured input Connection.
+ *
+ * @param conf configuration containing connection information
+ * @param props custom connection properties
+ * @return the configured input connection
+ */
+ public static Connection getInputConnection(final Configuration conf , final Properties props) throws SQLException {
+ Preconditions.checkNotNull(conf);
+ return getConnection(PhoenixConfigurationUtil.getInputCluster(conf),
+ extractProperties(props, conf));
+ }
+
+ /**
+ * Create the configured output Connection.
+ *
+ * @param conf configuration containing the connection information
+ * @return the configured output connection
*/
- public static Connection getConnection(final Configuration configuration) throws SQLException {
- return getConnection(configuration, null);
+ public static Connection getOutputConnection(final Configuration conf) throws SQLException {
+ return getOutputConnection(conf, new Properties());
}
/**
- * Used primarily in cases where we need to pass few additional/overriding properties
- * @param configuration
- * @param properties
- * @return
- * @throws SQLException
+ * Create the configured output Connection.
+ *
+ * @param conf configuration containing the connection information
+ * @param props custom connection properties
+ * @return the configured output connection
+ */
+ public static Connection getOutputConnection(final Configuration conf, Properties props) throws SQLException {
+ Preconditions.checkNotNull(conf);
+ return getConnection(PhoenixConfigurationUtil.getOutputCluster(conf),
+ extractProperties(props, conf));
+ }
+
+ /**
+ * Returns the {@link Connection} from a ZooKeeper cluster string.
+ *
+ * @param quorum a ZooKeeper quorum connection string
+ * @return a Phoenix connection to the given connection string
*/
- public static Connection getConnection(final Configuration configuration , final Properties properties) throws SQLException {
- Preconditions.checkNotNull(configuration);
- final Properties props = new Properties();
- Iterator<Map.Entry<String, String>> iterator = configuration.iterator();
+ private static Connection getConnection(final String quorum, Properties props) throws SQLException {
+ Preconditions.checkNotNull(quorum);
+ return DriverManager.getConnection(QueryUtil.getUrl(quorum), props);
+ }
+
+ /**
+ * Add properties from the given Configuration to the provided Properties.
+ *
+ * @param props properties to which connection information from the Configuration will be added
+ * @param conf configuration containing connection information
+ * @return the input Properties value, with additional connection information from the
+ * given Configuration
+ */
+ private static Properties extractProperties(Properties props, final Configuration conf) {
+ Iterator<Map.Entry<String, String>> iterator = conf.iterator();
if(iterator != null) {
while (iterator.hasNext()) {
Map.Entry<String, String> entry = iterator.next();
props.setProperty(entry.getKey(), entry.getValue());
}
}
- if(properties != null && !properties.isEmpty()) {
- props.putAll(properties);
- }
- final Connection conn = DriverManager.getConnection(QueryUtil.getUrl(configuration.get(HConstants.ZOOKEEPER_QUORUM)), props);
- return conn;
+ return props;
}
-
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/7de8ee1e/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java
index b8b64b2..6e0e5e4 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java
@@ -17,18 +17,21 @@
*/
package org.apache.phoenix.mapreduce.util;
-import static org.apache.commons.lang.StringUtils.isNotEmpty;
-
import java.io.IOException;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.List;
import java.util.Map;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Splitter;
+import com.google.common.collect.Lists;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.db.DBInputFormat.NullDBWritable;
@@ -42,10 +45,7 @@ import org.apache.phoenix.util.ColumnInfo;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.QueryUtil;
-import com.google.common.base.Joiner;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Splitter;
-import com.google.common.collect.Lists;
+import static org.apache.commons.lang.StringUtils.isNotEmpty;
/**
* A utility class to set properties on the {#link Configuration} instance.
@@ -90,7 +90,11 @@ public final class PhoenixConfigurationUtil {
/** Configuration key for the class name of an ImportPreUpsertKeyValueProcessor */
public static final String UPSERT_HOOK_CLASS_CONFKEY = "phoenix.mapreduce.import.kvprocessor";
+
+ public static final String MAPREDUCE_INPUT_CLUSTER_QUORUM = "phoenix.mapreduce.input.cluster.quorum";
+ public static final String MAPREDUCE_OUTPUT_CLUSTER_QUORUM = "phoneix.mapreduce.output.cluster.quorum";
+
public enum SchemaType {
TABLE,
QUERY;
@@ -165,6 +169,28 @@ public final class PhoenixConfigurationUtil {
configuration.setLong(UPSERT_BATCH_SIZE, batchSize);
}
+ /**
+ * Sets which HBase cluster a Phoenix MapReduce job should read from
+ * @param configuration
+ * @param quorum ZooKeeper quorum string for HBase cluster the MapReduce job will read from
+ */
+ public static void setInputCluster(final Configuration configuration,
+ final String quorum) {
+ Preconditions.checkNotNull(configuration);
+ configuration.set(MAPREDUCE_INPUT_CLUSTER_QUORUM, quorum);
+ }
+
+ /**
+ * Sets which HBase cluster a Phoenix MapReduce job should write to
+ * @param configuration
+ * @param quorum ZooKeeper quorum string for HBase cluster the MapReduce job will write to
+ */
+ public static void setOutputCluster(final Configuration configuration,
+ final String quorum) {
+ Preconditions.checkNotNull(configuration);
+ configuration.set(MAPREDUCE_OUTPUT_CLUSTER_QUORUM, quorum);
+ }
+
public static Class<?> getInputClass(final Configuration configuration) {
return configuration.getClass(INPUT_CLASS, NullDBWritable.class);
}
@@ -182,7 +208,7 @@ public final class PhoenixConfigurationUtil {
if(isNotEmpty(columnInfoStr)) {
return ColumnInfoToStringEncoderDecoder.decode(columnInfoStr);
}
- final Connection connection = ConnectionUtil.getConnection(configuration);
+ final Connection connection = ConnectionUtil.getOutputConnection(configuration);
String upsertColumns = configuration.get(UPSERT_COLUMNS);
List<String> upsertColumnList = null;
if(isNotEmpty(upsertColumns)) {
@@ -232,7 +258,7 @@ public final class PhoenixConfigurationUtil {
}
final String tableName = getInputTableName(configuration);
Preconditions.checkNotNull(tableName);
- final Connection connection = ConnectionUtil.getConnection(configuration);
+ final Connection connection = ConnectionUtil.getInputConnection(configuration);
final List<String> selectColumnList = getSelectColumnList(configuration);
final List<ColumnInfo> columnMetadataList = PhoenixRuntime.generateColumnInfo(connection, tableName, selectColumnList);
final String encodedColumnInfos = ColumnInfoToStringEncoderDecoder.encode(columnMetadataList);
@@ -276,7 +302,7 @@ public final class PhoenixConfigurationUtil {
Preconditions.checkNotNull(configuration);
long batchSize = configuration.getLong(UPSERT_BATCH_SIZE, DEFAULT_UPSERT_BATCH_SIZE);
if(batchSize <= 0) {
- Connection conn = ConnectionUtil.getConnection(configuration);
+ Connection conn = ConnectionUtil.getOutputConnection(configuration);
batchSize = ((PhoenixConnection) conn).getMutateBatchSize();
conn.close();
}
@@ -309,6 +335,34 @@ public final class PhoenixConfigurationUtil {
Preconditions.checkNotNull(configuration);
return configuration.get(OUTPUT_TABLE_NAME);
}
+
+ /**
+ * Returns the ZooKeeper quorum string for the HBase cluster a Phoenix MapReduce job will read from
+ * @param configuration
+ * @return ZooKeeper quorum string
+ */
+ public static String getInputCluster(final Configuration configuration) {
+ Preconditions.checkNotNull(configuration);
+ String quorum = configuration.get(MAPREDUCE_INPUT_CLUSTER_QUORUM);
+ if (quorum == null) {
+ quorum = configuration.get(HConstants.ZOOKEEPER_QUORUM);
+ }
+ return quorum;
+ }
+
+ /**
+ * Returns the ZooKeeper quorum string for the HBase cluster a Phoenix MapReduce job will write to
+ * @param configuration
+ * @return ZooKeeper quorum string
+ */
+ public static String getOutputCluster(final Configuration configuration) {
+ Preconditions.checkNotNull(configuration);
+ String quorum = configuration.get(MAPREDUCE_OUTPUT_CLUSTER_QUORUM);
+ if (quorum == null) {
+ quorum = configuration.get(HConstants.ZOOKEEPER_QUORUM);
+ }
+ return quorum;
+ }
public static void loadHBaseConfiguration(Job job) throws IOException {
// load hbase-site.xml
http://git-wip-us.apache.org/repos/asf/phoenix/blob/7de8ee1e/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixMapReduceUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixMapReduceUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixMapReduceUtil.java
index f1a7f5a..74d39bd 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixMapReduceUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixMapReduceUtil.java
@@ -49,7 +49,7 @@ public final class PhoenixMapReduceUtil {
PhoenixConfigurationUtil.setInputClass(configuration,inputClass);
PhoenixConfigurationUtil.setSchemaType(configuration, SchemaType.TABLE);
}
-
+
/**
*
* @param job
@@ -64,9 +64,19 @@ public final class PhoenixMapReduceUtil {
PhoenixConfigurationUtil.setInputQuery(configuration, inputQuery);
PhoenixConfigurationUtil.setInputClass(configuration,inputClass);
PhoenixConfigurationUtil.setSchemaType(configuration, SchemaType.QUERY);
+
}
/**
+ * A method to override which HBase cluster for {@link PhoenixInputFormat} to read from
+ * @param job MapReduce Job
+ * @param quorum an HBase cluster's ZooKeeper quorum
+ */
+ public static void setInputCluster(final Job job, final String quorum) {
+ final Configuration configuration = job.getConfiguration();
+ PhoenixConfigurationUtil.setInputCluster(configuration, quorum);
+ }
+ /**
*
* @param job
* @param outputClass
@@ -94,6 +104,16 @@ public final class PhoenixMapReduceUtil {
PhoenixConfigurationUtil.setOutputTableName(configuration, tableName);
PhoenixConfigurationUtil.setUpsertColumnNames(configuration,fieldNames);
}
+
+ /**
+ * A method to override which HBase cluster for {@link PhoenixOutputFormat} to write to
+ * @param job MapReduce Job
+ * @param quorum an HBase cluster's ZooKeeper quorum
+ */
+ public static void setOutputCluster(final Job job, final String quorum) {
+ final Configuration configuration = job.getConfiguration();
+ PhoenixConfigurationUtil.setOutputCluster(configuration, quorum);
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/7de8ee1e/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtilTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtilTest.java b/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtilTest.java
index 33c7531..f8f2a63 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtilTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtilTest.java
@@ -23,13 +23,12 @@ import static org.junit.Assert.assertEquals;
import java.sql.Connection;
import java.sql.DriverManager;
-import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.mapreduce.Job;
import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil.SchemaType;
import org.apache.phoenix.query.BaseConnectionlessQueryTest;
-import org.apache.phoenix.util.ColumnInfo;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.TestUtil;
@@ -39,7 +38,8 @@ import org.junit.Test;
* Test for {@link PhoenixConfigurationUtil}
*/
public class PhoenixConfigurationUtilTest extends BaseConnectionlessQueryTest {
-
+ private static final String ORIGINAL_CLUSTER_QUORUM = "myzookeeperhost";
+ private static final String OVERRIDE_CLUSTER_QUORUM = "myoverridezookeeperhost";
@Test
public void testUpsertStatement() throws Exception {
Connection conn = DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TestUtil.TEST_PROPERTIES));
@@ -121,4 +121,58 @@ public class PhoenixConfigurationUtilTest extends BaseConnectionlessQueryTest {
conn.close();
}
}
+
+ @Test
+ public void testInputClusterOverride() throws Exception {
+ final Configuration configuration = new Configuration();
+ configuration.set(HConstants.ZOOKEEPER_QUORUM, ORIGINAL_CLUSTER_QUORUM);
+ String zkQuorum = PhoenixConfigurationUtil.getInputCluster(configuration);
+ assertEquals(zkQuorum, ORIGINAL_CLUSTER_QUORUM);
+
+ configuration.set(PhoenixConfigurationUtil.MAPREDUCE_INPUT_CLUSTER_QUORUM,
+ OVERRIDE_CLUSTER_QUORUM);
+ String zkQuorumOverride = PhoenixConfigurationUtil.getInputCluster(configuration);
+ assertEquals(zkQuorumOverride, OVERRIDE_CLUSTER_QUORUM);
+
+ final Configuration configuration2 = new Configuration();
+ PhoenixConfigurationUtil.setInputCluster(configuration2, OVERRIDE_CLUSTER_QUORUM);
+ String zkQuorumOverride2 =
+ PhoenixConfigurationUtil.getInputCluster(configuration2);
+ assertEquals(zkQuorumOverride2, OVERRIDE_CLUSTER_QUORUM);
+
+ final Job job = Job.getInstance();
+ PhoenixMapReduceUtil.setInputCluster(job, OVERRIDE_CLUSTER_QUORUM);
+ Configuration configuration3 = job.getConfiguration();
+ String zkQuorumOverride3 =
+ PhoenixConfigurationUtil.getInputCluster(configuration3);
+ assertEquals(zkQuorumOverride3, OVERRIDE_CLUSTER_QUORUM);
+
+ }
+
+ @Test
+ public void testOutputClusterOverride() throws Exception {
+ final Configuration configuration = new Configuration();
+ configuration.set(HConstants.ZOOKEEPER_QUORUM, ORIGINAL_CLUSTER_QUORUM);
+ String zkQuorum = PhoenixConfigurationUtil.getOutputCluster(configuration);
+ assertEquals(zkQuorum, ORIGINAL_CLUSTER_QUORUM);
+
+ configuration.set(PhoenixConfigurationUtil.MAPREDUCE_OUTPUT_CLUSTER_QUORUM,
+ OVERRIDE_CLUSTER_QUORUM);
+ String zkQuorumOverride = PhoenixConfigurationUtil.getOutputCluster(configuration);
+ assertEquals(zkQuorumOverride, OVERRIDE_CLUSTER_QUORUM);
+
+ final Configuration configuration2 = new Configuration();
+ PhoenixConfigurationUtil.setOutputCluster(configuration2, OVERRIDE_CLUSTER_QUORUM);
+ String zkQuorumOverride2 =
+ PhoenixConfigurationUtil.getOutputCluster(configuration2);
+ assertEquals(zkQuorumOverride2, OVERRIDE_CLUSTER_QUORUM);
+
+ final Job job = Job.getInstance();
+ PhoenixMapReduceUtil.setOutputCluster(job, OVERRIDE_CLUSTER_QUORUM);
+ Configuration configuration3 = job.getConfiguration();
+ String zkQuorumOverride3 =
+ PhoenixConfigurationUtil.getOutputCluster(configuration3);
+ assertEquals(zkQuorumOverride3, OVERRIDE_CLUSTER_QUORUM);
+
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/7de8ee1e/phoenix-pig/src/main/java/org/apache/phoenix/pig/util/QuerySchemaParserFunction.java
----------------------------------------------------------------------
diff --git a/phoenix-pig/src/main/java/org/apache/phoenix/pig/util/QuerySchemaParserFunction.java b/phoenix-pig/src/main/java/org/apache/phoenix/pig/util/QuerySchemaParserFunction.java
index f0148a6..4f43811 100644
--- a/phoenix-pig/src/main/java/org/apache/phoenix/pig/util/QuerySchemaParserFunction.java
+++ b/phoenix-pig/src/main/java/org/apache/phoenix/pig/util/QuerySchemaParserFunction.java
@@ -59,7 +59,7 @@ public class QuerySchemaParserFunction implements Function<String,Pair<String,St
Preconditions.checkArgument(!selectStatement.isEmpty(), "Select Query is empty!!");
Connection connection = null;
try {
- connection = ConnectionUtil.getConnection(this.configuration);
+ connection = ConnectionUtil.getInputConnection(this.configuration);
final Statement statement = connection.createStatement();
final PhoenixStatement pstmt = statement.unwrap(PhoenixStatement.class);
final QueryPlan queryPlan = pstmt.compileQuery(selectStatement);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/7de8ee1e/phoenix-pig/src/main/java/org/apache/phoenix/pig/util/SqlQueryToColumnInfoFunction.java
----------------------------------------------------------------------
diff --git a/phoenix-pig/src/main/java/org/apache/phoenix/pig/util/SqlQueryToColumnInfoFunction.java b/phoenix-pig/src/main/java/org/apache/phoenix/pig/util/SqlQueryToColumnInfoFunction.java
index 3ed35bb..2ea2c06 100644
--- a/phoenix-pig/src/main/java/org/apache/phoenix/pig/util/SqlQueryToColumnInfoFunction.java
+++ b/phoenix-pig/src/main/java/org/apache/phoenix/pig/util/SqlQueryToColumnInfoFunction.java
@@ -52,7 +52,7 @@ public final class SqlQueryToColumnInfoFunction implements Function<String,List<
Connection connection = null;
List<ColumnInfo> columnInfos = null;
try {
- connection = ConnectionUtil.getConnection(this.configuration);
+ connection = ConnectionUtil.getInputConnection(this.configuration);
final Statement statement = connection.createStatement();
final PhoenixStatement pstmt = statement.unwrap(PhoenixStatement.class);
final QueryPlan queryPlan = pstmt.compileQuery(sqlQuery);
[13/16] phoenix git commit: PHOENIX-258 minor test fixes.
Posted by la...@apache.org.
PHOENIX-258 minor test fixes.
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/da39bd4e
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/da39bd4e
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/da39bd4e
Branch: refs/heads/4.x-HBase-0.98
Commit: da39bd4e6a1b8fcc9ac813593a8e907d83a8b0e2
Parents: 0032a3d
Author: Lars Hofhansl <la...@apache.org>
Authored: Thu Jun 2 13:20:48 2016 -0700
Committer: Lars Hofhansl <la...@apache.org>
Committed: Thu Jun 2 13:20:48 2016 -0700
----------------------------------------------------------------------
.../it/java/org/apache/phoenix/end2end/SortMergeJoinMoreIT.java | 4 ++++
.../java/org/apache/phoenix/end2end/index/IndexExpressionIT.java | 2 +-
2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/da39bd4e/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortMergeJoinMoreIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortMergeJoinMoreIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortMergeJoinMoreIT.java
index c36b24d..8ede6ae 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortMergeJoinMoreIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortMergeJoinMoreIT.java
@@ -444,12 +444,14 @@ public class SortMergeJoinMoreIT extends BaseHBaseManagedTimeIT {
"SORT-MERGE-JOIN (INNER) TABLES\n" +
" CLIENT PARALLEL 2-WAY SKIP SCAN ON 2 RANGES OVER EVENT_COUNT [0,'5SEC',~1462993520000000000,'Tr/Bal'] - [1,'5SEC',~1462993420000000000,'Tr/Bal']\n" +
" SERVER FILTER BY FIRST KEY ONLY\n" +
+ " SERVER DISTINCT PREFIX FILTER OVER [BUCKET, TIMESTAMP, LOCATION]\n" +
" SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [BUCKET, TIMESTAMP, LOCATION]\n" +
" CLIENT MERGE SORT\n" +
" CLIENT SORTED BY [BUCKET, TIMESTAMP]\n" +
"AND (SKIP MERGE)\n" +
" CLIENT PARALLEL 2-WAY SKIP SCAN ON 2 RANGES OVER " + t[i] + " [0,'5SEC',~1462993520000000000,'Tr/Bal'] - [1,'5SEC',~1462993420000000000,'Tr/Bal']\n" +
" SERVER FILTER BY FIRST KEY ONLY AND SRC_LOCATION = DST_LOCATION\n" +
+ " SERVER DISTINCT PREFIX FILTER OVER [BUCKET, TIMESTAMP, SRC_LOCATION, DST_LOCATION]\n" +
" SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [BUCKET, TIMESTAMP, SRC_LOCATION, DST_LOCATION]\n" +
" CLIENT MERGE SORT\n" +
" CLIENT SORTED BY [BUCKET, TIMESTAMP]\n" +
@@ -459,12 +461,14 @@ public class SortMergeJoinMoreIT extends BaseHBaseManagedTimeIT {
"SORT-MERGE-JOIN (INNER) TABLES\n" +
" CLIENT PARALLEL 2-WAY SKIP SCAN ON 2 RANGES OVER EVENT_COUNT [0,'5SEC',~1462993520000000000,'Tr/Bal'] - [1,'5SEC',~1462993420000000000,'Tr/Bal']\n" +
" SERVER FILTER BY FIRST KEY ONLY\n" +
+ " SERVER DISTINCT PREFIX FILTER OVER [BUCKET, TIMESTAMP, LOCATION]\n" +
" SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [BUCKET, TIMESTAMP, LOCATION]\n" +
" CLIENT MERGE SORT\n" +
" CLIENT SORTED BY [BUCKET, TIMESTAMP]\n" +
"AND (SKIP MERGE)\n" +
" CLIENT PARALLEL 2-WAY SKIP SCAN ON 2 RANGES OVER " + t[i] + " [0,'5SEC',1462993420000000001,'Tr/Bal'] - [1,'5SEC',1462993520000000000,'Tr/Bal']\n" +
" SERVER FILTER BY FIRST KEY ONLY AND SRC_LOCATION = DST_LOCATION\n" +
+ " SERVER DISTINCT PREFIX FILTER OVER [BUCKET, TIMESTAMP, SRC_LOCATION, DST_LOCATION]\n" +
" SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [BUCKET, TIMESTAMP, SRC_LOCATION, DST_LOCATION]\n" +
" CLIENT MERGE SORT\n" +
"CLIENT SORTED BY [E.BUCKET, E.TIMESTAMP]\n" +
http://git-wip-us.apache.org/repos/asf/phoenix/blob/da39bd4e/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java
index 5b513d0..cecaf50 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java
@@ -531,7 +531,7 @@ public class IndexExpressionIT extends BaseHBaseManagedTimeIT {
String expectedPlan = "CLIENT PARALLEL 1-WAY RANGE SCAN OVER "
+ (localIndex ? "_LOCAL_IDX_" + fullDataTableName + " [-32768,0] - [-32768,*]"
: "INDEX_TEST.IDX [0] - [*]")
- + "\n SERVER FILTER BY FIRST KEY ONLY\n SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [TO_BIGINT(\"(A.INT_COL1 + 1)\")]"
+ + "\n SERVER FILTER BY FIRST KEY ONLY\n SERVER DISTINCT PREFIX FILTER OVER [TO_BIGINT(\"(A.INT_COL1 + 1)\")]\n SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [TO_BIGINT(\"(A.INT_COL1 + 1)\")]"
+ (localIndex ? "\nCLIENT MERGE SORT" : "");
assertEquals(expectedPlan, QueryUtil.getExplainPlan(rs));
rs = conn.createStatement().executeQuery(sql);
[10/16] phoenix git commit: PHOENIX-1744 Allow Integer,
UnsignedInt and UnsignedLong to be Cast to TIMESTAMP (Dave Hacker)
Posted by la...@apache.org.
PHOENIX-1744 Allow Integer, UnsignedInt and UnsignedLong to be Cast to TIMESTAMP (Dave Hacker)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/250474de
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/250474de
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/250474de
Branch: refs/heads/4.x-HBase-1.x
Commit: 250474deb381b376d5ed442186470f65b36a8117
Parents: 7de8ee1
Author: David <dh...@salesforce.com>
Authored: Wed Mar 18 13:37:20 2015 -0700
Committer: Thomas <td...@salesforce.com>
Committed: Tue Mar 24 14:00:20 2015 -0700
----------------------------------------------------------------------
.../phoenix/end2end/ToDateFunctionIT.java | 57 ++++++++++++++++++++
.../phoenix/schema/types/PUnsignedLong.java | 5 ++
2 files changed, 62 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/250474de/phoenix-core/src/it/java/org/apache/phoenix/end2end/ToDateFunctionIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ToDateFunctionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ToDateFunctionIT.java
index bda4ea5..8de39b7 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ToDateFunctionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ToDateFunctionIT.java
@@ -33,6 +33,7 @@ import java.sql.Timestamp;
import java.util.Properties;
import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.schema.TypeMismatchException;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -176,4 +177,60 @@ public class ToDateFunctionIT extends BaseHBaseManagedTimeIT {
callToDateFunction(
customTimeZoneConn, "TO_DATE('1970-01-01', 'yyyy-MM-dd')").getTime());
}
+
+ @Test
+ public void testTimestampCast() throws SQLException {
+ Properties props = new Properties();
+ props.setProperty(QueryServices.DATE_FORMAT_TIMEZONE_ATTRIB, "GMT+1");
+ Connection customTimeZoneConn = DriverManager.getConnection(getUrl(), props);
+
+ assertEquals(
+ 1426188807198L,
+ callToDateFunction(
+ customTimeZoneConn, "CAST(1426188807198 AS TIMESTAMP)").getTime());
+
+
+ try {
+ callToDateFunction(
+ customTimeZoneConn, "CAST(22005 AS TIMESTAMP)");
+ fail();
+ } catch (TypeMismatchException e) {
+
+ }
+ }
+
+ @Test
+ public void testUnsignedLongToTimestampCast() throws SQLException {
+ Properties props = new Properties();
+ props.setProperty(QueryServices.DATE_FORMAT_TIMEZONE_ATTRIB, "GMT+1");
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ conn.setAutoCommit(false);
+ try {
+ conn.prepareStatement(
+ "create table TT("
+ + "a unsigned_int not null, "
+ + "b unsigned_int not null, "
+ + "ts unsigned_long not null "
+ + "constraint PK primary key (a, b, ts))").execute();
+ conn.commit();
+
+ conn.prepareStatement("upsert into TT values (0, 22120, 1426188807198)").execute();
+ conn.commit();
+
+ ResultSet rs = conn.prepareStatement("select a, b, ts, CAST(ts AS TIMESTAMP) from TT").executeQuery();
+ assertTrue(rs.next());
+ assertEquals(new Date(1426188807198L), rs.getObject(4));
+ rs.close();
+
+ try {
+ rs = conn.prepareStatement("select a, b, ts, CAST(b AS TIMESTAMP) from TT").executeQuery();
+ fail();
+ } catch (TypeMismatchException e) {
+
+ }
+
+ } finally {
+ conn.close();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/250474de/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PUnsignedLong.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PUnsignedLong.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PUnsignedLong.java
index 67ae05a..a21ccc3 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PUnsignedLong.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PUnsignedLong.java
@@ -95,6 +95,11 @@ public class PUnsignedLong extends PWholeNumber<Long> {
}
@Override
+ public boolean isCastableTo(PDataType targetType) {
+ return super.isCastableTo(targetType) || targetType.isCoercibleTo(PTimestamp.INSTANCE);
+ }
+
+ @Override
public boolean isCoercibleTo(PDataType targetType) {
return targetType == this || targetType == PUnsignedDouble.INSTANCE || PLong.INSTANCE
.isCoercibleTo(targetType);
[15/16] phoenix git commit: PHOENIX-258 minor test fixes.
Posted by la...@apache.org.
PHOENIX-258 minor test fixes.
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/cf551d53
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/cf551d53
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/cf551d53
Branch: refs/heads/4.x-HBase-1.1
Commit: cf551d5368ae557004ffe65bc09e930b8bf3e423
Parents: 7b25276
Author: Lars Hofhansl <la...@apache.org>
Authored: Thu Jun 2 13:20:48 2016 -0700
Committer: Lars Hofhansl <la...@apache.org>
Committed: Thu Jun 2 13:21:54 2016 -0700
----------------------------------------------------------------------
.../it/java/org/apache/phoenix/end2end/SortMergeJoinMoreIT.java | 4 ++++
.../java/org/apache/phoenix/end2end/index/IndexExpressionIT.java | 2 +-
2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cf551d53/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortMergeJoinMoreIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortMergeJoinMoreIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortMergeJoinMoreIT.java
index c36b24d..8ede6ae 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortMergeJoinMoreIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortMergeJoinMoreIT.java
@@ -444,12 +444,14 @@ public class SortMergeJoinMoreIT extends BaseHBaseManagedTimeIT {
"SORT-MERGE-JOIN (INNER) TABLES\n" +
" CLIENT PARALLEL 2-WAY SKIP SCAN ON 2 RANGES OVER EVENT_COUNT [0,'5SEC',~1462993520000000000,'Tr/Bal'] - [1,'5SEC',~1462993420000000000,'Tr/Bal']\n" +
" SERVER FILTER BY FIRST KEY ONLY\n" +
+ " SERVER DISTINCT PREFIX FILTER OVER [BUCKET, TIMESTAMP, LOCATION]\n" +
" SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [BUCKET, TIMESTAMP, LOCATION]\n" +
" CLIENT MERGE SORT\n" +
" CLIENT SORTED BY [BUCKET, TIMESTAMP]\n" +
"AND (SKIP MERGE)\n" +
" CLIENT PARALLEL 2-WAY SKIP SCAN ON 2 RANGES OVER " + t[i] + " [0,'5SEC',~1462993520000000000,'Tr/Bal'] - [1,'5SEC',~1462993420000000000,'Tr/Bal']\n" +
" SERVER FILTER BY FIRST KEY ONLY AND SRC_LOCATION = DST_LOCATION\n" +
+ " SERVER DISTINCT PREFIX FILTER OVER [BUCKET, TIMESTAMP, SRC_LOCATION, DST_LOCATION]\n" +
" SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [BUCKET, TIMESTAMP, SRC_LOCATION, DST_LOCATION]\n" +
" CLIENT MERGE SORT\n" +
" CLIENT SORTED BY [BUCKET, TIMESTAMP]\n" +
@@ -459,12 +461,14 @@ public class SortMergeJoinMoreIT extends BaseHBaseManagedTimeIT {
"SORT-MERGE-JOIN (INNER) TABLES\n" +
" CLIENT PARALLEL 2-WAY SKIP SCAN ON 2 RANGES OVER EVENT_COUNT [0,'5SEC',~1462993520000000000,'Tr/Bal'] - [1,'5SEC',~1462993420000000000,'Tr/Bal']\n" +
" SERVER FILTER BY FIRST KEY ONLY\n" +
+ " SERVER DISTINCT PREFIX FILTER OVER [BUCKET, TIMESTAMP, LOCATION]\n" +
" SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [BUCKET, TIMESTAMP, LOCATION]\n" +
" CLIENT MERGE SORT\n" +
" CLIENT SORTED BY [BUCKET, TIMESTAMP]\n" +
"AND (SKIP MERGE)\n" +
" CLIENT PARALLEL 2-WAY SKIP SCAN ON 2 RANGES OVER " + t[i] + " [0,'5SEC',1462993420000000001,'Tr/Bal'] - [1,'5SEC',1462993520000000000,'Tr/Bal']\n" +
" SERVER FILTER BY FIRST KEY ONLY AND SRC_LOCATION = DST_LOCATION\n" +
+ " SERVER DISTINCT PREFIX FILTER OVER [BUCKET, TIMESTAMP, SRC_LOCATION, DST_LOCATION]\n" +
" SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [BUCKET, TIMESTAMP, SRC_LOCATION, DST_LOCATION]\n" +
" CLIENT MERGE SORT\n" +
"CLIENT SORTED BY [E.BUCKET, E.TIMESTAMP]\n" +
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cf551d53/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java
index 5b513d0..cecaf50 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java
@@ -531,7 +531,7 @@ public class IndexExpressionIT extends BaseHBaseManagedTimeIT {
String expectedPlan = "CLIENT PARALLEL 1-WAY RANGE SCAN OVER "
+ (localIndex ? "_LOCAL_IDX_" + fullDataTableName + " [-32768,0] - [-32768,*]"
: "INDEX_TEST.IDX [0] - [*]")
- + "\n SERVER FILTER BY FIRST KEY ONLY\n SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [TO_BIGINT(\"(A.INT_COL1 + 1)\")]"
+ + "\n SERVER FILTER BY FIRST KEY ONLY\n SERVER DISTINCT PREFIX FILTER OVER [TO_BIGINT(\"(A.INT_COL1 + 1)\")]\n SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [TO_BIGINT(\"(A.INT_COL1 + 1)\")]"
+ (localIndex ? "\nCLIENT MERGE SORT" : "");
assertEquals(expectedPlan, QueryUtil.getExplainPlan(rs));
rs = conn.createStatement().executeQuery(sql);
[11/16] phoenix git commit: PHOENIX-1684 Functional Index using
REGEXP_SUBSTR doesn't work correctly
Posted by la...@apache.org.
PHOENIX-1684 Functional Index using REGEXP_SUBSTR doesn't work correctly
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/a8b27e3f
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/a8b27e3f
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/a8b27e3f
Branch: refs/heads/4.x-HBase-1.x
Commit: a8b27e3f010d15d3f3b519c38fbb052ebb4a6cdb
Parents: 250474d
Author: Thomas <td...@salesforce.com>
Authored: Thu Mar 19 13:57:27 2015 -0700
Committer: Thomas <td...@salesforce.com>
Committed: Tue Mar 24 15:56:25 2015 -0700
----------------------------------------------------------------------
.../end2end/index/IndexExpressionIT.java | 161 ++++++++++++++-----
.../phoenix/compile/PostIndexDDLCompiler.java | 4 +-
.../parse/IndexExpressionParseNodeRewriter.java | 30 +---
.../apache/phoenix/schema/MetaDataClient.java | 4 +-
.../org/apache/phoenix/util/StringUtil.java | 5 +
.../phoenix/compile/QueryCompilerTest.java | 22 ++-
6 files changed, 153 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a8b27e3f/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java
index 1e3733b..0203e35 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java
@@ -1202,54 +1202,60 @@ public class IndexExpressionIT extends BaseHBaseManagedTimeIT {
@Test
public void testViewUsesTableIndex() throws Exception {
- ResultSet rs;
Connection conn = DriverManager.getConnection(getUrl());
- String ddl = "CREATE TABLE t (k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, s1 VARCHAR, s2 VARCHAR, s3 VARCHAR, s4 VARCHAR CONSTRAINT pk PRIMARY KEY (k1, k2))";
- conn.createStatement().execute(ddl);
- conn.createStatement().execute("CREATE INDEX i1 ON t(k2, s2, s3, s1)");
- conn.createStatement().execute("CREATE INDEX i2 ON t(k2, s2||'_'||s3, s1, s4)");
-
- ddl = "CREATE VIEW v AS SELECT * FROM t WHERE s1 = 'foo'";
- conn.createStatement().execute(ddl);
- conn.createStatement().execute("UPSERT INTO t VALUES(1,1,'foo','abc','cab')");
- conn.createStatement().execute("UPSERT INTO t VALUES(2,2,'bar','xyz','zyx')");
- conn.commit();
-
- rs = conn.createStatement().executeQuery("SELECT count(*) FROM v");
- assertTrue(rs.next());
- assertEquals(1, rs.getLong(1));
- assertFalse(rs.next());
-
- //i2 should be used since it contains s3||'_'||s4 i
- String query = "SELECT s2||'_'||s3 FROM v WHERE k2=1 AND (s2||'_'||s3)='abc_cab'";
- rs = conn.createStatement( ).executeQuery("EXPLAIN " + query);
- String queryPlan = QueryUtil.getExplainPlan(rs);
- assertEquals(
- "CLIENT PARALLEL 1-WAY RANGE SCAN OVER I2 [1,'abc_cab','foo']\n" +
- " SERVER FILTER BY FIRST KEY ONLY", queryPlan);
- rs = conn.createStatement().executeQuery(query);
- assertTrue(rs.next());
- assertEquals("abc_cab", rs.getString(1));
- assertFalse(rs.next());
-
- conn.createStatement().execute("ALTER VIEW v DROP COLUMN s4");
- //i2 cannot be used since s4 has been dropped from the view, so i1 will be used
- rs = conn.createStatement().executeQuery("EXPLAIN " + query);
- queryPlan = QueryUtil.getExplainPlan(rs);
- assertEquals(
- "CLIENT PARALLEL 1-WAY RANGE SCAN OVER I1 [1]\n" +
- " SERVER FILTER BY FIRST KEY ONLY AND ((\"S2\" || '_' || \"S3\") = 'abc_cab' AND \"S1\" = 'foo')", queryPlan);
- rs = conn.createStatement().executeQuery(query);
- assertTrue(rs.next());
- assertEquals("abc_cab", rs.getString(1));
- assertFalse(rs.next());
+ try
+ {
+ ResultSet rs;
+ String ddl = "CREATE TABLE t (k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, s1 VARCHAR, s2 VARCHAR, s3 VARCHAR, s4 VARCHAR CONSTRAINT pk PRIMARY KEY (k1, k2))";
+ conn.createStatement().execute(ddl);
+ conn.createStatement().execute("CREATE INDEX i1 ON t(k2, s2, s3, s1)");
+ conn.createStatement().execute("CREATE INDEX i2 ON t(k2, s2||'_'||s3, s1, s4)");
+
+ ddl = "CREATE VIEW v AS SELECT * FROM t WHERE s1 = 'foo'";
+ conn.createStatement().execute(ddl);
+ conn.createStatement().execute("UPSERT INTO t VALUES(1,1,'foo','abc','cab')");
+ conn.createStatement().execute("UPSERT INTO t VALUES(2,2,'bar','xyz','zyx')");
+ conn.commit();
+
+ rs = conn.createStatement().executeQuery("SELECT count(*) FROM v");
+ assertTrue(rs.next());
+ assertEquals(1, rs.getLong(1));
+ assertFalse(rs.next());
+
+ //i2 should be used since it contains s3||'_'||s4 i
+ String query = "SELECT s2||'_'||s3 FROM v WHERE k2=1 AND (s2||'_'||s3)='abc_cab'";
+ rs = conn.createStatement( ).executeQuery("EXPLAIN " + query);
+ String queryPlan = QueryUtil.getExplainPlan(rs);
+ assertEquals(
+ "CLIENT PARALLEL 1-WAY RANGE SCAN OVER I2 [1,'abc_cab','foo']\n" +
+ " SERVER FILTER BY FIRST KEY ONLY", queryPlan);
+ rs = conn.createStatement().executeQuery(query);
+ assertTrue(rs.next());
+ assertEquals("abc_cab", rs.getString(1));
+ assertFalse(rs.next());
+
+ conn.createStatement().execute("ALTER VIEW v DROP COLUMN s4");
+ //i2 cannot be used since s4 has been dropped from the view, so i1 will be used
+ rs = conn.createStatement().executeQuery("EXPLAIN " + query);
+ queryPlan = QueryUtil.getExplainPlan(rs);
+ assertEquals(
+ "CLIENT PARALLEL 1-WAY RANGE SCAN OVER I1 [1]\n" +
+ " SERVER FILTER BY FIRST KEY ONLY AND ((\"S2\" || '_' || \"S3\") = 'abc_cab' AND \"S1\" = 'foo')", queryPlan);
+ rs = conn.createStatement().executeQuery(query);
+ assertTrue(rs.next());
+ assertEquals("abc_cab", rs.getString(1));
+ assertFalse(rs.next());
+ }
+ finally {
+ conn.close();
+ }
}
@Test
public void testExpressionThrowsException() throws Exception {
Connection conn = DriverManager.getConnection(getUrl());
- String ddl = "CREATE TABLE t (k1 INTEGER PRIMARY KEY, k2 INTEGER)";
try {
+ String ddl = "CREATE TABLE t (k1 INTEGER PRIMARY KEY, k2 INTEGER)";
conn.createStatement().execute(ddl);
ddl = "CREATE INDEX i on t(k1/k2)";
conn.createStatement().execute(ddl);
@@ -1261,6 +1267,79 @@ public class IndexExpressionIT extends BaseHBaseManagedTimeIT {
conn.commit();
fail();
} catch (CommitException e) {
+ } finally {
+ conn.close();
+ }
+ }
+
+ @Test
+ public void testImmutableCaseSensitiveFunctionIndex() throws Exception {
+ helpTestCaseSensitiveFunctionIndex(false, false);
+ }
+
+ @Test
+ public void testImmutableLocalCaseSensitiveFunctionIndex() throws Exception {
+ helpTestCaseSensitiveFunctionIndex(false, true);
+ }
+
+ @Test
+ public void testMutableCaseSensitiveFunctionIndex() throws Exception {
+ helpTestCaseSensitiveFunctionIndex(true, false);
+ }
+
+ @Test
+ public void testMutableLocalCaseSensitiveFunctionIndex() throws Exception {
+ helpTestCaseSensitiveFunctionIndex(true, true);
+ }
+
+ protected void helpTestCaseSensitiveFunctionIndex(boolean mutable,
+ boolean localIndex) throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ try {
+ conn.createStatement().execute(
+ "CREATE TABLE t (k VARCHAR NOT NULL PRIMARY KEY, v VARCHAR) "
+ + (mutable ? "IMMUTABLE_ROWS=true" : ""));
+ String query = "SELECT * FROM t";
+ ResultSet rs = conn.createStatement().executeQuery(query);
+ assertFalse(rs.next());
+ String ddl = "CREATE " + (localIndex ? "LOCAL" : "")
+ + " INDEX idx ON t (REGEXP_SUBSTR(v,'id:\\\\w+'))";
+ PreparedStatement stmt = conn.prepareStatement(ddl);
+ stmt.execute();
+ query = "SELECT * FROM idx";
+ rs = conn.createStatement().executeQuery(query);
+ assertFalse(rs.next());
+
+ stmt = conn.prepareStatement("UPSERT INTO t VALUES(?,?)");
+ stmt.setString(1, "k1");
+ stmt.setString(2, "{id:id1}");
+ stmt.execute();
+ stmt.setString(1, "k2");
+ stmt.setString(2, "{id:id2}");
+ stmt.execute();
+ conn.commit();
+
+ query = "SELECT k FROM t WHERE REGEXP_SUBSTR(v,'id:\\\\w+') = 'id:id1'";
+ rs = conn.createStatement().executeQuery("EXPLAIN " + query);
+ if (localIndex) {
+ assertEquals(
+ "CLIENT PARALLEL 1-WAY RANGE SCAN OVER _LOCAL_IDX_T [-32768,'id:id1']\n"
+ + " SERVER FILTER BY FIRST KEY ONLY\nCLIENT MERGE SORT",
+ QueryUtil.getExplainPlan(rs));
+ } else {
+ assertEquals(
+ "CLIENT PARALLEL 1-WAY RANGE SCAN OVER IDX ['id:id1']\n"
+ + " SERVER FILTER BY FIRST KEY ONLY",
+ QueryUtil.getExplainPlan(rs));
+ }
+
+ rs = conn.createStatement().executeQuery(query);
+ assertTrue(rs.next());
+ assertEquals("k1", rs.getString(1));
+ assertFalse(rs.next());
+ } finally {
+ conn.close();
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a8b27e3f/phoenix-core/src/main/java/org/apache/phoenix/compile/PostIndexDDLCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/PostIndexDDLCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/PostIndexDDLCompiler.java
index 5836b99..9f99f1c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/PostIndexDDLCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/PostIndexDDLCompiler.java
@@ -27,6 +27,7 @@ import org.apache.phoenix.schema.PColumnFamily;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.TableRef;
import org.apache.phoenix.util.IndexUtil;
+import org.apache.phoenix.util.StringUtil;
import com.google.common.collect.Lists;
@@ -73,7 +74,8 @@ public class PostIndexDDLCompiler {
for (int i = posOffset; i < nIndexPKColumns; i++) {
PColumn col = indexPKColumns.get(i);
String indexColName = col.getName().getString();
- String dataColName = col.getExpressionStr();
+ // need to escape backslash as this used in the SELECT statement
+ String dataColName = StringUtil.escapeBackslash(col.getExpressionStr());
dataColumns.append(dataColName).append(",");
indexColumns.append('"').append(indexColName).append("\",");
indexColumnNames.add(indexColName);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a8b27e3f/phoenix-core/src/main/java/org/apache/phoenix/parse/IndexExpressionParseNodeRewriter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/parse/IndexExpressionParseNodeRewriter.java b/phoenix-core/src/main/java/org/apache/phoenix/parse/IndexExpressionParseNodeRewriter.java
index 43cb9f3..0273041 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/parse/IndexExpressionParseNodeRewriter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/parse/IndexExpressionParseNodeRewriter.java
@@ -37,26 +37,6 @@ public class IndexExpressionParseNodeRewriter extends ParseNodeRewriter {
private final Map<ParseNode, ParseNode> indexedParseNodeToColumnParseNodeMap;
- private static class ColumnParseNodeVisitor extends StatelessTraverseAllParseNodeVisitor {
-
- private boolean isParseNodeCaseSensitive;
-
- public void reset() {
- this.isParseNodeCaseSensitive = false;
- }
-
- @Override
- public Void visit(ColumnParseNode node) throws SQLException {
- isParseNodeCaseSensitive = isParseNodeCaseSensitive || node.isCaseSensitive() || node.isTableNameCaseSensitive();
- return null;
- }
-
- public boolean isParseNodeCaseSensitive() {
- return isParseNodeCaseSensitive;
- }
-
- }
-
public IndexExpressionParseNodeRewriter(PTable index, PhoenixConnection connection) throws SQLException {
indexedParseNodeToColumnParseNodeMap = Maps.newHashMapWithExpectedSize(index.getColumns().size());
NamedTableNode tableNode = NamedTableNode.create(null,
@@ -66,21 +46,13 @@ public class IndexExpressionParseNodeRewriter extends ParseNodeRewriter {
StatementContext context = new StatementContext(new PhoenixStatement(connection), dataResolver);
IndexStatementRewriter rewriter = new IndexStatementRewriter(dataResolver, null);
ExpressionCompiler expressionCompiler = new ExpressionCompiler(context);
- ColumnParseNodeVisitor columnParseNodeVisitor = new ColumnParseNodeVisitor();
int indexPosOffset = (index.getBucketNum() == null ? 0 : 1) + (index.isMultiTenant() ? 1 : 0) + (index.getViewIndexId() == null ? 0 : 1);
List<PColumn> pkColumns = index.getPKColumns();
for (int i=indexPosOffset; i<pkColumns.size(); ++i) {
PColumn column = pkColumns.get(i);
String expressionStr = IndexUtil.getIndexColumnExpressionStr(column);
ParseNode expressionParseNode = SQLParser.parseCondition(expressionStr);
- columnParseNodeVisitor.reset();
- expressionParseNode.accept(columnParseNodeVisitor);
- String colName = column.getName().getString();
- if (columnParseNodeVisitor.isParseNodeCaseSensitive()) {
- // force column name to be case sensitive name by surround with double quotes
- colName = "\"" + colName + "\"";
- }
-
+ String colName = "\"" + column.getName().getString() + "\"";
Expression dataExpression = expressionParseNode.accept(expressionCompiler);
PDataType expressionDataType = dataExpression.getDataType();
ParseNode indexedParseNode = expressionParseNode.accept(rewriter);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a8b27e3f/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index 2ba0cde..e414039 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -164,6 +164,7 @@ import org.apache.phoenix.util.MetaDataUtil;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.SchemaUtil;
+import org.apache.phoenix.util.StringUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -1036,7 +1037,8 @@ public class MetaDataClient {
// can lose information during compilation
StringBuilder buf = new StringBuilder();
parseNode.toSQL(resolver, buf);
- String expressionStr = buf.toString();
+ // need to escape backslash as this expression will be re-parsed later
+ String expressionStr = StringUtil.escapeBackslash(buf.toString());
ColumnName colName = null;
ColumnRef colRef = expressionIndexCompiler.getColumnRef();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a8b27e3f/phoenix-core/src/main/java/org/apache/phoenix/util/StringUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/StringUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/StringUtil.java
index a83098a..4a7ae38 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/StringUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/StringUtil.java
@@ -331,4 +331,9 @@ public class StringUtil {
public static String escapeStringConstant(String pattern) {
return StringEscapeUtils.escapeSql(pattern); // Need to escape double quotes
}
+
+ public static String escapeBackslash(String input) {
+ // see http://stackoverflow.com/questions/4653831/regex-how-to-escape-backslashes-and-special-characters
+ return input.replaceAll("\\\\","\\\\\\\\");
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a8b27e3f/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
index e17c528..83c984b 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
@@ -1564,5 +1564,25 @@ public class QueryCompilerTest extends BaseConnectionlessQueryTest {
stmt.close();
}
}
-
+
+ @Test
+ public void testRegex() throws Exception {
+ Connection conn = DriverManager.getConnection(getUrl());
+ Statement stmt = conn.createStatement();
+ stmt.execute("CREATE TABLE t (k1 INTEGER PRIMARY KEY, v VARCHAR)");
+
+ //character classes
+ stmt.executeQuery("select * from T where REGEXP_SUBSTR(v, '[abc]') = 'val'");
+ stmt.executeQuery("select * from T where REGEXP_SUBSTR(v, '[^abc]') = 'val'");
+ stmt.executeQuery("select * from T where REGEXP_SUBSTR(v, '[a-zA-Z]') = 'val'");
+ stmt.executeQuery("select * from T where REGEXP_SUBSTR(v, '[a-d[m-p]]') = 'val'");
+ stmt.executeQuery("select * from T where REGEXP_SUBSTR(v, '[a-z&&[def]]') = 'val'");
+ stmt.executeQuery("select * from T where REGEXP_SUBSTR(v, '[a-z&&[^bc]]') = 'val'");
+ stmt.executeQuery("select * from T where REGEXP_SUBSTR(v, '[a-z&&[^m-p]]') = 'val'");
+
+ // predefined character classes
+ stmt.executeQuery("select * from T where REGEXP_SUBSTR(v, '.\\\\d\\\\D\\\\s\\\\S\\\\w\\\\W') = 'val'");
+ }
+
+
}
[04/16] phoenix git commit: PHOENIX-1703 Fail connection when server
minor version is less than client minor version
Posted by la...@apache.org.
PHOENIX-1703 Fail connection when server minor version is less than client minor version
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/4bc162d8
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/4bc162d8
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/4bc162d8
Branch: refs/heads/4.x-HBase-1.x
Commit: 4bc162d8f254d01bfff71d429a124e5c2d146054
Parents: a29e163
Author: James Taylor <jt...@salesforce.com>
Authored: Thu Mar 19 22:23:20 2015 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Sat Mar 21 11:10:04 2015 -0700
----------------------------------------------------------------------
.../org/apache/phoenix/util/MetaDataUtil.java | 22 +++++++++++++-------
.../apache/phoenix/util/MetaDataUtilTest.java | 18 +++++++++-------
2 files changed, 24 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/4bc162d8/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
index c1aa2cc..f916f5b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
@@ -49,16 +49,16 @@ import org.apache.phoenix.hbase.index.util.VersionUtil;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.query.QueryConstants;
-import org.apache.phoenix.schema.types.PBoolean;
-import org.apache.phoenix.schema.types.PDataType;
-import org.apache.phoenix.schema.types.PLong;
import org.apache.phoenix.schema.PName;
-import org.apache.phoenix.schema.types.PSmallint;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.schema.SequenceKey;
import org.apache.phoenix.schema.SortOrder;
import org.apache.phoenix.schema.TableNotFoundException;
+import org.apache.phoenix.schema.types.PBoolean;
+import org.apache.phoenix.schema.types.PDataType;
+import org.apache.phoenix.schema.types.PLong;
+import org.apache.phoenix.schema.types.PSmallint;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -79,21 +79,22 @@ public class MetaDataUtil {
public static final String PARENT_TABLE_KEY = "PARENT_TABLE";
public static final byte[] PARENT_TABLE_KEY_BYTES = Bytes.toBytes("PARENT_TABLE");
- public static boolean areClientAndServerCompatible(long version) {
+ public static boolean areClientAndServerCompatible(long serverHBaseAndPhoenixVersion) {
// As of 3.0, we allow a client and server to differ for the minor version.
// Care has to be taken to upgrade the server before the client, as otherwise
// the client may call expressions that don't yet exist on the server.
// Differing by the patch version has always been allowed.
// Only differing by the major version is not allowed.
- return areClientAndServerCompatible(MetaDataUtil.decodePhoenixVersion(version), MetaDataProtocol.PHOENIX_MAJOR_VERSION);
+ return areClientAndServerCompatible(MetaDataUtil.decodePhoenixVersion(serverHBaseAndPhoenixVersion), MetaDataProtocol.PHOENIX_MAJOR_VERSION, MetaDataProtocol.PHOENIX_MINOR_VERSION);
}
// Default scope for testing
- static boolean areClientAndServerCompatible(int version, int pMajor) {
+ static boolean areClientAndServerCompatible(int serverVersion, int clientMajorVersion, int clientMinorVersion) {
// A server and client with the same major and minor version number must be compatible.
// So it's important that we roll the PHOENIX_MAJOR_VERSION or PHOENIX_MINOR_VERSION
// when we make an incompatible change.
- return VersionUtil.encodeMaxMinorVersion(pMajor) >= version && VersionUtil.encodeMinMinorVersion(pMajor) <= version;
+ return VersionUtil.encodeMinPatchVersion(clientMajorVersion, clientMinorVersion) <= serverVersion && // Minor major and minor cannot be ahead of server
+ VersionUtil.encodeMaxMinorVersion(clientMajorVersion) >= serverVersion; // Major version must at least be up to server version
}
// Given the encoded integer representing the phoenix version in the encoded version value.
@@ -129,6 +130,11 @@ public class MetaDataUtil {
return major + "." + minor + "." + patch;
}
+ public static int encodePhoenixVersion() {
+ return VersionUtil.encodeVersion(MetaDataProtocol.PHOENIX_MAJOR_VERSION, MetaDataProtocol.PHOENIX_MINOR_VERSION,
+ MetaDataProtocol.PHOENIX_PATCH_NUMBER);
+ }
+
public static long encodeHBaseAndPhoenixVersions(String hbaseVersion) {
return (((long) VersionUtil.encodeVersion(hbaseVersion)) << (Byte.SIZE * 5)) |
(((long) VersionUtil.encodeVersion(MetaDataProtocol.PHOENIX_MAJOR_VERSION, MetaDataProtocol.PHOENIX_MINOR_VERSION,
http://git-wip-us.apache.org/repos/asf/phoenix/blob/4bc162d8/phoenix-core/src/test/java/org/apache/phoenix/util/MetaDataUtilTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/MetaDataUtilTest.java b/phoenix-core/src/test/java/org/apache/phoenix/util/MetaDataUtilTest.java
index 427b3bf..9597b9c 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/util/MetaDataUtilTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/util/MetaDataUtilTest.java
@@ -45,14 +45,16 @@ public class MetaDataUtilTest {
@Test
public void testCompatibility() {
- assertTrue(MetaDataUtil.areClientAndServerCompatible(VersionUtil.encodeVersion(1,2,1), 1));
- assertTrue(MetaDataUtil.areClientAndServerCompatible(VersionUtil.encodeVersion(1,2,10), 1));
- assertTrue(MetaDataUtil.areClientAndServerCompatible(VersionUtil.encodeVersion(1,2,0), 1));
- assertTrue(MetaDataUtil.areClientAndServerCompatible(VersionUtil.encodeVersion(1,2,255), 1));
- assertTrue(MetaDataUtil.areClientAndServerCompatible(VersionUtil.encodeVersion(2,2,0), 2));
- assertTrue(MetaDataUtil.areClientAndServerCompatible(VersionUtil.encodeVersion(2,10,36), 2));
- assertFalse(MetaDataUtil.areClientAndServerCompatible(VersionUtil.encodeVersion(3,1,10), 4));
- assertFalse(MetaDataUtil.areClientAndServerCompatible(VersionUtil.encodeVersion(3,1,10), 2));
+ assertTrue(MetaDataUtil.areClientAndServerCompatible(VersionUtil.encodeVersion(1,2,1), 1, 2));
+ assertTrue(MetaDataUtil.areClientAndServerCompatible(VersionUtil.encodeVersion(1,2,10), 1, 1));
+ assertTrue(MetaDataUtil.areClientAndServerCompatible(VersionUtil.encodeVersion(1,2,0), 1, 2));
+ assertTrue(MetaDataUtil.areClientAndServerCompatible(VersionUtil.encodeVersion(1,2,255), 1, 2));
+ assertTrue(MetaDataUtil.areClientAndServerCompatible(VersionUtil.encodeVersion(2,2,0), 2, 0));
+ assertTrue(MetaDataUtil.areClientAndServerCompatible(VersionUtil.encodeVersion(2,10,36), 2, 9));
+ assertFalse(MetaDataUtil.areClientAndServerCompatible(VersionUtil.encodeVersion(3,1,10), 4, 0));
+ assertFalse(MetaDataUtil.areClientAndServerCompatible(VersionUtil.encodeVersion(3,1,10), 2, 0));
+ assertFalse(MetaDataUtil.areClientAndServerCompatible(VersionUtil.encodeVersion(3,1,10), 3, 2));
+ assertFalse(MetaDataUtil.areClientAndServerCompatible(VersionUtil.encodeVersion(3,1,10), 3, 5));
}
/**