You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ja...@apache.org on 2016/10/10 20:07:26 UTC
[1/3] phoenix git commit: PHOENIX-3361 Collect stats correct for
local indexes
Repository: phoenix
Updated Branches:
refs/heads/master ad9995279 -> eaf99f23d
http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java
index 291cb6d..43fe7f3 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java
@@ -22,51 +22,53 @@ import java.util.List;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.phoenix.util.ByteUtil;
+import org.apache.phoenix.util.SizedUtil;
+
+import com.google.common.primitives.Longs;
/**
* A class that holds the guidePosts of a region and also allows combining the
* guidePosts of different regions when the GuidePostsInfo is formed for a table.
*/
public class GuidePostsInfo {
+ public final static GuidePostsInfo EMPTY_GUIDEPOST = new GuidePostsInfo(new ArrayList<Long>(),
+ new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY), new ArrayList<Long>(), 0, 0) {
+ @Override
+ public int getEstimatedSize() {
+ return 0;
+ }
+ };
+ public final static GuidePostsInfo NO_GUIDEPOST = new GuidePostsInfo(new ArrayList<Long>(),
+ new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY), new ArrayList<Long>(), 0, 0) {
+ @Override
+ public int getEstimatedSize() {
+ return 0;
+ }
+ };
/**
* the total number of guidePosts for the table combining all the guidePosts per region per cf.
*/
- private ImmutableBytesWritable guidePosts;
-
+ private final ImmutableBytesWritable guidePosts;
/**
* Maximum length of a guidePost collected
*/
- private int maxLength;
-
- public final static GuidePostsInfo NO_GUIDEPOST = new GuidePostsInfo(new ArrayList<Long>(),
- new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY), new ArrayList<Long>(), 0, 0);
-
- public int getMaxLength() {
- return maxLength;
- }
-
+ private final int maxLength;
/**
* Number of guidePosts
*/
- private int guidePostsCount;
-
+ private final int guidePostsCount;
/**
* The rowCounts of each guidePost traversed
*/
- private List<Long> rowCounts;
-
+ private final long[] rowCounts;
/**
* The bytecounts of each guidePost traversed
*/
- private List<Long> byteCounts;
-
- public List<Long> getRowCounts() {
- return rowCounts;
- }
-
- public List<Long> getByteCounts() {
- return byteCounts;
- }
+ private final long[] byteCounts;
+ /**
+ * Estimate of byte size of this instance
+ */
+ private final int estimatedSize;
/**
* Constructor that creates GuidePostsInfo per region
@@ -87,8 +89,16 @@ public class GuidePostsInfo {
this.guidePosts = new ImmutableBytesWritable(guidePosts);
this.maxLength = maxLength;
this.guidePostsCount = guidePostsCount;
- this.rowCounts = rowCounts;
- this.byteCounts = byteCounts;
+ this.rowCounts = Longs.toArray(rowCounts);
+ this.byteCounts = Longs.toArray(byteCounts);
+ int estimatedSize = SizedUtil.OBJECT_SIZE
+ + SizedUtil.IMMUTABLE_BYTES_WRITABLE_SIZE + guidePosts.getLength() // guidePosts
+ + SizedUtil.INT_SIZE // maxLength
+ + SizedUtil.INT_SIZE // guidePostsCount
+ + SizedUtil.ARRAY_SIZE + this.rowCounts.length * SizedUtil.LONG_SIZE // rowCounts
+ + SizedUtil.ARRAY_SIZE + this.byteCounts.length * SizedUtil.LONG_SIZE // byteCounts
+ + SizedUtil.INT_SIZE; // estimatedSize
+ this.estimatedSize = estimatedSize;
}
public ImmutableBytesWritable getGuidePosts() {
@@ -98,5 +108,20 @@ public class GuidePostsInfo {
public int getGuidePostsCount() {
return guidePostsCount;
}
+
+ public int getMaxLength() {
+ return maxLength;
+ }
+ public long[] getRowCounts() {
+ return rowCounts;
+ }
+
+ public long[] getByteCounts() {
+ return byteCounts;
+ }
+
+ public int getEstimatedSize() {
+ return estimatedSize;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java
index 435f30a..246ef6c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java
@@ -51,6 +51,10 @@ public class GuidePostsInfoBuilder {
private List<Long> rowCounts = new ArrayList<Long>();
private List<Long> byteCounts = new ArrayList<Long>();
+ public boolean isEmpty() {
+ return rowCounts.size() == 0;
+ }
+
public List<Long> getRowCounts() {
return rowCounts;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsKey.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsKey.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsKey.java
new file mode 100644
index 0000000..3bde602
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsKey.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.schema.stats;
+
+import java.util.Arrays;
+
+import javax.annotation.Nonnull;
+
+import org.apache.hadoop.hbase.util.Bytes;
+
+import com.google.common.base.Preconditions;
+
+/**
+ *
+ * Key for the client-side caching of the guideposts information
+ *
+ */
+public final class GuidePostsKey {
+ private final int hashCode;
+ @Nonnull private final byte[] physicalName;
+ @Nonnull private final byte[] columnFamily;
+
+ public GuidePostsKey(byte[] physicalName, byte[] columnFamily) {
+ Preconditions.checkNotNull(physicalName);
+ Preconditions.checkNotNull(columnFamily);
+ this.physicalName = physicalName;
+ this.columnFamily = columnFamily;
+ this.hashCode = computeHashCode();
+ }
+
+ public byte[] getPhysicalName() {
+ return physicalName;
+ }
+
+ public byte[] getColumnFamily() {
+ return columnFamily;
+ }
+
+ @Override
+ public int hashCode() {
+ return hashCode;
+ }
+
+ private int computeHashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + Arrays.hashCode(columnFamily);
+ result = prime * result + Arrays.hashCode(physicalName);
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) return true;
+ if (obj == null) return false;
+ if (getClass() != obj.getClass()) return false;
+ GuidePostsKey other = (GuidePostsKey)obj;
+ if (other.hashCode != this.hashCode) return false;
+ if (!Arrays.equals(columnFamily, other.columnFamily)) return false;
+ if (!Arrays.equals(physicalName, other.physicalName)) return false;
+ return true;
+ }
+
+ @Override
+ public String toString() {
+ return "GuidePostsKey[physicalName=" + Bytes.toStringBinary(physicalName)
+ + ",columnFamily=" + Bytes.toStringBinary(columnFamily) + "]";
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java
index a16a48a..74d1710 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.util.List;
import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
import org.apache.hadoop.hbase.regionserver.Region;
@@ -44,7 +45,7 @@ public class NoOpStatisticsCollector implements StatisticsCollector {
}
@Override
- public void updateStatistic(Region region) {
+ public void updateStatistic(Region region, Scan scan) {
// No-op
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStats.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStats.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStats.java
deleted file mode 100644
index f297b3b..0000000
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStats.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.phoenix.schema.stats;
-
-import java.util.SortedMap;
-
-import com.google.common.collect.ImmutableSortedMap;
-
-
-/*
- * The table is defined on the client side, but it is populated on the server side. The client should not populate any data to the
- * statistics object.
- */
-public interface PTableStats {
- public static final PTableStats EMPTY_STATS = new PTableStats() {
- @Override
- public SortedMap<byte[], GuidePostsInfo> getGuidePosts() {
- return ImmutableSortedMap.of();
- }
-
- @Override
- public int getEstimatedSize() {
- return 0;
- }
-
- @Override
- public long getTimestamp() {
- return DefaultStatisticsCollector.NO_TIMESTAMP;
- }
- };
-
- /**
- * TODO: Change from TreeMap to Map
- * Returns a tree map of the guide posts collected against a column family
- * @return
- */
- SortedMap<byte[], GuidePostsInfo> getGuidePosts();
-
- int getEstimatedSize();
-
- long getTimestamp();
-}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java
deleted file mode 100644
index 5eb02fd..0000000
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.phoenix.schema.stats;
-
-import java.io.ByteArrayInputStream;
-import java.io.DataInput;
-import java.io.DataInputStream;
-import java.io.EOFException;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.phoenix.coprocessor.MetaDataProtocol;
-import org.apache.phoenix.util.Closeables;
-import org.apache.phoenix.util.PrefixByteCodec;
-import org.apache.phoenix.util.PrefixByteDecoder;
-import org.apache.phoenix.util.SizedUtil;
-
-import com.sun.istack.NotNull;
-
- /**
- * Implementation for PTableStats.
- */
-public class PTableStatsImpl implements PTableStats {
- private final SortedMap<byte[], GuidePostsInfo> guidePosts;
- private final int estimatedSize;
- private final long timeStamp;
-
- public PTableStatsImpl() {
- this(new TreeMap<byte[], GuidePostsInfo>(Bytes.BYTES_COMPARATOR), MetaDataProtocol.MIN_TABLE_TIMESTAMP);
- }
-
- public PTableStatsImpl(@NotNull SortedMap<byte[], GuidePostsInfo> guidePosts, long timeStamp) {
- this.guidePosts = guidePosts;
- this.timeStamp = timeStamp;
- int estimatedSize = SizedUtil.OBJECT_SIZE + SizedUtil.INT_SIZE + SizedUtil.sizeOfTreeMap(guidePosts.size());
- for (Map.Entry<byte[], GuidePostsInfo> entry : guidePosts.entrySet()) {
- byte[] cf = entry.getKey();
- estimatedSize += SizedUtil.ARRAY_SIZE + cf.length;
- estimatedSize += entry.getValue().getGuidePosts().getLength();
- estimatedSize += SizedUtil.LONG_SIZE;
- estimatedSize += SizedUtil.INT_SIZE;
- estimatedSize += SizedUtil.INT_SIZE;
- }
- this.estimatedSize = estimatedSize;
- }
-
- @Override
- public SortedMap<byte[], GuidePostsInfo> getGuidePosts() {
- return guidePosts;
- }
-
- @Override
- public String toString() {
- StringBuilder buf = new StringBuilder();
-
- buf.append("PTableStats [");
- for (Map.Entry<byte[], GuidePostsInfo> entry : guidePosts.entrySet()) {
- buf.append(Bytes.toStringBinary(entry.getKey()));
- buf.append(":(");
- ImmutableBytesWritable keys = entry.getValue().getGuidePosts();
- ByteArrayInputStream stream = new ByteArrayInputStream(keys.get(), keys.getOffset(), keys.getLength());
- try {
- if (keys.getLength() != 0) {
- DataInput input = new DataInputStream(stream);
- PrefixByteDecoder decoder = new PrefixByteDecoder(entry.getValue().getMaxLength());
- try {
- while (true) {
- ImmutableBytesWritable ptr = PrefixByteCodec.decode(decoder, input);
- buf.append(Bytes.toStringBinary(ptr.get()));
- buf.append(",");
- }
- } catch (EOFException e) { // Ignore as this signifies we're done
-
- } finally {
- Closeables.closeQuietly(stream);
- }
- buf.setLength(buf.length() - 1);
- }
- buf.append(")");
- } finally {
- Closeables.closeQuietly(stream);
- }
- }
- buf.append("]");
- return buf.toString();
- }
-
- @Override
- public int getEstimatedSize() {
- return estimatedSize;
- }
-
- @Override
- public long getTimestamp() {
- return timeStamp;
- }
-}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
index 43ec6c7..60e83a8 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.List;
import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
import org.apache.hadoop.hbase.regionserver.Region;
@@ -42,9 +43,9 @@ public interface StatisticsCollector extends Closeable {
long getMaxTimeStamp();
/**
- * Write the collected statistics for the given region.
+ * Write the collected statistics for the given region over the scan provided.
*/
- void updateStatistic(Region region);
+ void updateStatistic(Region region, Scan scan);
/**
* Collect statistics for the given list of cells. This method can be called multiple times
http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
index 685c24e..07b412f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
@@ -75,11 +75,11 @@ public class StatisticsUtil {
/** Number of parts in our complex key */
protected static final int NUM_KEY_PARTS = 3;
- public static byte[] getRowKey(byte[] table, ImmutableBytesPtr fam, byte[] guidePostStartKey) {
+ public static byte[] getRowKey(byte[] table, ImmutableBytesWritable fam, byte[] guidePostStartKey) {
return getRowKey(table, fam, new ImmutableBytesWritable(guidePostStartKey,0,guidePostStartKey.length));
}
- public static byte[] getRowKey(byte[] table, ImmutableBytesPtr fam, ImmutableBytesWritable guidePostStartKey) {
+ public static byte[] getRowKey(byte[] table, ImmutableBytesWritable fam, ImmutableBytesWritable guidePostStartKey) {
// always starts with the source table
int guidePostLength = guidePostStartKey.getLength();
boolean hasGuidePost = guidePostLength > 0;
@@ -97,21 +97,36 @@ public class StatisticsUtil {
return rowKey;
}
- private static byte[] getKey(byte[] table, ImmutableBytesPtr fam) {
+ private static byte[] getStartKey(byte[] table, ImmutableBytesWritable fam) {
+ return getKey(table, fam, false);
+ }
+
+ private static byte[] getEndKey(byte[] table, ImmutableBytesWritable fam) {
+ byte[] key = getKey(table, fam, true);
+ ByteUtil.nextKey(key, key.length);
+ return key;
+ }
+
+ private static byte[] getKey(byte[] table, ImmutableBytesWritable fam, boolean terminateWithSeparator) {
// always starts with the source table and column family
- byte[] rowKey = new byte[table.length + fam.getLength() + 1];
+ byte[] rowKey = new byte[table.length + fam.getLength() + 1 + (terminateWithSeparator ? 1 : 0)];
int offset = 0;
System.arraycopy(table, 0, rowKey, offset, table.length);
offset += table.length;
rowKey[offset++] = QueryConstants.SEPARATOR_BYTE; // assumes stats table columns not DESC
System.arraycopy(fam.get(), fam.getOffset(), rowKey, offset, fam.getLength());
offset += fam.getLength();
+ if (terminateWithSeparator) {
+ rowKey[offset] = QueryConstants.SEPARATOR_BYTE;
+ }
return rowKey;
}
- private static byte[] getAdjustedKey(byte[] key, byte[] tableNameBytes, ImmutableBytesPtr cf, boolean nextKey) {
- if (Bytes.compareTo(key, ByteUtil.EMPTY_BYTE_ARRAY) != 0) { return getRowKey(tableNameBytes, cf, key); }
- key = ByteUtil.concat(getKey(tableNameBytes, cf), QueryConstants.SEPARATOR_BYTE_ARRAY);
+ private static byte[] getAdjustedKey(byte[] key, byte[] tableNameBytes, ImmutableBytesWritable cf, boolean nextKey) {
+ if (Bytes.compareTo(key, ByteUtil.EMPTY_BYTE_ARRAY) != 0) {
+ return getRowKey(tableNameBytes, cf, key);
+ }
+ key = getKey(tableNameBytes, cf, nextKey);
if (nextKey) {
ByteUtil.nextKey(key, key.length);
}
@@ -121,8 +136,10 @@ public class StatisticsUtil {
public static List<Result> readStatistics(HTableInterface statsHTable, byte[] tableNameBytes, ImmutableBytesPtr cf,
byte[] startKey, byte[] stopKey, long clientTimeStamp) throws IOException {
List<Result> statsForRegion = new ArrayList<Result>();
- Scan s = MetaDataUtil.newTableRowsScan(getAdjustedKey(startKey, tableNameBytes, cf, false),
- getAdjustedKey(stopKey, tableNameBytes, cf, true), MetaDataProtocol.MIN_TABLE_TIMESTAMP,
+ Scan s = MetaDataUtil.newTableRowsScan(
+ getAdjustedKey(startKey, tableNameBytes, cf, false),
+ getAdjustedKey(stopKey, tableNameBytes, cf, true),
+ MetaDataProtocol.MIN_TABLE_TIMESTAMP,
clientTimeStamp);
s.addColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES);
ResultScanner scanner = null;
@@ -140,40 +157,27 @@ public class StatisticsUtil {
return statsForRegion;
}
- public static PTableStats readStatistics(HTableInterface statsHTable, byte[] tableNameBytes, long clientTimeStamp)
+ public static GuidePostsInfo readStatistics(HTableInterface statsHTable, GuidePostsKey key, long clientTimeStamp)
throws IOException {
ImmutableBytesWritable ptr = new ImmutableBytesWritable();
- Scan s = MetaDataUtil.newTableRowsScan(tableNameBytes, MetaDataProtocol.MIN_TABLE_TIMESTAMP, clientTimeStamp);
+ ptr.set(key.getColumnFamily());
+ byte[] tableNameBytes = key.getPhysicalName();
+ byte[] startKey = getStartKey(tableNameBytes, ptr);
+ byte[] endKey = getEndKey(tableNameBytes, ptr);
+ Scan s = MetaDataUtil.newTableRowsScan(startKey, endKey, MetaDataProtocol.MIN_TABLE_TIMESTAMP, clientTimeStamp);
s.addColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH_BYTES);
s.addColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT_BYTES);
s.addColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES);
- ResultScanner scanner = null;
- long timeStamp = MetaDataProtocol.MIN_TABLE_TIMESTAMP;
- TreeMap<byte[], GuidePostsInfoBuilder> guidePostsInfoWriterPerCf = new TreeMap<byte[], GuidePostsInfoBuilder>(Bytes.BYTES_COMPARATOR);
- try {
- scanner = statsHTable.getScanner(s);
+ GuidePostsInfoBuilder guidePostsInfoWriter = new GuidePostsInfoBuilder();
+ Cell current = null;
+ try (ResultScanner scanner = statsHTable.getScanner(s)) {
Result result = null;
while ((result = scanner.next()) != null) {
CellScanner cellScanner = result.cellScanner();
long rowCount = 0;
long byteCount = 0;
- byte[] cfName = null;
- int tableNameLength;
- int cfOffset;
- int cfLength;
- boolean valuesSet = false;
- // Only the two cells with quals GUIDE_POSTS_ROW_COUNT_BYTES and GUIDE_POSTS_BYTES would be retrieved
- while (cellScanner.advance()) {
- Cell current = cellScanner.current();
- if (!valuesSet) {
- tableNameLength = tableNameBytes.length + 1;
- cfOffset = current.getRowOffset() + tableNameLength;
- cfLength = getVarCharLength(current.getRowArray(), cfOffset,
- current.getRowLength() - tableNameLength);
- ptr.set(current.getRowArray(), cfOffset, cfLength);
- valuesSet = true;
- }
- cfName = ByteUtil.copyKeyBytesIfNecessary(ptr);
+ while (cellScanner.advance()) {
+ current = cellScanner.current();
if (Bytes.equals(current.getQualifierArray(), current.getQualifierOffset(),
current.getQualifierLength(), PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT_BYTES, 0,
PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT_BYTES.length)) {
@@ -185,28 +189,22 @@ public class StatisticsUtil {
byteCount = PLong.INSTANCE.getCodec().decodeLong(current.getValueArray(),
current.getValueOffset(), SortOrder.getDefault());
}
- if (current.getTimestamp() > timeStamp) {
- timeStamp = current.getTimestamp();
- }
}
- if (cfName != null) {
+ if (current != null) {
+ int tableNameLength = tableNameBytes.length + 1;
+ int cfOffset = current.getRowOffset() + tableNameLength;
+ int cfLength = getVarCharLength(current.getRowArray(), cfOffset,
+ current.getRowLength() - tableNameLength);
+ ptr.set(current.getRowArray(), cfOffset, cfLength);
+ byte[] cfName = ByteUtil.copyKeyBytesIfNecessary(ptr);
byte[] newGPStartKey = getGuidePostsInfoFromRowKey(tableNameBytes, cfName, result.getRow());
- GuidePostsInfoBuilder guidePostsInfoWriter = guidePostsInfoWriterPerCf.get(cfName);
- if (guidePostsInfoWriter == null) {
- guidePostsInfoWriter = new GuidePostsInfoBuilder();
- guidePostsInfoWriterPerCf.put(cfName, guidePostsInfoWriter);
- }
guidePostsInfoWriter.addGuidePosts(newGPStartKey, byteCount, rowCount);
}
}
- if (!guidePostsInfoWriterPerCf.isEmpty()) { return new PTableStatsImpl(
- getGuidePostsPerCf(guidePostsInfoWriterPerCf), timeStamp); }
- } finally {
- if (scanner != null) {
- scanner.close();
- }
}
- return PTableStats.EMPTY_STATS;
+ // We write a row with an empty KeyValue in the case that stats were generated but without enough data
+ // for any guideposts. If we have no rows, it means stats were never generated.
+ return current == null ? GuidePostsInfo.NO_GUIDEPOST : guidePostsInfoWriter.isEmpty() ? GuidePostsInfo.EMPTY_GUIDEPOST : guidePostsInfoWriter.build();
}
private static SortedMap<byte[], GuidePostsInfo> getGuidePostsPerCf(
http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java
index b40d34c..fb8d664 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java
@@ -135,8 +135,8 @@ public class StatisticsWriter implements Closeable {
}
GuidePostsInfo gps = tracker.getGuidePosts(cfKey);
if (gps != null) {
- List<Long> byteCounts = gps.getByteCounts();
- List<Long> rowCounts = gps.getRowCounts();
+ long[] byteCounts = gps.getByteCounts();
+ long[] rowCounts = gps.getRowCounts();
ImmutableBytesWritable keys = gps.getGuidePosts();
boolean hasGuidePosts = keys.getLength() > 0;
if (hasGuidePosts) {
@@ -146,7 +146,7 @@ public class StatisticsWriter implements Closeable {
PrefixByteDecoder decoder = new PrefixByteDecoder(gps.getMaxLength());
do {
ImmutableBytesWritable ptr = decoder.decode(input);
- addGuidepost(cfKey, mutations, ptr, byteCounts.get(guidePostCount), rowCounts.get(guidePostCount), timeStamp);
+ addGuidepost(cfKey, mutations, ptr, byteCounts[guidePostCount], rowCounts[guidePostCount], timeStamp);
guidePostCount++;
} while (decoder != null);
} catch (EOFException e) { // Ignore as this signifies we're done
http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
index 52346a5..7acd30b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
@@ -639,4 +639,8 @@ public class MetaDataUtil {
return cfPtr.getLength() >= QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX_BYTES.length &&
Bytes.compareTo(cfPtr.get(), cfPtr.getOffset(), QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX_BYTES.length, QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX_BYTES, 0, QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX_BYTES.length) == 0;
}
+
+ public static boolean isLocalIndexFamily(byte[] cf) {
+ return Bytes.startsWith(cf, QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX_BYTES);
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java
index ad99514..9ba7f6d 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java
@@ -24,7 +24,6 @@ import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.util.Map;
import java.util.Properties;
-import java.util.SortedMap;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.compile.QueryPlan;
@@ -32,7 +31,6 @@ import org.apache.phoenix.end2end.Shadower;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixStatement;
import org.apache.phoenix.query.BaseConnectionlessQueryTest;
-import org.apache.phoenix.query.ConnectionQueryServicesImpl;
import org.apache.phoenix.query.ConnectionlessQueryServicesImpl;
import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.query.QueryServices;
@@ -40,7 +38,7 @@ import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTableKey;
import org.apache.phoenix.schema.stats.GuidePostsInfo;
import org.apache.phoenix.schema.stats.GuidePostsInfoBuilder;
-import org.apache.phoenix.schema.stats.PTableStats;
+import org.apache.phoenix.schema.stats.GuidePostsKey;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.ReadOnlyProps;
import org.junit.BeforeClass;
@@ -653,29 +651,10 @@ public class SkipScanBigFilterTest extends BaseConnectionlessQueryTest {
gpWriter.addGuidePosts(gp, 1000);
}
GuidePostsInfo info = gpWriter.build();
- final SortedMap<byte[], GuidePostsInfo> gpMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
- gpMap.put(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, info);
- PTableStats stats = new PTableStats() {
-
- @Override
- public SortedMap<byte[], GuidePostsInfo> getGuidePosts() {
- return gpMap;
- }
-
- @Override
- public int getEstimatedSize() {
- return 10000;
- }
-
- @Override
- public long getTimestamp() {
- return table.getTimeStamp()+1;
- }
- };
PhoenixConnection pConn = conn.unwrap(PhoenixConnection.class);
pConn.addTable(table, System.currentTimeMillis());
((ConnectionlessQueryServicesImpl) pConn.getQueryServices())
- .addTableStats(table.getName().getBytesPtr(), stats);
+ .addTableStats(new GuidePostsKey(table.getName().getBytes(), QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES), info);
String query = "SELECT count(1) cnt,\n" +
" coalesce(SUM(impressions), 0.0) AS \"impressions\",\n" +
http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/test/java/org/apache/phoenix/query/PhoenixStatsCacheRemovalListenerTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/PhoenixStatsCacheRemovalListenerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/PhoenixStatsCacheRemovalListenerTest.java
index b4e01a7..7f84219 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/query/PhoenixStatsCacheRemovalListenerTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/PhoenixStatsCacheRemovalListenerTest.java
@@ -19,7 +19,7 @@ package org.apache.phoenix.query;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
-import org.apache.phoenix.query.TableStatsCache.PhoenixStatsCacheRemovalListener;
+import org.apache.phoenix.query.GuidePostsCache.PhoenixStatsCacheRemovalListener;
import org.junit.Test;
import com.google.common.cache.RemovalCause;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
index de5af46..03469e2 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
@@ -52,6 +52,7 @@ import java.util.Properties;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.HBaseAdmin;
@@ -105,11 +106,12 @@ import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.PColumn;
import org.apache.phoenix.schema.PLongColumn;
import org.apache.phoenix.schema.PName;
+import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.RowKeyValueAccessor;
import org.apache.phoenix.schema.SortOrder;
import org.apache.phoenix.schema.TableRef;
import org.apache.phoenix.schema.stats.GuidePostsInfo;
-import org.apache.phoenix.schema.stats.PTableStats;
+import org.apache.phoenix.schema.stats.GuidePostsKey;
import org.apache.phoenix.schema.tuple.Tuple;
import org.apache.phoenix.schema.types.PDataType;
@@ -595,9 +597,9 @@ public class TestUtil {
pstmt.execute();
TableRef tableRef = pstmt.getQueryPlan().getTableRef();
PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class);
- long scn = null == pconn.getSCN() ? Long.MAX_VALUE : pconn.getSCN();
- PTableStats tableStats = pconn.getQueryServices().getTableStats(tableRef.getTable().getName().getBytes(), scn);
- return tableStats.getGuidePosts().values();
+ PTable table = tableRef.getTable();
+ GuidePostsInfo info = pconn.getQueryServices().getTableStats(new GuidePostsKey(table.getName().getBytes(), SchemaUtil.getEmptyColumnFamily(table)));
+ return Collections.singletonList(info);
}
public static void analyzeTable(Connection conn, String tableName) throws IOException, SQLException {
@@ -793,5 +795,23 @@ public class TestUtil {
public static void createTransactionalTable(Connection conn, String tableName) throws SQLException {
conn.createStatement().execute("create table " + tableName + TestUtil.TEST_TABLE_SCHEMA + "TRANSACTIONAL=true");
}
+
+ public static void dumpTable(HTableInterface table) throws IOException {
+ System.out.println("************ dumping " + table + " **************");
+ Scan s = new Scan();
+ try (ResultScanner scanner = table.getScanner(s)) {
+ Result result = null;
+ while ((result = scanner.next()) != null) {
+ CellScanner cellScanner = result.cellScanner();
+ Cell current = null;
+ while (cellScanner.advance()) {
+ current = cellScanner.current();
+ System.out.println(current);
+ }
+ }
+ }
+ System.out.println("-----------------------------------------------");
+ }
+
}
[2/3] phoenix git commit: PHOENIX-3361 Collect stats correct for
local indexes
Posted by ja...@apache.org.
PHOENIX-3361 Collect stats correct for local indexes
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/eaf99f23
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/eaf99f23
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/eaf99f23
Branch: refs/heads/master
Commit: eaf99f23da47b86e7577837c9c73ad47ad9ef0d1
Parents: 711d7f0
Author: James Taylor <ja...@apache.org>
Authored: Thu Oct 6 22:27:10 2016 -0700
Committer: James Taylor <ja...@apache.org>
Committed: Fri Oct 7 17:19:15 2016 -0700
----------------------------------------------------------------------
.../apache/phoenix/end2end/GroupByCaseIT.java | 7 +-
.../apache/phoenix/end2end/IndexExtendedIT.java | 7 +-
.../phoenix/end2end/RoundFloorCeilFuncIT.java | 1 -
.../phoenix/end2end/StatsCollectorIT.java | 53 +++--
.../end2end/index/MutableIndexFailureIT.java | 2 -
.../phoenix/end2end/index/MutableIndexIT.java | 1 -
.../phoenix/end2end/index/ViewIndexIT.java | 85 +++----
.../UngroupedAggregateRegionObserver.java | 8 +-
.../phoenix/iterate/BaseResultIterators.java | 84 +++----
.../phoenix/jdbc/PhoenixDatabaseMetaData.java | 6 +-
.../phoenix/query/ConnectionQueryServices.java | 20 +-
.../query/ConnectionQueryServicesImpl.java | 55 +++--
.../query/ConnectionlessQueryServicesImpl.java | 28 +--
.../query/DelegateConnectionQueryServices.java | 12 +-
.../apache/phoenix/query/GuidePostsCache.java | 231 +++++++++++++++++++
.../apache/phoenix/query/QueryConstants.java | 2 +-
.../apache/phoenix/query/TableStatsCache.java | 192 ---------------
.../apache/phoenix/schema/MetaDataClient.java | 49 ++--
.../stats/DefaultStatisticsCollector.java | 44 ++--
.../phoenix/schema/stats/GuidePostsInfo.java | 77 ++++---
.../schema/stats/GuidePostsInfoBuilder.java | 4 +
.../phoenix/schema/stats/GuidePostsKey.java | 84 +++++++
.../schema/stats/NoOpStatisticsCollector.java | 3 +-
.../phoenix/schema/stats/PTableStats.java | 57 -----
.../phoenix/schema/stats/PTableStatsImpl.java | 115 ---------
.../schema/stats/StatisticsCollector.java | 5 +-
.../phoenix/schema/stats/StatisticsUtil.java | 96 ++++----
.../phoenix/schema/stats/StatisticsWriter.java | 6 +-
.../org/apache/phoenix/util/MetaDataUtil.java | 4 +
.../phoenix/filter/SkipScanBigFilterTest.java | 25 +-
.../PhoenixStatsCacheRemovalListenerTest.java | 2 +-
.../java/org/apache/phoenix/util/TestUtil.java | 28 ++-
32 files changed, 697 insertions(+), 696 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java
index 56f4503..1b32cfc 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java
@@ -33,6 +33,7 @@ import java.sql.Statement;
import java.util.List;
import java.util.Properties;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.query.KeyRange;
import org.apache.phoenix.schema.AmbiguousColumnException;
import org.apache.phoenix.schema.types.PChar;
@@ -409,11 +410,11 @@ public class GroupByCaseIT extends ParallelStatsDisabledIT {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
Connection conn = DriverManager.getConnection(getUrl(), props);
String tableName = generateUniqueName();
- initAvgGroupTable(conn, tableName, " GUIDE_POST_WIDTH=20 ");
+ initAvgGroupTable(conn, tableName, PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + "=20 ");
testAvgGroupByOrderPreserving(conn, tableName, 13);
- conn.createStatement().execute("ALTER TABLE " + tableName + " SET GUIDE_POST_WIDTH=" + 100);
+ conn.createStatement().execute("ALTER TABLE " + tableName + " SET " + PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + "=100");
testAvgGroupByOrderPreserving(conn, tableName, 6);
- conn.createStatement().execute("ALTER TABLE " + tableName + " SET GUIDE_POST_WIDTH=null");
+ conn.createStatement().execute("ALTER TABLE " + tableName + " SET " + PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + "=null");
testAvgGroupByOrderPreserving(conn, tableName, 4);
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexExtendedIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexExtendedIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexExtendedIT.java
index cfbc9eb..93d7cec 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexExtendedIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexExtendedIT.java
@@ -92,10 +92,11 @@ public class IndexExtendedIT extends BaseOwnClusterIT {
@BeforeClass
public static void doSetup() throws Exception {
- Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(1);
+ Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(2);
serverProps.put(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB, QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS);
- Map<String, String> clientProps = Maps.newHashMapWithExpectedSize(1);
+ Map<String, String> clientProps = Maps.newHashMapWithExpectedSize(2);
clientProps.put(QueryServices.TRANSACTIONS_ENABLED, Boolean.TRUE.toString());
+ clientProps.put(QueryServices.FORCE_ROW_KEY_ORDER_ATTRIB, Boolean.TRUE.toString());
setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), new ReadOnlyProps(clientProps.entrySet()
.iterator()));
}
@@ -442,7 +443,7 @@ public class IndexExtendedIT extends BaseOwnClusterIT {
rs = conn1.createStatement().executeQuery(query);
Thread.sleep(1000);
for (int j = 0; j < 26; j++) {
- assertTrue(rs.next());
+ assertTrue("No row found at " + j, rs.next());
tIdColumnValues[j] = rs.getString("t_id");
k1ColumnValue[j] = rs.getInt("k1");
v1ColumnValues[j] = rs.getString("V1");
http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/it/java/org/apache/phoenix/end2end/RoundFloorCeilFuncIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/RoundFloorCeilFuncIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/RoundFloorCeilFuncIT.java
index c247bc9..9961199 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/RoundFloorCeilFuncIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/RoundFloorCeilFuncIT.java
@@ -175,7 +175,6 @@ public class RoundFloorCeilFuncIT extends ParallelStatsDisabledIT {
expectedDate = DateUtil.parseDate("2012-01-01 14:25:29");
assertEquals(expectedDate, rs.getDate(4));
expectedDate = DateUtil.parseDate("2012-01-02 00:00:00");
- System.out.println(String.format(" the expected time is [%s] and the actual time is [%s]",expectedDate.getTime(),rs.getDate(5).getTime()));
assertEquals(expectedDate, rs.getDate(5));
expectedDate = DateUtil.parseDate("2012-02-01 00:00:00");
assertEquals(expectedDate, rs.getDate(6));
http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
index 2445948..6193cad 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
@@ -40,13 +40,15 @@ import java.util.Random;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.query.ConnectionQueryServices;
import org.apache.phoenix.query.KeyRange;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTableKey;
+import org.apache.phoenix.schema.stats.GuidePostsKey;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.SchemaUtil;
@@ -69,10 +71,14 @@ public class StatsCollectorIT extends ParallelStatsEnabledIT {
}
private static Connection getConnection() throws SQLException {
+ return getConnection(Integer.MAX_VALUE);
+ }
+
+ private static Connection getConnection(Integer statsUpdateFreq) throws SQLException {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
props.setProperty(QueryServices.EXPLAIN_CHUNK_COUNT_ATTRIB, Boolean.TRUE.toString());
props.setProperty(QueryServices.EXPLAIN_ROW_COUNT_ATTRIB, Boolean.TRUE.toString());
- props.setProperty(QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB, Long.toString(Long.MAX_VALUE));
+ props.setProperty(QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB, Integer.toString(statsUpdateFreq));
return DriverManager.getConnection(getUrl(), props);
}
@@ -334,7 +340,7 @@ public class StatsCollectorIT extends ParallelStatsEnabledIT {
@Test
public void testCompactUpdatesStats() throws Exception {
- testCompactUpdatesStats(null, fullTableName);
+ testCompactUpdatesStats(0, fullTableName);
}
@Test
@@ -342,9 +348,17 @@ public class StatsCollectorIT extends ParallelStatsEnabledIT {
testCompactUpdatesStats(QueryServicesOptions.DEFAULT_STATS_UPDATE_FREQ_MS, fullTableName);
}
- private void testCompactUpdatesStats(Integer minStatsUpdateFreq, String tableName) throws Exception {
+ private static void invalidateStats(Connection conn, String tableName) throws SQLException {
+ PTable ptable = conn.unwrap(PhoenixConnection.class)
+ .getMetaDataCache().getTableRef(new PTableKey(null, tableName))
+ .getTable();
+ byte[] name = ptable.getPhysicalName().getBytes();
+ conn.unwrap(PhoenixConnection.class).getQueryServices().invalidateStats(new GuidePostsKey(name, SchemaUtil.getEmptyColumnFamily(ptable)));
+ }
+
+ private void testCompactUpdatesStats(Integer statsUpdateFreq, String tableName) throws Exception {
int nRows = 10;
- Connection conn = getConnection();
+ Connection conn = getConnection(statsUpdateFreq);
PreparedStatement stmt;
conn.createStatement().execute("CREATE TABLE " + tableName + "(k CHAR(1) PRIMARY KEY, v INTEGER, w INTEGER) "
+ HColumnDescriptor.KEEP_DELETED_CELLS + "=" + Boolean.FALSE);
@@ -358,9 +372,8 @@ public class StatsCollectorIT extends ParallelStatsEnabledIT {
conn.commit();
compactTable(conn, tableName);
- if (minStatsUpdateFreq == null) {
- ImmutableBytesPtr ptr = new ImmutableBytesPtr(Bytes.toBytes(tableName));
- conn.unwrap(PhoenixConnection.class).getQueryServices().invalidateStats(ptr);
+ if (statsUpdateFreq == null) {
+ invalidateStats(conn, tableName);
} else {
// Confirm that when we have a non zero MIN_STATS_UPDATE_FREQ_MS_ATTRIB, after we run
// UPDATATE STATISTICS, the new statistics are faulted in as expected.
@@ -379,13 +392,12 @@ public class StatsCollectorIT extends ParallelStatsEnabledIT {
assertEquals(5, nDeletedRows);
compactTable(conn, tableName);
- if (minStatsUpdateFreq == null) {
- ImmutableBytesPtr ptr = new ImmutableBytesPtr(Bytes.toBytes(tableName));
- conn.unwrap(PhoenixConnection.class).getQueryServices().invalidateStats(ptr);
+ if (statsUpdateFreq == null) {
+ invalidateStats(conn, tableName);
}
keyRanges = getAllSplits(conn, tableName);
- if (minStatsUpdateFreq != null) {
+ if (statsUpdateFreq != null) {
assertEquals(nRows+1, keyRanges.size());
// If we've set MIN_STATS_UPDATE_FREQ_MS_ATTRIB, an UPDATE STATISTICS will invalidate the cache
// and force us to pull over the new stats
@@ -403,7 +415,7 @@ public class StatsCollectorIT extends ParallelStatsEnabledIT {
@Test
public void testWithMultiCF() throws Exception {
int nRows = 20;
- Connection conn = getConnection();
+ Connection conn = getConnection(0);
PreparedStatement stmt;
conn.createStatement().execute(
"CREATE TABLE " + fullTableName
@@ -478,6 +490,20 @@ public class StatsCollectorIT extends ParallelStatsEnabledIT {
assertEquals(6, rs.getInt(4));
assertFalse(rs.next());
+
+ // Disable stats
+ conn.createStatement().execute("ALTER TABLE " + fullTableName +
+ " SET " + PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + "=0");
+ TestUtil.analyzeTable(conn, fullTableName);
+ // Assert that there are no more guideposts
+ rs = conn.createStatement().executeQuery("SELECT count(1) FROM " + PhoenixDatabaseMetaData.SYSTEM_STATS_NAME +
+ " WHERE " + PhoenixDatabaseMetaData.PHYSICAL_NAME + "='" + fullTableName + "' AND " + PhoenixDatabaseMetaData.COLUMN_FAMILY + " IS NOT NULL");
+ assertTrue(rs.next());
+ assertEquals(0, rs.getLong(1));
+ assertFalse(rs.next());
+ rs = conn.createStatement().executeQuery("EXPLAIN SELECT * FROM " + fullTableName);
+ assertEquals("CLIENT 1-CHUNK PARALLEL 1-WAY FULL SCAN OVER " + fullTableName,
+ QueryUtil.getExplainPlan(rs));
}
@Test
@@ -506,7 +532,6 @@ public class StatsCollectorIT extends ParallelStatsEnabledIT {
int endIndex = r.nextInt(strings.length - startIndex) + startIndex;
long rows = endIndex - startIndex;
long c2Bytes = rows * 35;
- System.out.println(rows + ":" + startIndex + ":" + endIndex);
rs = conn.createStatement().executeQuery(
"SELECT COLUMN_FAMILY,SUM(GUIDE_POSTS_ROW_COUNT),SUM(GUIDE_POSTS_WIDTH) from SYSTEM.STATS where PHYSICAL_NAME = '"
+ fullTableName + "' AND GUIDE_POST_KEY>= cast('" + strings[startIndex]
http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
index d740013..d6c1e9c 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
@@ -377,9 +377,7 @@ public class MutableIndexFailureIT extends BaseOwnClusterIT {
conn.commit();
fail();
} catch (SQLException e) {
- System.out.println();
} catch (Exception e) {
- System.out.println();
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java
index 1bc01d2..38c9f43 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java
@@ -706,7 +706,6 @@ public class MutableIndexIT extends ParallelStatsDisabledIT {
while (true) {
rs = conn1.createStatement().executeQuery(query);
assertTrue(rs.next());
- System.out.println("Number of rows returned:" + rs.getInt(1));
assertEquals(4, rs.getInt(1)); //TODO this returns 5 sometimes instead of 4, duplicate results?
try {
List<HRegionInfo> indexRegions = admin.getTableRegions(indexTable);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java
index b714a11..9e63093 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java
@@ -31,11 +31,14 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Arrays;
import java.util.Collection;
+import java.util.List;
import java.util.Properties;
import org.apache.phoenix.compile.QueryPlan;
import org.apache.phoenix.end2end.ParallelStatsDisabledIT;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.jdbc.PhoenixStatement;
+import org.apache.phoenix.query.KeyRange;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.schema.PNameFactory;
import org.apache.phoenix.schema.PTableType;
@@ -145,19 +148,19 @@ public class ViewIndexIT extends ParallelStatsDisabledIT {
stmt.setString(4, "x1");
stmt.setInt(5, 100);
stmt.execute();
- stmt.setString(1, "20");
+ stmt.setString(1, "10");
stmt.setString(2, "b");
stmt.setInt(3, 2);
stmt.setString(4, "x2");
stmt.setInt(5, 200);
stmt.execute();
- stmt.setString(1, "30");
+ stmt.setString(1, "10");
stmt.setString(2, "c");
stmt.setInt(3, 3);
stmt.setString(4, "x3");
stmt.setInt(5, 300);
stmt.execute();
- stmt.setString(1, "40");
+ stmt.setString(1, "20");
stmt.setString(2, "d");
stmt.setInt(3, 4);
stmt.setString(4, "x4");
@@ -185,24 +188,24 @@ public class ViewIndexIT extends ParallelStatsDisabledIT {
assertTrue(rs.next());
assertFalse(rs.next());
-// TestUtil.analyzeTable(conn, fullTableName);
-// List<KeyRange> guideposts = TestUtil.getAllSplits(conn, fullTableName);
-// assertEquals(1, guideposts.size());
-// assertEquals(KeyRange.EVERYTHING_RANGE, guideposts.get(0));
-//
-// conn.createStatement().execute("ALTER TABLE " + fullTableName + " SET GUIDE_POST_WIDTH=20");
-//
-// TestUtil.analyzeTable(conn, fullTableName);
-// guideposts = TestUtil.getAllSplits(conn, fullTableName);
-// assertEquals(5, guideposts.size());
-//
-// // Confirm that when view index used, the GUIDE_POST_WIDTH from the data physical table
-// // was used
-// sql = "SELECT * FROM " + viewName + " WHERE v2 > 100";
-// stmt = conn1.prepareStatement(sql);
-// stmt.executeQuery();
-// QueryPlan plan = stmt.unwrap(PhoenixStatement.class).getQueryPlan();
-// assertEquals(5, plan.getSplits().size());
+ TestUtil.analyzeTable(conn, fullTableName);
+ List<KeyRange> guideposts = TestUtil.getAllSplits(conn, fullTableName);
+ assertEquals(1, guideposts.size());
+ assertEquals(KeyRange.EVERYTHING_RANGE, guideposts.get(0));
+
+ conn.createStatement().execute("ALTER TABLE " + fullTableName + " SET " + PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + "=20");
+
+ TestUtil.analyzeTable(conn, fullTableName);
+ guideposts = TestUtil.getAllSplits(conn, fullTableName);
+ assertEquals(5, guideposts.size());
+
+ // Confirm that when view index used, the GUIDE_POSTS_WIDTH from the data physical table
+ // was used
+ sql = "SELECT * FROM " + viewName + " WHERE v2 >= 100";
+ stmt = conn1.prepareStatement(sql);
+ stmt.executeQuery();
+ QueryPlan plan = stmt.unwrap(PhoenixStatement.class).getQueryPlan();
+ assertEquals(4, plan.getSplits().size());
}
@@ -259,26 +262,26 @@ public class ViewIndexIT extends ParallelStatsDisabledIT {
assertEquals("KV1", rs.getString(1));
assertFalse(rs.next());
-// TestUtil.analyzeTable(conn, baseTable);
-// List<KeyRange> guideposts = TestUtil.getAllSplits(conn, baseTable);
-// assertEquals(1, guideposts.size());
-// assertEquals(KeyRange.EVERYTHING_RANGE, guideposts.get(0));
-//
-// conn.createStatement().execute("ALTER TABLE " + baseTable + " SET GUIDE_POST_WIDTH=20");
-//
-// TestUtil.analyzeTable(conn, baseTable);
-// guideposts = TestUtil.getAllSplits(conn, baseTable);
-// assertEquals(6, guideposts.size());
-//
-// // Confirm that when view index used, the GUIDE_POST_WIDTH from the data physical table
-// // was used
-// stmt = conn.prepareStatement("SELECT KV1 FROM " + globalView + " WHERE PK3 = ? AND KV3 >= ?");
-// stmt.setInt(1, 1);
-// stmt.setString(2, "KV3");
-// rs = stmt.executeQuery();
-// plan = stmt.unwrap(PhoenixStatement.class).getQueryPlan();
-// assertTrue(plan.getTableRef().getTable().getName().getString().equals(globalViewIdx));
-// assertEquals(6, plan.getSplits().size());
+ TestUtil.analyzeTable(conn, baseTable);
+ List<KeyRange> guideposts = TestUtil.getAllSplits(conn, baseTable);
+ assertEquals(1, guideposts.size());
+ assertEquals(KeyRange.EVERYTHING_RANGE, guideposts.get(0));
+
+ conn.createStatement().execute("ALTER TABLE " + baseTable + " SET " + PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + "=20");
+
+ TestUtil.analyzeTable(conn, baseTable);
+ guideposts = TestUtil.getAllSplits(conn, baseTable);
+ assertEquals(6, guideposts.size());
+
+ // Confirm that when view index used, the GUIDE_POSTS_WIDTH from the data physical table
+ // was used
+ stmt = conn.prepareStatement("SELECT KV1 FROM " + globalView + " WHERE PK3 = ? AND KV3 >= ?");
+ stmt.setInt(1, 1);
+ stmt.setString(2, "KV3");
+ rs = stmt.executeQuery();
+ plan = stmt.unwrap(PhoenixStatement.class).getQueryPlan();
+ assertTrue(plan.getTableRef().getTable().getName().getString().equals(globalViewIdx));
+ assertEquals(6, plan.getSplits().size());
}
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
index 2a4bfca..d8d313d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
@@ -718,7 +718,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
private RegionScanner collectStats(final RegionScanner innerScanner, StatisticsCollector stats,
final Region region, final Scan scan, Configuration config) throws IOException {
StatsCollectionCallable callable =
- new StatsCollectionCallable(stats, region, innerScanner, config);
+ new StatsCollectionCallable(stats, region, innerScanner, config, scan);
byte[] asyncBytes = scan.getAttribute(BaseScannerRegionObserver.RUN_UPDATE_STATS_ASYNC_ATTRIB);
boolean async = false;
if (asyncBytes != null) {
@@ -785,13 +785,15 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
private final Region region;
private final RegionScanner innerScanner;
private final Configuration config;
+ private final Scan scan;
StatsCollectionCallable(StatisticsCollector s, Region r, RegionScanner rs,
- Configuration config) {
+ Configuration config, Scan scan) {
this.stats = s;
this.region = r;
this.innerScanner = rs;
this.config = config;
+ this.scan = scan;
}
@Override
@@ -832,7 +834,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
} finally {
try {
if (noErrors && !compactionRunning) {
- stats.updateStatistic(region);
+ stats.updateStatistic(region, scan);
logger.info("UPDATE STATISTICS finished successfully for scanner: "
+ innerScanner + ". Number of rows scanned: " + rowCount
+ ". Time: " + (System.currentTimeMillis() - startTime));
http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
index 2685b93..d4c8bef 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
@@ -86,7 +86,7 @@ import org.apache.phoenix.schema.PTable.ViewType;
import org.apache.phoenix.schema.StaleRegionBoundaryCacheException;
import org.apache.phoenix.schema.TableRef;
import org.apache.phoenix.schema.stats.GuidePostsInfo;
-import org.apache.phoenix.schema.stats.PTableStats;
+import org.apache.phoenix.schema.stats.GuidePostsKey;
import org.apache.phoenix.schema.stats.StatisticsUtil;
import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.Closeables;
@@ -120,7 +120,6 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
private final List<List<Scan>> scans;
private final List<KeyRange> splits;
- private final PTableStats tableStats;
private final byte[] physicalTableName;
protected final QueryPlan plan;
protected final String scanId;
@@ -363,9 +362,6 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
if (null == currentSCN) {
currentSCN = HConstants.LATEST_TIMESTAMP;
}
- tableStats = useStats() && StatisticsUtil.isStatsEnabled(TableName.valueOf(physicalTableName))
- ? context.getConnection().getQueryServices().getTableStats(physicalTableName, currentSCN)
- : PTableStats.EMPTY_STATS;
// Used to tie all the scans together during logging
scanId = new UUID(ThreadLocalRandom.current().nextLong(), ThreadLocalRandom.current().nextLong()).toString();
@@ -425,55 +421,40 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
return guideIndex;
}
- private GuidePostsInfo getGuidePosts(Set<byte[]> whereConditions) {
- if (!useStats()) { return GuidePostsInfo.NO_GUIDEPOST; }
+ private GuidePostsInfo getGuidePosts() throws SQLException {
+ if (!useStats() || !StatisticsUtil.isStatsEnabled(TableName.valueOf(physicalTableName))) {
+ return GuidePostsInfo.NO_GUIDEPOST;
+ }
- GuidePostsInfo gps = null;
+ TreeSet<byte[]> whereConditions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
+ for(Pair<byte[], byte[]> where : context.getWhereConditionColumns()) {
+ byte[] cf = where.getFirst();
+ if (cf != null) {
+ whereConditions.add(cf);
+ }
+ }
PTable table = getTable();
- Map<byte[], GuidePostsInfo> guidePostMap = tableStats.getGuidePosts();
byte[] defaultCF = SchemaUtil.getEmptyColumnFamily(getTable());
- if (table.getColumnFamilies().isEmpty()) {
- // For sure we can get the defaultCF from the table
- gps = getDefaultFamilyGuidePosts(guidePostMap, defaultCF);
- } else {
- if (whereConditions.isEmpty() || whereConditions.contains(defaultCF)) {
- gps = getDefaultFamilyGuidePosts(guidePostMap, defaultCF);
- } else {
- byte[] familyInWhere = whereConditions.iterator().next();
- GuidePostsInfo guidePostsInfo = guidePostMap.get(familyInWhere);
- if (guidePostsInfo != null) {
- gps = guidePostsInfo;
- } else {
- // As there are no guideposts collected for the where family we go with the default CF
- gps = getDefaultFamilyGuidePosts(guidePostMap, defaultCF);
+ byte[] cf = null;
+ if ( !table.getColumnFamilies().isEmpty() && !whereConditions.isEmpty() ) {
+ for(Pair<byte[], byte[]> where : context.getWhereConditionColumns()) {
+ byte[] whereCF = where.getFirst();
+ if (Bytes.compareTo(defaultCF, whereCF) == 0) {
+ cf = defaultCF;
+ break;
}
}
+ if (cf == null) {
+ cf = context.getWhereConditionColumns().get(0).getFirst();
+ }
}
- if (gps == null) { return GuidePostsInfo.NO_GUIDEPOST; }
- return gps;
- }
-
- private GuidePostsInfo getDefaultFamilyGuidePosts(Map<byte[], GuidePostsInfo> guidePostMap, byte[] defaultCF) {
- if (guidePostMap.get(defaultCF) != null) {
- return guidePostMap.get(defaultCF);
+ if (cf == null) {
+ cf = defaultCF;
}
- return null;
+ GuidePostsKey key = new GuidePostsKey(physicalTableName, cf);
+ return context.getConnection().getQueryServices().getTableStats(key);
}
- private static String toString(List<byte[]> gps) {
- StringBuilder buf = new StringBuilder(gps.size() * 100);
- buf.append("[");
- for (int i = 0; i < gps.size(); i++) {
- buf.append(Bytes.toStringBinary(gps.get(i)));
- buf.append(",");
- if (i > 0 && i < gps.size()-1 && (i % 10) == 0) {
- buf.append("\n");
- }
- }
- buf.setCharAt(buf.length()-1, ']');
- return buf.toString();
- }
-
private List<Scan> addNewScan(List<List<Scan>> parallelScans, List<Scan> scans, Scan scan, byte[] startKey, boolean crossedRegionBoundary, HRegionLocation regionLocation) {
boolean startNewScan = scanGrouper.shouldStartNewScan(plan, scans, startKey, crossedRegionBoundary);
if (scan != null) {
@@ -565,14 +546,7 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
PTable table = getTable();
boolean isSalted = table.getBucketNum() != null;
boolean isLocalIndex = table.getIndexType() == IndexType.LOCAL;
- TreeSet<byte[]> whereConditions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
- for(Pair<byte[], byte[]> where : context.getWhereConditionColumns()) {
- byte[] cf = where.getFirst();
- if (cf != null) {
- whereConditions.add(cf);
- }
- }
- GuidePostsInfo gps = getGuidePosts(whereConditions);
+ GuidePostsInfo gps = getGuidePosts();
hasGuidePosts = gps != GuidePostsInfo.NO_GUIDEPOST;
boolean traverseAllRegions = isSalted || isLocalIndex;
if (!traverseAllRegions) {
@@ -649,8 +623,8 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
if (newScan != null) {
ScanUtil.setLocalIndexAttributes(newScan, keyOffset, regionInfo.getStartKey(),
regionInfo.getEndKey(), newScan.getStartRow(), newScan.getStopRow());
- estimatedRows += gps.getRowCounts().get(guideIndex);
- estimatedSize += gps.getByteCounts().get(guideIndex);
+ estimatedRows += gps.getRowCounts()[guideIndex];
+ estimatedSize += gps.getByteCounts()[guideIndex];
}
scans = addNewScan(parallelScans, scans, newScan, currentGuidePostBytes, false, regionLocation);
currentKeyBytes = currentGuidePostBytes;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
index 47dfd4e..35b754f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
@@ -1087,7 +1087,8 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData {
SQLViewTypeFunction.NAME + "(" + VIEW_TYPE + ") AS " + VIEW_TYPE + "," +
SQLIndexTypeFunction.NAME + "(" + INDEX_TYPE + ") AS " + INDEX_TYPE + "," +
TRANSACTIONAL + "," +
- IS_NAMESPACE_MAPPED +
+ IS_NAMESPACE_MAPPED + "," +
+ GUIDE_POSTS_WIDTH +
" from " + SYSTEM_CATALOG + " " + SYSTEM_CATALOG_ALIAS +
" where " + COLUMN_NAME + " is null" +
" and " + COLUMN_FAMILY + " is null" +
@@ -1126,7 +1127,8 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData {
"'' " + VIEW_TYPE + "," +
"'' " + INDEX_TYPE + "," +
"CAST(null AS BOOLEAN) " + TRANSACTIONAL + "," +
- "CAST(null AS BOOLEAN) " + IS_NAMESPACE_MAPPED + "\n");
+ "CAST(null AS BOOLEAN) " + IS_NAMESPACE_MAPPED + "," +
+ "CAST(null AS BIGINT) " + GUIDE_POSTS_WIDTH + "\n");
buf.append(
" from " + SYSTEM_SEQUENCE + "\n");
StringBuilder whereClause = new StringBuilder();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
index bc2c93b..0478e07 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
@@ -33,7 +33,6 @@ import org.apache.hadoop.hbase.util.Pair;
import org.apache.phoenix.compile.MutationPlan;
import org.apache.phoenix.coprocessor.MetaDataProtocol.MetaDataMutationResult;
import org.apache.phoenix.execute.MutationState;
-import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.hbase.index.util.KeyValueBuilder;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.parse.PFunction;
@@ -43,7 +42,8 @@ import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.schema.Sequence;
import org.apache.phoenix.schema.SequenceAllocation;
import org.apache.phoenix.schema.SequenceKey;
-import org.apache.phoenix.schema.stats.PTableStats;
+import org.apache.phoenix.schema.stats.GuidePostsInfo;
+import org.apache.phoenix.schema.stats.GuidePostsKey;
import org.apache.tephra.TransactionSystemClient;
@@ -118,7 +118,14 @@ public interface ConnectionQueryServices extends QueryServices, MetaDataMutated
public String getUserName();
public void clearTableFromCache(final byte[] tenantId, final byte[] schemaName, final byte[] tableName, long clientTS) throws SQLException;
- public PTableStats getTableStats(byte[] physicalName, long clientTimeStamp) throws SQLException;
+ public GuidePostsInfo getTableStats(GuidePostsKey key) throws SQLException;
+ /**
+ * Removes cache {@link GuidePostsInfo} for the table with the given name. If no cached guideposts are present, this does nothing.
+ *
+ * @param tableName The table to remove stats for
+ */
+ void invalidateStats(GuidePostsKey key);
+
public long clearCache() throws SQLException;
public int getSequenceSaltBuckets();
@@ -133,13 +140,6 @@ public interface ConnectionQueryServices extends QueryServices, MetaDataMutated
public MetaDataMutationResult dropSchema(List<Mutation> schemaMetaData, String schemaName) throws SQLException;
- /**
- * Removes cache {@link PTableStats} for the table with the given name. If no cached stats are present, this does nothing.
- *
- * @param tableName The table to remove stats for
- */
- void invalidateStats(ImmutableBytesPtr tableName);
-
boolean isUpgradeRequired();
void upgradeSystemTables(String url, Properties props) throws SQLException;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index 4982578..4770d22 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -140,7 +140,6 @@ import org.apache.phoenix.execute.MutationState;
import org.apache.phoenix.hbase.index.IndexRegionSplitPolicy;
import org.apache.phoenix.hbase.index.Indexer;
import org.apache.phoenix.hbase.index.covered.NonTxIndexBuilder;
-import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.hbase.index.util.KeyValueBuilder;
import org.apache.phoenix.hbase.index.util.VersionUtil;
import org.apache.phoenix.index.PhoenixIndexBuilder;
@@ -179,7 +178,8 @@ import org.apache.phoenix.schema.SequenceKey;
import org.apache.phoenix.schema.TableAlreadyExistsException;
import org.apache.phoenix.schema.TableNotFoundException;
import org.apache.phoenix.schema.TableProperty;
-import org.apache.phoenix.schema.stats.PTableStats;
+import org.apache.phoenix.schema.stats.GuidePostsInfo;
+import org.apache.phoenix.schema.stats.GuidePostsKey;
import org.apache.phoenix.schema.types.PBoolean;
import org.apache.phoenix.schema.types.PDataType;
import org.apache.phoenix.schema.types.PInteger;
@@ -235,7 +235,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
private final ReadOnlyProps props;
private final String userName;
private final ConcurrentHashMap<ImmutableBytesWritable,ConnectionQueryServices> childServices;
- private final TableStatsCache tableStatsCache;
+ private final GuidePostsCache tableStatsCache;
// Cache the latest meta data here for future connections
// writes guarded by "latestMetaDataLock"
@@ -348,7 +348,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
connectionQueues = ImmutableList.copyOf(list);
// A little bit of a smell to leak `this` here, but should not be a problem
- this.tableStatsCache = new TableStatsCache(this, config);
+ this.tableStatsCache = new GuidePostsCache(this, config);
this.isAutoUpgradeEnabled = config.getBoolean(AUTO_UPGRADE_ENABLED, QueryServicesOptions.DEFAULT_AUTO_UPGRADE_ENABLED);
}
@@ -1317,13 +1317,11 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
private boolean ensureViewIndexTableDropped(byte[] physicalTableName, long timestamp) throws SQLException {
byte[] physicalIndexName = MetaDataUtil.getViewIndexPhysicalName(physicalTableName);
- HTableDescriptor desc = null;
boolean wasDeleted = false;
try (HBaseAdmin admin = getAdmin()) {
try {
- desc = admin.getTableDescriptor(physicalIndexName);
+ HTableDescriptor desc = admin.getTableDescriptor(physicalIndexName);
if (Boolean.TRUE.equals(PBoolean.INSTANCE.toObject(desc.getValue(MetaDataUtil.IS_VIEW_INDEX_TABLE_PROP_BYTES)))) {
- this.tableStatsCache.invalidate(new ImmutableBytesPtr(physicalIndexName));
final ReadOnlyProps props = this.getProps();
final boolean dropMetadata = props.getBoolean(DROP_METADATA_ATTRIB, DEFAULT_DROP_METADATA);
if (dropMetadata) {
@@ -1331,6 +1329,8 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
admin.deleteTable(physicalIndexName);
clearTableRegionCache(physicalIndexName);
wasDeleted = true;
+ } else {
+ this.tableStatsCache.invalidateAll(desc);
}
}
} catch (org.apache.hadoop.hbase.TableNotFoundException ignore) {
@@ -1348,7 +1348,9 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
try (HBaseAdmin admin = getAdmin()) {
try {
desc = admin.getTableDescriptor(physicalTableName);
- this.tableStatsCache.invalidate(new ImmutableBytesPtr(physicalTableName));
+ for (byte[] fam : desc.getFamiliesKeys()) {
+ this.tableStatsCache.invalidate(new GuidePostsKey(physicalTableName, fam));
+ }
final ReadOnlyProps props = this.getProps();
final boolean dropMetadata = props.getBoolean(DROP_METADATA_ATTRIB, DEFAULT_DROP_METADATA);
if (dropMetadata) {
@@ -1530,14 +1532,15 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
if (dropMetadata) {
flushParentPhysicalTable(table);
dropTables(result.getTableNamesToDelete());
+ } else {
+ invalidateTableStats(result.getTableNamesToDelete());
}
- invalidateTables(result.getTableNamesToDelete());
long timestamp = MetaDataUtil.getClientTimeStamp(tableMetaData);
if (tableType == PTableType.TABLE) {
byte[] physicalName = table.getPhysicalName().getBytes();
ensureViewIndexTableDropped(physicalName, timestamp);
ensureLocalIndexTableDropped(physicalName, timestamp);
- tableStatsCache.invalidate(new ImmutableBytesPtr(physicalName));
+ tableStatsCache.invalidateAll(table);
}
break;
default:
@@ -1598,24 +1601,30 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
return result;
}
- private void invalidateTables(final List<byte[]> tableNamesToDelete) {
+ private void invalidateTableStats(final List<byte[]> tableNamesToDelete) {
if (tableNamesToDelete != null) {
for (byte[] tableName : tableNamesToDelete) {
- tableStatsCache.invalidate(new ImmutableBytesPtr(Bytes.toString(tableName)
- .replace(QueryConstants.NAMESPACE_SEPARATOR, QueryConstants.NAME_SEPARATOR).getBytes()));
+ tableStatsCache.invalidateAll(tableName);
}
}
}
+ private void dropTable(byte[] tableNameToDelete) throws SQLException {
+ dropTables(Collections.<byte[]>singletonList(tableNameToDelete));
+ }
+
private void dropTables(final List<byte[]> tableNamesToDelete) throws SQLException {
SQLException sqlE = null;
try (HBaseAdmin admin = getAdmin()) {
if (tableNamesToDelete != null){
for ( byte[] tableName : tableNamesToDelete ) {
- if ( admin.tableExists(tableName) ) {
+ try {
+ HTableDescriptor htableDesc = this.getTableDescriptor(tableName);
admin.disableTable(tableName);
admin.deleteTable(tableName);
+ tableStatsCache.invalidateAll(htableDesc);
clearTableRegionCache(tableName);
+ } catch (TableNotFoundException ignore) {
}
}
}
@@ -2248,8 +2257,9 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
final boolean dropMetadata = props.getBoolean(DROP_METADATA_ATTRIB, DEFAULT_DROP_METADATA);
if (dropMetadata) {
dropTables(result.getTableNamesToDelete());
+ } else {
+ invalidateTableStats(result.getTableNamesToDelete());
}
- invalidateTables(result.getTableNamesToDelete());
break;
default:
break;
@@ -3478,7 +3488,6 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
sqlE = new SQLException(e);
} finally {
try {
- if (tenantId.length == 0) tableStatsCache.invalidate(new ImmutableBytesPtr(SchemaUtil.getTableNameAsBytes(schemaName, tableName)));
htable.close();
} catch (IOException e) {
if (sqlE == null) {
@@ -3680,9 +3689,9 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
@Override
- public PTableStats getTableStats(final byte[] physicalName, final long clientTimeStamp) throws SQLException {
+ public GuidePostsInfo getTableStats(GuidePostsKey key) throws SQLException {
try {
- return tableStatsCache.get(new ImmutableBytesPtr(physicalName));
+ return tableStatsCache.get(key);
} catch (ExecutionException e) {
throw ServerUtil.parseServerException(e);
}
@@ -4042,19 +4051,19 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
/**
- * Manually adds {@link PTableStats} for a table to the client-side cache. Not a
+ * Manually adds {@link GuidePostsInfo} for a table to the client-side cache. Not a
* {@link ConnectionQueryServices} method. Exposed for testing purposes.
*
* @param tableName Table name
* @param stats Stats instance
*/
- public void addTableStats(ImmutableBytesPtr tableName, PTableStats stats) {
- this.tableStatsCache.put(Objects.requireNonNull(tableName), stats);
+ public void addTableStats(GuidePostsKey key, GuidePostsInfo info) {
+ this.tableStatsCache.put(Objects.requireNonNull(key), Objects.requireNonNull(info));
}
@Override
- public void invalidateStats(ImmutableBytesPtr tableName) {
- this.tableStatsCache.invalidate(Objects.requireNonNull(tableName));
+ public void invalidateStats(GuidePostsKey key) {
+ this.tableStatsCache.invalidate(Objects.requireNonNull(key));
}
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
index 8c9f3ff..1b1e429 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
@@ -51,7 +51,6 @@ import org.apache.phoenix.exception.SQLExceptionCode;
import org.apache.phoenix.exception.SQLExceptionInfo;
import org.apache.phoenix.execute.MutationState;
import org.apache.phoenix.hbase.index.util.GenericKeyValueBuilder;
-import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.hbase.index.util.KeyValueBuilder;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
@@ -79,7 +78,8 @@ import org.apache.phoenix.schema.SequenceKey;
import org.apache.phoenix.schema.SequenceNotFoundException;
import org.apache.phoenix.schema.TableAlreadyExistsException;
import org.apache.phoenix.schema.TableNotFoundException;
-import org.apache.phoenix.schema.stats.PTableStats;
+import org.apache.phoenix.schema.stats.GuidePostsInfo;
+import org.apache.phoenix.schema.stats.GuidePostsKey;
import org.apache.phoenix.util.JDBCUtil;
import org.apache.phoenix.util.MetaDataUtil;
import org.apache.phoenix.util.PhoenixRuntime;
@@ -113,7 +113,7 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
private volatile boolean initialized;
private volatile SQLException initializationException;
private final Map<String, List<HRegionLocation>> tableSplits = Maps.newHashMap();
- private final TableStatsCache tableStatsCache;
+ private final GuidePostsCache guidePostsCache;
public ConnectionlessQueryServicesImpl(QueryServices services, ConnectionInfo connInfo, Properties info) {
super(services);
@@ -140,7 +140,7 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
config = HBaseFactoryProvider.getConfigurationFactory().getConfiguration(config);
TransactionManager txnManager = new TransactionManager(config);
this.txSystemClient = new InMemoryTxSystemClient(txnManager);
- this.tableStatsCache = new TableStatsCache(this, config);
+ this.guidePostsCache = new GuidePostsCache(this, config);
}
private PMetaData newEmptyMetaData() {
@@ -520,12 +520,12 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
}
@Override
- public PTableStats getTableStats(byte[] physicalName, long clientTimeStamp) {
- PTableStats stats = tableStatsCache.getCache().getIfPresent(physicalName);
- if (null == stats) {
- return PTableStats.EMPTY_STATS;
+ public GuidePostsInfo getTableStats(GuidePostsKey key) {
+ GuidePostsInfo info = guidePostsCache.getCache().getIfPresent(key);
+ if (null == info) {
+ return GuidePostsInfo.NO_GUIDEPOST;
}
- return stats;
+ return info;
}
@Override
@@ -640,19 +640,19 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
}
/**
- * Manually adds {@link PTableStats} for a table to the client-side cache. Not a
+ * Manually adds {@link GuidePostsInfo} for a table to the client-side cache. Not a
* {@link ConnectionQueryServices} method. Exposed for testing purposes.
*
* @param tableName Table name
* @param stats Stats instance
*/
- public void addTableStats(ImmutableBytesPtr tableName, PTableStats stats) {
- this.tableStatsCache.put(Objects.requireNonNull(tableName), stats);
+ public void addTableStats(GuidePostsKey key, GuidePostsInfo info) {
+ this.guidePostsCache.put(Objects.requireNonNull(key), info);
}
@Override
- public void invalidateStats(ImmutableBytesPtr tableName) {
- this.tableStatsCache.invalidate(Objects.requireNonNull(tableName));
+ public void invalidateStats(GuidePostsKey key) {
+ this.guidePostsCache.invalidate(Objects.requireNonNull(key));
}
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
index b00e92b..7466e9c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
@@ -33,7 +33,6 @@ import org.apache.hadoop.hbase.util.Pair;
import org.apache.phoenix.compile.MutationPlan;
import org.apache.phoenix.coprocessor.MetaDataProtocol.MetaDataMutationResult;
import org.apache.phoenix.execute.MutationState;
-import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.hbase.index.util.KeyValueBuilder;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.parse.PFunction;
@@ -45,7 +44,8 @@ import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.schema.Sequence;
import org.apache.phoenix.schema.SequenceAllocation;
import org.apache.phoenix.schema.SequenceKey;
-import org.apache.phoenix.schema.stats.PTableStats;
+import org.apache.phoenix.schema.stats.GuidePostsInfo;
+import org.apache.phoenix.schema.stats.GuidePostsKey;
import org.apache.tephra.TransactionSystemClient;
@@ -250,8 +250,8 @@ public class DelegateConnectionQueryServices extends DelegateQueryServices imple
}
@Override
- public PTableStats getTableStats(byte[] physicalName, long clientTimeStamp) throws SQLException {
- return getDelegate().getTableStats(physicalName, clientTimeStamp);
+ public GuidePostsInfo getTableStats(GuidePostsKey key) throws SQLException {
+ return getDelegate().getTableStats(key);
}
@@ -342,8 +342,8 @@ public class DelegateConnectionQueryServices extends DelegateQueryServices imple
}
@Override
- public void invalidateStats(ImmutableBytesPtr tableName) {
- getDelegate().invalidateStats(tableName);
+ public void invalidateStats(GuidePostsKey key) {
+ getDelegate().invalidateStats(key);
}
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/query/GuidePostsCache.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/GuidePostsCache.java b/phoenix-core/src/main/java/org/apache/phoenix/query/GuidePostsCache.java
new file mode 100644
index 0000000..d27be1b
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/GuidePostsCache.java
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.query;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
+import org.apache.phoenix.schema.PColumnFamily;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.stats.GuidePostsInfo;
+import org.apache.phoenix.schema.stats.GuidePostsKey;
+import org.apache.phoenix.schema.stats.StatisticsUtil;
+import org.apache.phoenix.util.SchemaUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.cache.RemovalCause;
+import com.google.common.cache.RemovalListener;
+import com.google.common.cache.RemovalNotification;
+import com.google.common.cache.Weigher;
+
+/**
+ * "Client-side" cache for storing {@link GuidePostsInfo} for a column family. Intended to decouple
+ * Phoenix from a specific version of Guava's cache.
+ */
+public class GuidePostsCache {
+ private static final Logger logger = LoggerFactory.getLogger(GuidePostsCache.class);
+
+ private final ConnectionQueryServices queryServices;
+ private final LoadingCache<GuidePostsKey, GuidePostsInfo> cache;
+
+ public GuidePostsCache(ConnectionQueryServices queryServices, Configuration config) {
+ this.queryServices = Objects.requireNonNull(queryServices);
+ // Number of millis to expire cache values after write
+ final long statsUpdateFrequency = config.getLong(
+ QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB,
+ QueryServicesOptions.DEFAULT_STATS_UPDATE_FREQ_MS);
+ // Maximum number of entries (tables) to store in the cache at one time
+ final long maxTableStatsCacheSize = config.getLong(
+ QueryServices.STATS_MAX_CACHE_SIZE,
+ QueryServicesOptions.DEFAULT_STATS_MAX_CACHE_SIZE);
+ cache = CacheBuilder.newBuilder()
+ // Expire entries a given amount of time after they were written
+ .expireAfterWrite(statsUpdateFrequency, TimeUnit.MILLISECONDS)
+ // Maximum total weight (size in bytes) of stats entries
+ .maximumWeight(maxTableStatsCacheSize)
+ // Defer actual size to the PTableStats.getEstimatedSize()
+ .weigher(new Weigher<GuidePostsKey, GuidePostsInfo>() {
+ @Override public int weigh(GuidePostsKey key, GuidePostsInfo info) {
+ return info.getEstimatedSize();
+ }
+ })
+ // Log removals at TRACE for debugging
+ .removalListener(new PhoenixStatsCacheRemovalListener())
+ // Automatically load the cache when entries are missing
+ .build(new StatsLoader());
+ }
+
+ /**
+ * {@link CacheLoader} implementation for the Phoenix Table Stats cache.
+ */
+ protected class StatsLoader extends CacheLoader<GuidePostsKey, GuidePostsInfo> {
+ @Override
+ public GuidePostsInfo load(GuidePostsKey statsKey) throws Exception {
+ @SuppressWarnings("deprecation")
+ HTableInterface statsHTable = queryServices.getTable(SchemaUtil.getPhysicalName(
+ PhoenixDatabaseMetaData.SYSTEM_STATS_NAME_BYTES,
+ queryServices.getProps()).getName());
+ try {
+ GuidePostsInfo guidePostsInfo = StatisticsUtil.readStatistics(statsHTable, statsKey,
+ HConstants.LATEST_TIMESTAMP);
+ traceStatsUpdate(statsKey, guidePostsInfo);
+ return guidePostsInfo;
+ } catch (TableNotFoundException e) {
+ // On a fresh install, stats might not yet be created, don't warn about this.
+ logger.debug("Unable to locate Phoenix stats table", e);
+ return GuidePostsInfo.NO_GUIDEPOST;
+ } catch (IOException e) {
+ logger.warn("Unable to read from stats table", e);
+ // Just cache empty stats. We'll try again after some time anyway.
+ return GuidePostsInfo.NO_GUIDEPOST;
+ } finally {
+ try {
+ statsHTable.close();
+ } catch (IOException e) {
+ // Log, but continue. We have our stats anyway now.
+ logger.warn("Unable to close stats table", e);
+ }
+ }
+ }
+
+ /**
+ * Logs a trace message for newly inserted entries to the stats cache.
+ */
+ void traceStatsUpdate(GuidePostsKey key, GuidePostsInfo info) {
+ if (logger.isTraceEnabled()) {
+ logger.trace("Updating local TableStats cache (id={}) for {}, size={}bytes",
+ new Object[] {Objects.hashCode(GuidePostsCache.this), key,
+ info.getEstimatedSize()});
+ }
+ }
+ }
+
+ /**
+ * Returns the underlying cache. Try to use the provided methods instead of accessing the cache
+ * directly.
+ */
+ LoadingCache<GuidePostsKey, GuidePostsInfo> getCache() {
+ return cache;
+ }
+
+ /**
+ * Returns the PTableStats for the given <code>tableName</code, using the provided
+ * <code>valueLoader</code> if no such mapping exists.
+ *
+ * @see com.google.common.cache.LoadingCache#get(Object)
+ */
+ public GuidePostsInfo get(GuidePostsKey key) throws ExecutionException {
+ return getCache().get(key);
+ }
+
+ /**
+ * Cache the given <code>stats</code> to the cache for the given <code>tableName</code>.
+ *
+ * @see com.google.common.cache.Cache#put(Object, Object)
+ */
+ public void put(GuidePostsKey key, GuidePostsInfo info) {
+ getCache().put(Objects.requireNonNull(key), Objects.requireNonNull(info));
+ }
+
+ /**
+ * Removes the mapping for <code>tableName</code> if it exists.
+ *
+ * @see com.google.common.cache.Cache#invalidate(Object)
+ */
+ public void invalidate(GuidePostsKey key) {
+ getCache().invalidate(Objects.requireNonNull(key));
+ }
+
+ /**
+ * Removes all mappings from the cache.
+ *
+ * @see com.google.common.cache.Cache#invalidateAll()
+ */
+ public void invalidateAll() {
+ getCache().invalidateAll();
+ }
+
+ /**
+ * Removes all mappings where the {@link org.apache.phoenix.schema.stats.GuidePostsKey#getPhysicalName()}
+ * equals physicalName. Because all keys in the map must be iterated, this method should be avoided.
+ * @param physicalName
+ */
+ public void invalidateAll(byte[] physicalName) {
+ for (GuidePostsKey key : getCache().asMap().keySet()) {
+ if (Bytes.compareTo(key.getPhysicalName(), physicalName) == 0) {
+ invalidate(key);
+ }
+ }
+ }
+
+ public void invalidateAll(HTableDescriptor htableDesc) {
+ byte[] tableName = htableDesc.getTableName().getName();
+ for (byte[] fam : htableDesc.getFamiliesKeys()) {
+ invalidate(new GuidePostsKey(tableName, fam));
+ }
+ }
+
+ public void invalidateAll(PTable table) {
+ byte[] physicalName = table.getPhysicalName().getBytes();
+ List<PColumnFamily> families = table.getColumnFamilies();
+ if (families.isEmpty()) {
+ invalidate(new GuidePostsKey(physicalName, SchemaUtil.getEmptyColumnFamily(table)));
+ } else {
+ for (PColumnFamily family : families) {
+ invalidate(new GuidePostsKey(physicalName, family.getName().getBytes()));
+ }
+ }
+ }
+
+ /**
+ * A {@link RemovalListener} implementation to track evictions from the table stats cache.
+ */
+ static class PhoenixStatsCacheRemovalListener implements
+ RemovalListener<GuidePostsKey, GuidePostsInfo> {
+ @Override
+ public void onRemoval(RemovalNotification<GuidePostsKey, GuidePostsInfo> notification) {
+ if (logger.isTraceEnabled()) {
+ final RemovalCause cause = notification.getCause();
+ if (wasEvicted(cause)) {
+ GuidePostsKey key = notification.getKey();
+ logger.trace("Cached stats for {} with size={}bytes was evicted due to cause={}",
+ new Object[] {key, notification.getValue().getEstimatedSize(),
+ cause});
+ }
+ }
+ }
+
+ boolean wasEvicted(RemovalCause cause) {
+ // This is actually a method on RemovalCause but isn't exposed
+ return RemovalCause.EXPLICIT != cause && RemovalCause.REPLACED != cause;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
index fd37328..89f7aba 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
@@ -301,7 +301,7 @@ public interface QueryConstants {
"CREATE TABLE " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_STATS_TABLE + "\"(\n" +
// PK columns
PHYSICAL_NAME + " VARCHAR NOT NULL," +
- COLUMN_FAMILY + " VARCHAR NOT NULL," +
+ COLUMN_FAMILY + " VARCHAR," +
GUIDE_POST_KEY + " VARBINARY," +
GUIDE_POSTS_WIDTH + " BIGINT," +
LAST_STATS_UPDATE_TIME+ " DATE, "+
http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/query/TableStatsCache.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/TableStatsCache.java b/phoenix-core/src/main/java/org/apache/phoenix/query/TableStatsCache.java
deleted file mode 100644
index 2c7b2db..0000000
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/TableStatsCache.java
+++ /dev/null
@@ -1,192 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.phoenix.query;
-
-import java.io.IOException;
-import java.util.Objects;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
-import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
-import org.apache.phoenix.schema.stats.PTableStats;
-import org.apache.phoenix.schema.stats.StatisticsUtil;
-import org.apache.phoenix.util.SchemaUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.common.cache.RemovalCause;
-import com.google.common.cache.RemovalListener;
-import com.google.common.cache.RemovalNotification;
-import com.google.common.cache.Weigher;
-
-/**
- * "Client-side" cache for storing {@link PTableStats} for Phoenix tables. Intended to decouple
- * Phoenix from a specific version of Guava's cache.
- */
-public class TableStatsCache {
- private static final Logger logger = LoggerFactory.getLogger(TableStatsCache.class);
-
- private final ConnectionQueryServices queryServices;
- private final LoadingCache<ImmutableBytesPtr, PTableStats> cache;
-
- public TableStatsCache(ConnectionQueryServices queryServices, Configuration config) {
- this.queryServices = Objects.requireNonNull(queryServices);
- // Number of millis to expire cache values after write
- final long statsUpdateFrequency = config.getLong(
- QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB,
- QueryServicesOptions.DEFAULT_STATS_UPDATE_FREQ_MS);
- // Maximum number of entries (tables) to store in the cache at one time
- final long maxTableStatsCacheSize = config.getLong(
- QueryServices.STATS_MAX_CACHE_SIZE,
- QueryServicesOptions.DEFAULT_STATS_MAX_CACHE_SIZE);
- cache = CacheBuilder.newBuilder()
- // Expire entries a given amount of time after they were written
- .expireAfterWrite(statsUpdateFrequency, TimeUnit.MILLISECONDS)
- // Maximum total weight (size in bytes) of stats entries
- .maximumWeight(maxTableStatsCacheSize)
- // Defer actual size to the PTableStats.getEstimatedSize()
- .weigher(new Weigher<ImmutableBytesPtr, PTableStats>() {
- @Override public int weigh(ImmutableBytesPtr key, PTableStats stats) {
- return stats.getEstimatedSize();
- }
- })
- // Log removals at TRACE for debugging
- .removalListener(new PhoenixStatsCacheRemovalListener())
- // Automatically load the cache when entries are missing
- .build(new StatsLoader());
- }
-
- /**
- * {@link CacheLoader} implementation for the Phoenix Table Stats cache.
- */
- protected class StatsLoader extends CacheLoader<ImmutableBytesPtr, PTableStats> {
- @Override
- public PTableStats load(ImmutableBytesPtr tableName) throws Exception {
- @SuppressWarnings("deprecation")
- HTableInterface statsHTable = queryServices.getTable(SchemaUtil.getPhysicalName(
- PhoenixDatabaseMetaData.SYSTEM_STATS_NAME_BYTES,
- queryServices.getProps()).getName());
- final byte[] tableNameBytes = tableName.copyBytesIfNecessary();
- try {
- PTableStats stats = StatisticsUtil.readStatistics(statsHTable, tableNameBytes,
- Long.MAX_VALUE);
- traceStatsUpdate(tableNameBytes, stats);
- return stats;
- } catch (TableNotFoundException e) {
- // On a fresh install, stats might not yet be created, don't warn about this.
- logger.debug("Unable to locate Phoenix stats table", e);
- return PTableStats.EMPTY_STATS;
- } catch (IOException e) {
- logger.warn("Unable to read from stats table", e);
- // Just cache empty stats. We'll try again after some time anyway.
- return PTableStats.EMPTY_STATS;
- } finally {
- try {
- statsHTable.close();
- } catch (IOException e) {
- // Log, but continue. We have our stats anyway now.
- logger.warn("Unable to close stats table", e);
- }
- }
- }
-
- /**
- * Logs a trace message for newly inserted entries to the stats cache.
- */
- void traceStatsUpdate(byte[] tableName, PTableStats stats) {
- logger.trace("Updating local TableStats cache (id={}) for {}, size={}bytes",
- new Object[] {Objects.hashCode(TableStatsCache.this), Bytes.toString(tableName),
- stats.getEstimatedSize()});
- }
- }
-
- /**
- * Returns the underlying cache. Try to use the provided methods instead of accessing the cache
- * directly.
- */
- LoadingCache<ImmutableBytesPtr, PTableStats> getCache() {
- return cache;
- }
-
- /**
- * Returns the PTableStats for the given <code>tableName</code, using the provided
- * <code>valueLoader</code> if no such mapping exists.
- *
- * @see com.google.common.cache.LoadingCache#get(Object)
- */
- public PTableStats get(ImmutableBytesPtr tableName) throws ExecutionException {
- return getCache().get(tableName);
- }
-
- /**
- * Cache the given <code>stats</code> to the cache for the given <code>tableName</code>.
- *
- * @see com.google.common.cache.Cache#put(Object, Object)
- */
- public void put(ImmutableBytesPtr tableName, PTableStats stats) {
- getCache().put(Objects.requireNonNull(tableName), Objects.requireNonNull(stats));
- }
-
- /**
- * Removes the mapping for <code>tableName</code> if it exists.
- *
- * @see com.google.common.cache.Cache#invalidate(Object)
- */
- public void invalidate(ImmutableBytesPtr tableName) {
- getCache().invalidate(Objects.requireNonNull(tableName));
- }
-
- /**
- * Removes all mappings from the cache.
- *
- * @see com.google.common.cache.Cache#invalidateAll()
- */
- public void invalidateAll() {
- getCache().invalidateAll();
- }
-
- /**
- * A {@link RemovalListener} implementation to track evictions from the table stats cache.
- */
- static class PhoenixStatsCacheRemovalListener implements
- RemovalListener<ImmutableBytesPtr, PTableStats> {
- @Override
- public void onRemoval(RemovalNotification<ImmutableBytesPtr, PTableStats> notification) {
- final RemovalCause cause = notification.getCause();
- if (wasEvicted(cause)) {
- ImmutableBytesPtr ptr = notification.getKey();
- String tableName = new String(ptr.get(), ptr.getOffset(), ptr.getLength());
- logger.trace("Cached stats for {} with size={}bytes was evicted due to cause={}",
- new Object[] {tableName, notification.getValue().getEstimatedSize(),
- cause});
- }
- }
-
- boolean wasEvicted(RemovalCause cause) {
- // This is actually a method on RemovalCause but isn't exposed
- return RemovalCause.EXPLICIT != cause && RemovalCause.REPLACED != cause;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index 0c53d3e..54c6361 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -189,7 +189,7 @@ import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.PTable.IndexType;
import org.apache.phoenix.schema.PTable.LinkType;
import org.apache.phoenix.schema.PTable.ViewType;
-import org.apache.phoenix.schema.stats.PTableStats;
+import org.apache.phoenix.schema.stats.GuidePostsKey;
import org.apache.phoenix.schema.types.PDataType;
import org.apache.phoenix.schema.types.PDate;
import org.apache.phoenix.schema.types.PInteger;
@@ -1085,9 +1085,20 @@ public class MetaDataClient {
* This supports scenarios in which a major compaction was manually initiated and the
* client wants the modified stats to be reflected immediately.
*/
- connection.getQueryServices().clearTableFromCache(tenantIdBytes,
- Bytes.toBytes(SchemaUtil.getSchemaNameFromFullName(physicalName.getString())),
- Bytes.toBytes(SchemaUtil.getTableNameFromFullName(physicalName.getString())), clientTimeStamp);
+ if (cfs == null) {
+ List<PColumnFamily> families = logicalTable.getColumnFamilies();
+ if (families.isEmpty()) {
+ connection.getQueryServices().invalidateStats(new GuidePostsKey(physicalName.getBytes(), SchemaUtil.getEmptyColumnFamily(logicalTable)));
+ } else {
+ for (PColumnFamily family : families) {
+ connection.getQueryServices().invalidateStats(new GuidePostsKey(physicalName.getBytes(), family.getName().getBytes()));
+ }
+ }
+ } else {
+ for (byte[] cf : cfs) {
+ connection.getQueryServices().invalidateStats(new GuidePostsKey(physicalName.getBytes(), cf));
+ }
+ }
return rowCount;
}
@@ -1760,7 +1771,7 @@ public class MetaDataClient {
updateCacheFrequency = updateCacheFrequencyProp;
}
String autoPartitionSeq = (String) TableProperty.AUTO_PARTITION_SEQ.getValue(tableProps);
- Long guidePostWidth = (Long) TableProperty.GUIDE_POSTS_WIDTH.getValue(tableProps);
+ Long guidePostsWidth = (Long) TableProperty.GUIDE_POSTS_WIDTH.getValue(tableProps);
Boolean storeNullsProp = (Boolean) TableProperty.STORE_NULLS.getValue(tableProps);
if (storeNullsProp == null) {
@@ -2227,10 +2238,10 @@ public class MetaDataClient {
tableUpsert.setString(23, autoPartitionSeq);
}
tableUpsert.setBoolean(24, isAppendOnlySchema);
- if (guidePostWidth == null) {
+ if (guidePostsWidth == null) {
tableUpsert.setNull(25, Types.BIGINT);
} else {
- tableUpsert.setLong(25, guidePostWidth);
+ tableUpsert.setLong(25, guidePostsWidth);
}
tableUpsert.execute();
@@ -3108,7 +3119,7 @@ public class MetaDataClient {
connection.setAutoCommit(true);
// Delete everything in the column. You'll still be able to do queries at earlier timestamps
long ts = (scn == null ? result.getMutationTime() : scn);
- MutationPlan plan = new PostDDLCompiler(connection).compile(Collections.singletonList(new TableRef(null, table, ts, false)), emptyCF, Collections.singletonList(projectCF), null, ts);
+ MutationPlan plan = new PostDDLCompiler(connection).compile(Collections.singletonList(new TableRef(null, table, ts, false)), emptyCF, projectCF == null ? null : Collections.singletonList(projectCF), null, ts);
return connection.getQueryServices().updateData(plan);
}
return new MutationState(0,connection);
@@ -3532,28 +3543,6 @@ public class MetaDataClient {
connection.addSchema(result.getSchema());
}
- public PTableStats getTableStats(PTable table) throws SQLException {
- /*
- * The shared view index case is tricky, because we don't have
- * table meta data for it, only an HBase table. We do have stats,
- * though, so we'll query them directly here and cache them so
- * we don't keep querying for them.
- */
- boolean isSharedIndex = table.getViewIndexId() != null;
- if (isSharedIndex) {
- // we are assuming the stats table is not transactional
- return connection.getQueryServices().getTableStats(table.getPhysicalName().getBytes(),
- getCurrentScn());
- }
- boolean isView = table.getType() == PTableType.VIEW;
- String physicalName = table.getPhysicalName().toString().replace(QueryConstants.NAMESPACE_SEPARATOR,
- QueryConstants.NAME_SEPARATOR);
- if (isView && table.getViewType() != ViewType.MAPPED) {
- return connection.getQueryServices().getTableStats(Bytes.toBytes(physicalName), getCurrentScn());
- }
- return connection.getQueryServices().getTableStats(table.getName().getBytes(), getCurrentScn());
- }
-
private void throwIfLastPKOfParentIsFixedLength(PTable parent, String viewSchemaName, String viewName, ColumnDef col) throws SQLException {
if (isLastPKVariableLength(parent)) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_MODIFY_VIEW_PK)
http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java
index 0cf5ed8..9cff48c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
@@ -64,7 +65,7 @@ class DefaultStatisticsCollector implements StatisticsCollector {
private final Pair<Long, GuidePostsInfoBuilder> cachedGuidePosts;
private final byte[] guidePostWidthBytes;
private final byte[] guidePostPerRegionBytes;
- // Where to look for GUIDE_POST_WIDTH in SYSTEM.CATALOG
+ // Where to look for GUIDE_POSTS_WIDTH in SYSTEM.CATALOG
private final byte[] ptableKey;
private final RegionCoprocessorEnvironment env;
@@ -124,12 +125,12 @@ class DefaultStatisticsCollector implements StatisticsCollector {
Get get = new Get(ptableKey);
get.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH_BYTES);
Result result = htable.get(get);
- long guidepostWidth = 0;
+ long guidepostWidth = -1;
if (!result.isEmpty()) {
Cell cell = result.listCells().get(0);
guidepostWidth = PLong.INSTANCE.getCodec().decodeLong(cell.getValueArray(), cell.getValueOffset(), SortOrder.getDefault());
}
- if (guidepostWidth > 0) {
+ if (guidepostWidth >= 0) {
this.guidePostDepth = guidepostWidth;
} else {
// Last use global config value
@@ -157,10 +158,10 @@ class DefaultStatisticsCollector implements StatisticsCollector {
}
@Override
- public void updateStatistic(Region region) {
+ public void updateStatistic(Region region, Scan scan) {
try {
ArrayList<Mutation> mutations = new ArrayList<Mutation>();
- writeStatsToStatsTable(region, true, mutations, TimeKeeper.SYSTEM.getCurrentTime());
+ writeStatistics(region, true, mutations, TimeKeeper.SYSTEM.getCurrentTime(), scan);
if (logger.isDebugEnabled()) {
logger.debug("Committing new stats for the region " + region.getRegionInfo());
}
@@ -170,16 +171,20 @@ class DefaultStatisticsCollector implements StatisticsCollector {
}
}
- private void writeStatsToStatsTable(final Region region, boolean delete, List<Mutation> mutations, long currentTime)
+ private void writeStatistics(final Region region, boolean delete, List<Mutation> mutations, long currentTime, Scan scan)
throws IOException {
try {
Set<ImmutableBytesPtr> fams = guidePostsInfoWriterMap.keySet();
- // update the statistics table
- // Delete statistics for a region if no guidepost is collected for that region during UPDATE STATISTICS
- // This will not impact a stats collection of single column family during compaction as
- // guidePostsInfoWriterMap cannot be empty in this case.
+ // Update the statistics table.
+ // Delete statistics for a region if no guide posts are collected for that region during
+ // UPDATE STATISTICS. This will not impact a stats collection of single column family during
+ // compaction as guidePostsInfoWriterMap cannot be empty in this case.
if (cachedGuidePosts == null) {
- boolean collectingForLocalIndex = !fams.isEmpty() && MetaDataUtil.isLocalIndexFamily(fams.iterator().next());
+ // We're either collecting stats for the data table or the local index table, but not both
+ // We can determine this based on the column families in the scan being prefixed with the
+ // local index column family prefix. We always explicitly specify the local index column
+ // families when we're collecting stats for a local index.
+ boolean collectingForLocalIndex = scan != null && !scan.getFamilyMap().isEmpty() && MetaDataUtil.isLocalIndexFamily(scan.getFamilyMap().keySet().iterator().next());
for (Store store : region.getStores()) {
ImmutableBytesPtr cfKey = new ImmutableBytesPtr(store.getFamily().getName());
boolean isLocalIndexStore = MetaDataUtil.isLocalIndexFamily(cfKey);
@@ -202,7 +207,10 @@ class DefaultStatisticsCollector implements StatisticsCollector {
if (logger.isDebugEnabled()) {
logger.debug("Adding new stats for the region " + region.getRegionInfo());
}
- statsWriter.addStats(this, fam, mutations);
+ // If we've disabled stats, don't write any, just delete them
+ if (this.guidePostDepth > 0) {
+ statsWriter.addStats(this, fam, mutations);
+ }
}
} catch (IOException e) {
logger.error("Failed to update statistics table!", e);
@@ -223,6 +231,10 @@ class DefaultStatisticsCollector implements StatisticsCollector {
*/
@Override
public void collectStatistics(final List<Cell> results) {
+ // A guide posts depth of zero disables the collection of stats
+ if (guidePostDepth == 0) {
+ return;
+ }
Map<ImmutableBytesPtr, Boolean> famMap = Maps.newHashMap();
boolean incrementRow = true;
for (Cell cell : results) {
@@ -274,8 +286,12 @@ class DefaultStatisticsCollector implements StatisticsCollector {
}
private InternalScanner getInternalScanner(RegionCoprocessorEnvironment env, InternalScanner internalScan,
- ImmutableBytesPtr family) {
- return new StatisticsScanner(this, statsWriter, env, internalScan, family);
+ ImmutableBytesPtr family) throws IOException {
+ StatisticsScanner scanner = new StatisticsScanner(this, statsWriter, env, internalScan, family);
+ // We need to initialize the scanner synchronously and potentially perform a cross region Get
+ // in order to use the correct guide posts width for the table being compacted.
+ init();
+ return scanner;
}
@Override
[3/3] phoenix git commit: PHOENIX-2675 Allow stats to be configured
on a table-by-table basis
Posted by ja...@apache.org.
PHOENIX-2675 Allow stats to be configured on a table-by-table basis
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/711d7f06
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/711d7f06
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/711d7f06
Branch: refs/heads/master
Commit: 711d7f0639d294974c2f56c7d604bc37c22e0a95
Parents: ad99952
Author: James Taylor <ja...@apache.org>
Authored: Tue Oct 4 10:34:21 2016 -0700
Committer: James Taylor <ja...@apache.org>
Committed: Fri Oct 7 17:19:15 2016 -0700
----------------------------------------------------------------------
.../apache/phoenix/end2end/GroupByCaseIT.java | 33 ++++-
.../phoenix/end2end/index/ViewIndexIT.java | 129 +++++++++++++++----
.../apache/phoenix/compile/PostDDLCompiler.java | 8 +-
.../phoenix/coprocessor/MetaDataProtocol.java | 8 +-
.../UngroupedAggregateRegionObserver.java | 13 +-
.../query/ConnectionQueryServicesImpl.java | 67 +++++++++-
.../apache/phoenix/query/QueryConstants.java | 1 +
.../apache/phoenix/schema/MetaDataClient.java | 71 ++++++----
.../apache/phoenix/schema/TableProperty.java | 7 +
.../stats/DefaultStatisticsCollector.java | 109 ++++++++++++----
.../schema/stats/NoOpStatisticsCollector.java | 6 +-
.../schema/stats/StatisticsCollector.java | 7 +-
.../stats/StatisticsCollectorFactory.java | 19 +--
.../phoenix/schema/stats/StatisticsScanner.java | 3 +-
.../phoenix/schema/stats/StatisticsUtil.java | 10 +-
.../java/org/apache/phoenix/util/IndexUtil.java | 2 +-
.../org/apache/phoenix/util/MetaDataUtil.java | 40 +++++-
.../org/apache/phoenix/util/UpgradeUtil.java | 2 +-
.../phoenix/compile/QueryCompilerTest.java | 16 +++
19 files changed, 429 insertions(+), 122 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java
index be59fd7..56f4503 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java
@@ -23,20 +23,24 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
+import java.util.List;
import java.util.Properties;
+import org.apache.phoenix.query.KeyRange;
import org.apache.phoenix.schema.AmbiguousColumnException;
import org.apache.phoenix.schema.types.PChar;
import org.apache.phoenix.schema.types.PInteger;
import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.QueryUtil;
+import org.apache.phoenix.util.TestUtil;
import org.junit.Test;
@@ -401,12 +405,29 @@ public class GroupByCaseIT extends ParallelStatsDisabledIT {
}
@Test
- public void testAvgGroupByOrderPreserving() throws Exception {
+ public void testAvgGroupByOrderPreservingWithStats() throws Exception {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
Connection conn = DriverManager.getConnection(getUrl(), props);
String tableName = generateUniqueName();
-
- PreparedStatement stmt = conn.prepareStatement("CREATE TABLE " + tableName + " (k1 char(1) not null, k2 integer not null, constraint pk primary key (k1,k2)) split on (?,?,?)");
+ initAvgGroupTable(conn, tableName, " GUIDE_POST_WIDTH=20 ");
+ testAvgGroupByOrderPreserving(conn, tableName, 13);
+ conn.createStatement().execute("ALTER TABLE " + tableName + " SET GUIDE_POST_WIDTH=" + 100);
+ testAvgGroupByOrderPreserving(conn, tableName, 6);
+ conn.createStatement().execute("ALTER TABLE " + tableName + " SET GUIDE_POST_WIDTH=null");
+ testAvgGroupByOrderPreserving(conn, tableName, 4);
+ }
+
+ @Test
+ public void testAvgGroupByOrderPreservingWithNoStats() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ initAvgGroupTable(conn, tableName, "");
+ testAvgGroupByOrderPreserving(conn, tableName, 4);
+ }
+
+ private void initAvgGroupTable(Connection conn, String tableName, String tableProps) throws SQLException {
+ PreparedStatement stmt = conn.prepareStatement("CREATE TABLE " + tableName + " (k1 char(1) not null, k2 integer not null, constraint pk primary key (k1,k2)) " + tableProps + " split on (?,?,?)");
stmt.setBytes(1, ByteUtil.concat(PChar.INSTANCE.toBytes("a"), PInteger.INSTANCE.toBytes(3)));
stmt.setBytes(2, ByteUtil.concat(PChar.INSTANCE.toBytes("j"), PInteger.INSTANCE.toBytes(3)));
stmt.setBytes(3, ByteUtil.concat(PChar.INSTANCE.toBytes("n"), PInteger.INSTANCE.toBytes(3)));
@@ -425,6 +446,9 @@ public class GroupByCaseIT extends ParallelStatsDisabledIT {
conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 3)");
conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 2)");
conn.commit();
+ }
+
+ private void testAvgGroupByOrderPreserving(Connection conn, String tableName, int nGuidePosts) throws SQLException, IOException {
String query = "SELECT k1,avg(k2) FROM " + tableName + " GROUP BY k1";
ResultSet rs = conn.createStatement().executeQuery(query);
assertTrue(rs.next());
@@ -445,6 +469,9 @@ public class GroupByCaseIT extends ParallelStatsDisabledIT {
"CLIENT PARALLEL 1-WAY FULL SCAN OVER " + tableName + "\n" +
" SERVER FILTER BY FIRST KEY ONLY\n" +
" SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]", QueryUtil.getExplainPlan(rs));
+ TestUtil.analyzeTable(conn, tableName);
+ List<KeyRange> splits = TestUtil.getAllSplits(conn, tableName);
+ assertEquals(nGuidePosts, splits.size());
}
@Test
http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java
index ca50aab..b714a11 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java
@@ -38,7 +38,11 @@ import org.apache.phoenix.end2end.ParallelStatsDisabledIT;
import org.apache.phoenix.jdbc.PhoenixStatement;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.schema.PNameFactory;
+import org.apache.phoenix.schema.PTableType;
+import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.QueryUtil;
+import org.apache.phoenix.util.SchemaUtil;
+import org.apache.phoenix.util.TestUtil;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@@ -59,7 +63,7 @@ public class ViewIndexIT extends ParallelStatsDisabledIT {
if (isNamespaceMapped) {
conn.createStatement().execute("CREATE SCHEMA IF NOT EXISTS " + schemaName);
}
- String ddl = "CREATE TABLE " + tableName + " (t_id VARCHAR NOT NULL,\n" +
+ String ddl = "CREATE TABLE " + SchemaUtil.getTableName(schemaName, tableName) + " (t_id VARCHAR NOT NULL,\n" +
"k1 VARCHAR NOT NULL,\n" +
"k2 INTEGER NOT NULL,\n" +
"v1 VARCHAR,\n" +
@@ -93,30 +97,31 @@ public class ViewIndexIT extends ParallelStatsDisabledIT {
@Test
public void testDeleteViewIndexSequences() throws Exception {
String schemaName = generateUniqueName();
- String tableName = schemaName + "." + generateUniqueName();
+ String tableName = generateUniqueName();
+ String fullTableName = SchemaUtil.getTableName(schemaName, tableName);
String indexName = "IND_" + generateUniqueName();
- String VIEW_NAME = "VIEW_" + generateUniqueName();
- String viewName = schemaName + "." + VIEW_NAME;
+ String viewName = "VIEW_" + generateUniqueName();
+ String fullViewName = SchemaUtil.getTableName(schemaName, viewName);
createBaseTable(schemaName, tableName, false, null, null);
Connection conn1 = getConnection();
Connection conn2 = getConnection();
- conn1.createStatement().execute("CREATE VIEW " + viewName + " AS SELECT * FROM " + tableName);
- conn1.createStatement().execute("CREATE INDEX " + indexName + " ON " + viewName + " (v1)");
- conn2.createStatement().executeQuery("SELECT * FROM " + tableName).next();
- String sequenceName = getViewIndexSequenceName(PNameFactory.newName(tableName), null, isNamespaceMapped);
- String sequenceSchemaName = getViewIndexSequenceSchemaName(PNameFactory.newName(tableName), isNamespaceMapped);
- String seqName = getViewIndexSequenceName(PNameFactory.newName(tableName), null, !isNamespaceMapped);
- String seqSchemaName = getViewIndexSequenceSchemaName(PNameFactory.newName(tableName), !isNamespaceMapped);
+ conn1.createStatement().execute("CREATE VIEW " + fullViewName + " AS SELECT * FROM " + fullTableName);
+ conn1.createStatement().execute("CREATE INDEX " + indexName + " ON " + fullViewName + " (v1)");
+ conn2.createStatement().executeQuery("SELECT * FROM " + fullTableName).next();
+ String sequenceName = getViewIndexSequenceName(PNameFactory.newName(fullTableName), null, isNamespaceMapped);
+ String sequenceSchemaName = getViewIndexSequenceSchemaName(PNameFactory.newName(fullTableName), isNamespaceMapped);
+ String seqName = getViewIndexSequenceName(PNameFactory.newName(fullTableName), null, !isNamespaceMapped);
+ String seqSchemaName = getViewIndexSequenceSchemaName(PNameFactory.newName(fullTableName), !isNamespaceMapped);
verifySequenceValue(null, sequenceName, sequenceSchemaName, -32767);
verifySequenceValue(null, sequenceName, sequenceSchemaName, -32767);
- conn1.createStatement().execute("CREATE INDEX " + indexName + "_2 ON " + viewName + " (v1)");
+ conn1.createStatement().execute("CREATE INDEX " + indexName + "_2 ON " + fullViewName + " (v1)");
verifySequenceValue(null, sequenceName, sequenceSchemaName, -32766);
// Check other format of sequence is not there as Sequences format is different for views/indexes created on
// table which are namespace mapped and which are not.
verifySequenceNotExists(null, seqName, seqSchemaName);
- conn1.createStatement().execute("DROP VIEW " + viewName);
- conn1.createStatement().execute("DROP TABLE "+ tableName);
+ conn1.createStatement().execute("DROP VIEW " + fullViewName);
+ conn1.createStatement().execute("DROP TABLE "+ fullTableName);
verifySequenceNotExists(null, sequenceName, sequenceSchemaName);
}
@@ -126,11 +131,13 @@ public class ViewIndexIT extends ParallelStatsDisabledIT {
String schemaName = generateUniqueName();
String tableName = generateUniqueName();
String indexName = "IND_" + generateUniqueName();
- String VIEW_NAME = "VIEW_" + generateUniqueName();
+ String viewName = "VIEW_" + generateUniqueName();
+ String fullTableName = SchemaUtil.getTableName(schemaName, tableName);
+
createBaseTable(schemaName, tableName, true, null, null);
Connection conn = DriverManager.getConnection(getUrl());
PreparedStatement stmt = conn.prepareStatement(
- "UPSERT INTO " + tableName
+ "UPSERT INTO " + fullTableName
+ " VALUES(?,?,?,?,?)");
stmt.setString(1, "10");
stmt.setString(2, "a");
@@ -144,30 +151,61 @@ public class ViewIndexIT extends ParallelStatsDisabledIT {
stmt.setString(4, "x2");
stmt.setInt(5, 200);
stmt.execute();
+ stmt.setString(1, "30");
+ stmt.setString(2, "c");
+ stmt.setInt(3, 3);
+ stmt.setString(4, "x3");
+ stmt.setInt(5, 300);
+ stmt.execute();
+ stmt.setString(1, "40");
+ stmt.setString(2, "d");
+ stmt.setInt(3, 4);
+ stmt.setString(4, "x4");
+ stmt.setInt(5, 400);
+ stmt.execute();
conn.commit();
- conn.close();
- Properties props = new Properties();
+ Properties props = PropertiesUtil.deepCopy(TestUtil.TEST_PROPERTIES);
props.setProperty("TenantId", "10");
Connection conn1 = DriverManager.getConnection(getUrl(), props);
- conn1.createStatement().execute("CREATE VIEW " + VIEW_NAME
- + " AS select * from " + tableName);
+ conn1.createStatement().execute("CREATE VIEW " + viewName
+ + " AS select * from " + fullTableName);
conn1.createStatement().execute("CREATE LOCAL INDEX "
+ indexName + " ON "
- + VIEW_NAME + "(v2)");
+ + viewName + "(v2)");
conn1.commit();
- String sql = "SELECT * FROM " + VIEW_NAME + " WHERE v2 = 100";
+ String sql = "SELECT * FROM " + viewName + " WHERE v2 = 100";
ResultSet rs = conn1.prepareStatement("EXPLAIN " + sql).executeQuery();
assertEquals(
- "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + tableName + " [1,'10',100]\n" +
+ "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + SchemaUtil.getPhysicalHBaseTableName(fullTableName, isNamespaceMapped, PTableType.TABLE) + " [1,'10',100]\n" +
" SERVER FILTER BY FIRST KEY ONLY\n" +
"CLIENT MERGE SORT", QueryUtil.getExplainPlan(rs));
rs = conn1.prepareStatement(sql).executeQuery();
assertTrue(rs.next());
assertFalse(rs.next());
+
+// TestUtil.analyzeTable(conn, fullTableName);
+// List<KeyRange> guideposts = TestUtil.getAllSplits(conn, fullTableName);
+// assertEquals(1, guideposts.size());
+// assertEquals(KeyRange.EVERYTHING_RANGE, guideposts.get(0));
+//
+// conn.createStatement().execute("ALTER TABLE " + fullTableName + " SET GUIDE_POST_WIDTH=20");
+//
+// TestUtil.analyzeTable(conn, fullTableName);
+// guideposts = TestUtil.getAllSplits(conn, fullTableName);
+// assertEquals(5, guideposts.size());
+//
+// // Confirm that when view index used, the GUIDE_POST_WIDTH from the data physical table
+// // was used
+// sql = "SELECT * FROM " + viewName + " WHERE v2 > 100";
+// stmt = conn1.prepareStatement(sql);
+// stmt.executeQuery();
+// QueryPlan plan = stmt.unwrap(PhoenixStatement.class).getQueryPlan();
+// assertEquals(5, plan.getSplits().size());
}
+
@Test
public void testCreatingIndexOnGlobalView() throws Exception {
String baseTable = generateUniqueName();
@@ -184,6 +222,30 @@ public class ViewIndexIT extends ParallelStatsDisabledIT {
stmt.setString(4, "KV1");
stmt.setString(5, "KV3");
stmt.executeUpdate();
+ stmt.setString(1, "tenantId");
+ stmt.setDate(2, new Date(100));
+ stmt.setInt(3, 2);
+ stmt.setString(4, "KV4");
+ stmt.setString(5, "KV5");
+ stmt.executeUpdate();
+ stmt.setString(1, "tenantId");
+ stmt.setDate(2, new Date(100));
+ stmt.setInt(3, 3);
+ stmt.setString(4, "KV6");
+ stmt.setString(5, "KV7");
+ stmt.executeUpdate();
+ stmt.setString(1, "tenantId");
+ stmt.setDate(2, new Date(100));
+ stmt.setInt(3, 4);
+ stmt.setString(4, "KV8");
+ stmt.setString(5, "KV9");
+ stmt.executeUpdate();
+ stmt.setString(1, "tenantId");
+ stmt.setDate(2, new Date(100));
+ stmt.setInt(3, 5);
+ stmt.setString(4, "KV10");
+ stmt.setString(5, "KV11");
+ stmt.executeUpdate();
conn.commit();
// Verify that query against the global view index works
@@ -196,6 +258,27 @@ public class ViewIndexIT extends ParallelStatsDisabledIT {
assertTrue(rs.next());
assertEquals("KV1", rs.getString(1));
assertFalse(rs.next());
+
+// TestUtil.analyzeTable(conn, baseTable);
+// List<KeyRange> guideposts = TestUtil.getAllSplits(conn, baseTable);
+// assertEquals(1, guideposts.size());
+// assertEquals(KeyRange.EVERYTHING_RANGE, guideposts.get(0));
+//
+// conn.createStatement().execute("ALTER TABLE " + baseTable + " SET GUIDE_POST_WIDTH=20");
+//
+// TestUtil.analyzeTable(conn, baseTable);
+// guideposts = TestUtil.getAllSplits(conn, baseTable);
+// assertEquals(6, guideposts.size());
+//
+// // Confirm that when view index used, the GUIDE_POST_WIDTH from the data physical table
+// // was used
+// stmt = conn.prepareStatement("SELECT KV1 FROM " + globalView + " WHERE PK3 = ? AND KV3 >= ?");
+// stmt.setInt(1, 1);
+// stmt.setString(2, "KV3");
+// rs = stmt.executeQuery();
+// plan = stmt.unwrap(PhoenixStatement.class).getQueryPlan();
+// assertTrue(plan.getTableRef().getTable().getName().getString().equals(globalViewIdx));
+// assertEquals(6, plan.getSplits().size());
}
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java
index e43b596..004e254 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java
@@ -82,7 +82,7 @@ public class PostDDLCompiler {
scan.setAttribute(BaseScannerRegionObserver.UNGROUPED_AGG, QueryConstants.TRUE);
}
- public MutationPlan compile(final List<TableRef> tableRefs, final byte[] emptyCF, final byte[] projectCF, final List<PColumn> deleteList,
+ public MutationPlan compile(final List<TableRef> tableRefs, final byte[] emptyCF, final List<byte[]> projectCFs, final List<PColumn> deleteList,
final long timestamp) throws SQLException {
PhoenixStatement statement = new PhoenixStatement(connection);
final StatementContext context = new StatementContext(
@@ -244,12 +244,14 @@ public class PostDDLCompiler {
}
}
List<byte[]> columnFamilies = Lists.newArrayListWithExpectedSize(tableRef.getTable().getColumnFamilies().size());
- if (projectCF == null) {
+ if (projectCFs == null) {
for (PColumnFamily family : tableRef.getTable().getColumnFamilies()) {
columnFamilies.add(family.getName().getBytes());
}
} else {
- columnFamilies.add(projectCF);
+ for (byte[] projectCF : projectCFs) {
+ columnFamilies.add(projectCF);
+ }
}
// Need to project all column families into the scan, since we haven't yet created our empty key value
RowProjector projector = ProjectionCompiler.compile(context, SelectStatement.COUNT_ONE, GroupBy.EMPTY_GROUP_BY);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
index bb720ee..4f0a34c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
@@ -62,8 +62,8 @@ import com.google.protobuf.ByteString;
*/
public abstract class MetaDataProtocol extends MetaDataService {
public static final int PHOENIX_MAJOR_VERSION = 4;
- public static final int PHOENIX_MINOR_VERSION = 8;
- public static final int PHOENIX_PATCH_NUMBER = 1;
+ public static final int PHOENIX_MINOR_VERSION = 9;
+ public static final int PHOENIX_PATCH_NUMBER = 0;
public static final int PHOENIX_VERSION =
VersionUtil.encodeVersion(PHOENIX_MAJOR_VERSION, PHOENIX_MINOR_VERSION, PHOENIX_PATCH_NUMBER);
@@ -84,8 +84,9 @@ public abstract class MetaDataProtocol extends MetaDataService {
public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 = MIN_TABLE_TIMESTAMP + 15;
public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0 = MIN_TABLE_TIMESTAMP + 18;
public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_8_1 = MIN_TABLE_TIMESTAMP + 18;
+ public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0 = MIN_TABLE_TIMESTAMP + 19;
// MIN_SYSTEM_TABLE_TIMESTAMP needs to be set to the max of all the MIN_SYSTEM_TABLE_TIMESTAMP_* constants
- public static final long MIN_SYSTEM_TABLE_TIMESTAMP = MIN_SYSTEM_TABLE_TIMESTAMP_4_8_1;
+ public static final long MIN_SYSTEM_TABLE_TIMESTAMP = MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0;
// ALWAYS update this map whenever rolling out a new release (major, minor or patch release).
// Key is the SYSTEM.CATALOG timestamp for the version and value is the version string.
@@ -99,6 +100,7 @@ public abstract class MetaDataProtocol extends MetaDataService {
TIMESTAMP_VERSION_MAP.put(MIN_SYSTEM_TABLE_TIMESTAMP_4_6_0, "4.6.x");
TIMESTAMP_VERSION_MAP.put(MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0, "4.7.x");
TIMESTAMP_VERSION_MAP.put(MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0, "4.8.x");
+ TIMESTAMP_VERSION_MAP.put(MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0, "4.9.x");
}
public static final String CURRENT_CLIENT_VERSION = PHOENIX_MAJOR_VERSION + "." + PHOENIX_MINOR_VERSION + "." + PHOENIX_PATCH_NUMBER;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
index a7c6fde..2a4bfca 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
@@ -49,10 +49,8 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.RegionTooBusyException;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Scan;
@@ -113,6 +111,7 @@ import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.ServerUtil;
import org.apache.phoenix.util.StringUtil;
import org.apache.phoenix.util.TimeKeeper;
+import org.apache.tephra.TxConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -120,8 +119,6 @@ import com.google.common.base.Throwables;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
-import org.apache.tephra.TxConstants;
-
/**
* Region observer that aggregates ungrouped rows(i.e. SQL query with aggregation function and no GROUP BY).
@@ -229,9 +226,10 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
throws IOException {
s = super.preScannerOpen(e, scan, s);
if (ScanUtil.isAnalyzeTable(scan)) {
- if (!ScanUtil.isLocalIndex(scan)) {
- scan.getFamilyMap().clear();
- }
+// if (!ScanUtil.isLocalIndex(scan)) {
+// scan.getFamilyMap().clear();
+// }
+// scan.getFamilyMap().clear();
// We are setting the start row and stop row such that it covers the entire region. As part
// of Phonenix-1263 we are storing the guideposts against the physical table rather than
// individual tenant specific tables.
@@ -815,6 +813,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
long rowCount = 0;
try {
if (!compactionRunning) {
+ stats.init();
synchronized (innerScanner) {
do {
List<Cell> results = new ArrayList<Cell>();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index 354a932..4982578 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -36,6 +36,7 @@ import static org.apache.phoenix.util.UpgradeUtil.upgradeTo4_5_0;
import java.io.IOException;
import java.lang.ref.WeakReference;
+import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -2257,6 +2258,41 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
+ private PhoenixConnection removeNotNullConstraint(PhoenixConnection oldMetaConnection, String schemaName, String tableName, long timestamp, String columnName) throws SQLException {
+ Properties props = PropertiesUtil.deepCopy(oldMetaConnection.getClientInfo());
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(timestamp));
+ // Cannot go through DriverManager or you end up in an infinite loop because it'll call init again
+ PhoenixConnection metaConnection = new PhoenixConnection(oldMetaConnection, this, props);
+ SQLException sqlE = null;
+ try {
+ metaConnection.createStatement().executeUpdate("UPSERT INTO " + SYSTEM_STATS_NAME + " (" +
+ PhoenixDatabaseMetaData.TENANT_ID + "," + PhoenixDatabaseMetaData.TABLE_SCHEM + "," +
+ PhoenixDatabaseMetaData.TABLE_NAME + "," + PhoenixDatabaseMetaData.COLUMN_NAME + "," +
+ PhoenixDatabaseMetaData.COLUMN_FAMILY + "," + PhoenixDatabaseMetaData.NULLABLE + ") VALUES (" +
+ "null," + schemaName + "," + tableName + "," + columnName + "," + QueryConstants.DEFAULT_COLUMN_FAMILY + "," +
+ ResultSetMetaData.columnNullable + ")");
+ metaConnection.commit();
+ } catch (NewerTableAlreadyExistsException e) {
+ logger.warn("Table already modified at this timestamp, so assuming column already nullable: " + columnName);
+ } catch (SQLException e) {
+ logger.warn("Add column failed due to:" + e);
+ sqlE = e;
+ } finally {
+ try {
+ oldMetaConnection.close();
+ } catch (SQLException e) {
+ if (sqlE != null) {
+ sqlE.setNextException(e);
+ } else {
+ sqlE = e;
+ }
+ }
+ if (sqlE != null) {
+ throw sqlE;
+ }
+ }
+ return metaConnection;
+ }
/**
* This closes the passed connection.
*/
@@ -2474,7 +2510,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
if (table.getValue(MetaDataUtil.PARENT_TABLE_KEY) == null
&& table.getValue(MetaDataUtil.IS_LOCAL_INDEX_TABLE_PROP_NAME) != null) {
table.setValue(MetaDataUtil.PARENT_TABLE_KEY,
- MetaDataUtil.getUserTableName(table.getNameAsString()));
+ MetaDataUtil.getLocalIndexUserTableName(table.getNameAsString()));
// Explicitly disable, modify and enable the table to ensure
// co-location of data and index regions. If we just modify the
// table descriptor when online schema change enabled may reopen
@@ -2620,6 +2656,18 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0);
clearCache();
}
+ if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0) {
+ metaConnection = addColumnsIfNotExists(
+ metaConnection,
+ PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0,
+ PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + " "
+ + PLong.INSTANCE.getSqlTypeName());
+ ConnectionQueryServicesImpl.this.removeTable(null,
+ PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, null,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0);
+ clearCache();
+ }
}
@@ -2676,7 +2724,9 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
try {
metaConnection.createStatement().executeUpdate(
QueryConstants.CREATE_STATS_TABLE_METADATA);
- } catch (NewerTableAlreadyExistsException ignore) {} catch (TableAlreadyExistsException e) {
+ } catch (NewerTableAlreadyExistsException ignore) {
+
+ } catch (TableAlreadyExistsException e) {
long currentServerSideTableTimeStamp = e.getTable().getTimeStamp();
if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_3_0) {
metaConnection = addColumnsIfNotExists(
@@ -2686,6 +2736,19 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT + " "
+ PLong.INSTANCE.getSqlTypeName());
}
+ if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0) {
+ // The COLUMN_FAMILY column should be nullable as we create a row in it without
+ // any column family to mark when guideposts were last collected.
+ metaConnection = removeNotNullConstraint(metaConnection,
+ PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME,
+ PhoenixDatabaseMetaData.SYSTEM_STATS_TABLE,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0,
+ PhoenixDatabaseMetaData.COLUMN_FAMILY);
+ ConnectionQueryServicesImpl.this.removeTable(null,
+ PhoenixDatabaseMetaData.SYSTEM_STATS_NAME, null,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0);
+ clearCache();
+ }
}
try {
metaConnection.createStatement().executeUpdate(QueryConstants.CREATE_FUNCTION_METADATA);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
index ace228b..fd37328 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
@@ -288,6 +288,7 @@ public interface QueryConstants {
IS_NAMESPACE_MAPPED + " BOOLEAN," +
AUTO_PARTITION_SEQ + " VARCHAR," +
APPEND_ONLY_SCHEMA + " BOOLEAN," +
+ GUIDE_POSTS_WIDTH + " BIGINT," +
"CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + TENANT_ID + ","
+ TABLE_SCHEM + "," + TABLE_NAME + "," + COLUMN_NAME + "," + COLUMN_FAMILY + "))\n" +
HConstants.VERSIONS + "=" + MetaDataProtocol.DEFAULT_MAX_META_DATA_VERSIONS + ",\n" +
http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index 1e635d8..0c53d3e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -42,6 +42,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_COLUMN_FAM
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_VALUE;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DISABLE_WAL;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.FUNCTION_NAME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_ROWS;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_STATE;
@@ -256,8 +257,9 @@ public class MetaDataClient {
UPDATE_CACHE_FREQUENCY + "," +
IS_NAMESPACE_MAPPED + "," +
AUTO_PARTITION_SEQ + "," +
- APPEND_ONLY_SCHEMA +
- ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
+ APPEND_ONLY_SCHEMA + "," +
+ GUIDE_POSTS_WIDTH +
+ ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
private static final String CREATE_SCHEMA = "UPSERT INTO " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE
+ "\"( " + TABLE_SCHEM + "," + TABLE_NAME + ") VALUES (?,?)";
@@ -993,30 +995,20 @@ public class MetaDataClient {
// If analyzing the indexes of a multi-tenant table or a table with view indexes
// then analyze all of those indexes too.
if (table.getType() != PTableType.VIEW) {
- List<PName> names = Lists.newArrayListWithExpectedSize(2);
- final List<PName> physicalNames = Lists.newArrayListWithExpectedSize(2);
if (table.isMultiTenant() || MetaDataUtil.hasViewIndexTable(connection, table.getPhysicalName())) {
- names.add(PNameFactory.newName(SchemaUtil.getTableName(
- MetaDataUtil.getViewIndexSchemaName(table.getSchemaName().getString()),
- MetaDataUtil.getViewIndexTableName(table.getTableName().getString()))));
- physicalNames.add(PNameFactory.newName(MetaDataUtil.getViewIndexPhysicalName(table.getPhysicalName().getBytes())));
- }
- if (MetaDataUtil.hasLocalIndexTable(connection, table.getPhysicalName())) {
- names.add(PNameFactory.newName(SchemaUtil.getTableName(
- MetaDataUtil.getLocalIndexSchemaName(table.getSchemaName().getString()),
- MetaDataUtil.getLocalIndexTableName(table.getTableName().getString()))));
- physicalNames.add(PNameFactory.newName(MetaDataUtil.getLocalIndexPhysicalName(table.getPhysicalName().getBytes())));
- }
- int i = 0;
- for (final PName name : names) {
- final int index = i++;
+ final PName physicalName = PNameFactory.newName(MetaDataUtil.getViewIndexPhysicalName(table.getPhysicalName().getBytes()));
PTable indexLogicalTable = new DelegateTable(table) {
@Override
public PName getPhysicalName() {
- return physicalNames.get(index);
+ return physicalName;
}
};
- rowCount += updateStatisticsInternal(name, indexLogicalTable, updateStatisticsStmt.getProps());
+ rowCount += updateStatisticsInternal(physicalName, indexLogicalTable, updateStatisticsStmt.getProps());
+ }
+ PName physicalName = table.getPhysicalName();
+ List<byte[]> localCFs = MetaDataUtil.getLocalIndexColumnFamilies(connection, physicalName.getBytes());
+ if (!localCFs.isEmpty()) {
+ rowCount += updateStatisticsInternal(physicalName, table, updateStatisticsStmt.getProps(), localCFs);
}
}
}
@@ -1030,6 +1022,10 @@ public class MetaDataClient {
}
private long updateStatisticsInternal(PName physicalName, PTable logicalTable, Map<String, Object> statsProps) throws SQLException {
+ return updateStatisticsInternal(physicalName, logicalTable, statsProps, null);
+ }
+
+ private long updateStatisticsInternal(PName physicalName, PTable logicalTable, Map<String, Object> statsProps, List<byte[]> cfs) throws SQLException {
ReadOnlyProps props = connection.getQueryServices().getProps();
final long msMinBetweenUpdates = props
.getLong(QueryServices.MIN_STATS_UPDATE_FREQ_MS_ATTRIB,
@@ -1063,7 +1059,7 @@ public class MetaDataClient {
}
};
TableRef tableRef = new TableRef(null, nonTxnLogicalTable, clientTimeStamp, false);
- MutationPlan plan = compiler.compile(Collections.singletonList(tableRef), null, null, null, clientTimeStamp);
+ MutationPlan plan = compiler.compile(Collections.singletonList(tableRef), null, cfs, null, clientTimeStamp);
Scan scan = plan.getContext().getScan();
scan.setCacheBlocks(false);
scan.setAttribute(ANALYZE_TABLE, TRUE_BYTES);
@@ -1764,6 +1760,7 @@ public class MetaDataClient {
updateCacheFrequency = updateCacheFrequencyProp;
}
String autoPartitionSeq = (String) TableProperty.AUTO_PARTITION_SEQ.getValue(tableProps);
+ Long guidePostWidth = (Long) TableProperty.GUIDE_POSTS_WIDTH.getValue(tableProps);
Boolean storeNullsProp = (Boolean) TableProperty.STORE_NULLS.getValue(tableProps);
if (storeNullsProp == null) {
@@ -2230,6 +2227,11 @@ public class MetaDataClient {
tableUpsert.setString(23, autoPartitionSeq);
}
tableUpsert.setBoolean(24, isAppendOnlySchema);
+ if (guidePostWidth == null) {
+ tableUpsert.setNull(25, Types.BIGINT);
+ } else {
+ tableUpsert.setLong(25, guidePostWidth);
+ }
tableUpsert.execute();
if (asyncCreatedDate != null) {
@@ -2631,11 +2633,12 @@ public class MetaDataClient {
}
private long incrementTableSeqNum(PTable table, PTableType expectedType, int columnCountDelta, Boolean isTransactional, Long updateCacheFrequency) throws SQLException {
- return incrementTableSeqNum(table, expectedType, columnCountDelta, isTransactional, updateCacheFrequency, null, null, null, null);
+ return incrementTableSeqNum(table, expectedType, columnCountDelta, isTransactional, updateCacheFrequency, null, null, null, null, -1L);
}
private long incrementTableSeqNum(PTable table, PTableType expectedType, int columnCountDelta,
- Boolean isTransactional, Long updateCacheFrequency, Boolean isImmutableRows, Boolean disableWAL, Boolean isMultiTenant, Boolean storeNulls)
+ Boolean isTransactional, Long updateCacheFrequency, Boolean isImmutableRows, Boolean disableWAL,
+ Boolean isMultiTenant, Boolean storeNulls, Long guidePostWidth)
throws SQLException {
String schemaName = table.getSchemaName().getString();
String tableName = table.getTableName().getString();
@@ -2673,6 +2676,9 @@ public class MetaDataClient {
if (updateCacheFrequency != null) {
mutateLongProperty(tenantId, schemaName, tableName, UPDATE_CACHE_FREQUENCY, updateCacheFrequency);
}
+ if (guidePostWidth == null || guidePostWidth >= 0) {
+ mutateLongProperty(tenantId, schemaName, tableName, GUIDE_POSTS_WIDTH, guidePostWidth);
+ }
return seqNum;
}
@@ -2694,7 +2700,7 @@ public class MetaDataClient {
}
private void mutateLongProperty(String tenantId, String schemaName, String tableName,
- String propertyName, long propertyValue) throws SQLException {
+ String propertyName, Long propertyValue) throws SQLException {
String updatePropertySql = "UPSERT INTO " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE + "\"( " +
TENANT_ID + "," +
TABLE_SCHEM + "," +
@@ -2705,7 +2711,11 @@ public class MetaDataClient {
tableBoolUpsert.setString(1, tenantId);
tableBoolUpsert.setString(2, schemaName);
tableBoolUpsert.setString(3, tableName);
- tableBoolUpsert.setLong(4, propertyValue);
+ if (propertyValue == null) {
+ tableBoolUpsert.setNull(4, Types.BIGINT);
+ } else {
+ tableBoolUpsert.setLong(4, propertyValue);
+ }
tableBoolUpsert.execute();
}
}
@@ -2733,6 +2743,7 @@ public class MetaDataClient {
Boolean storeNullsProp = null;
Boolean isTransactionalProp = null;
Long updateCacheFrequencyProp = null;
+ Long guidePostWidth = -1L;
Map<String, List<Pair<String, Object>>> properties = new HashMap<>(stmtProperties.size());
List<ColumnDef> columnDefs = null;
@@ -2791,6 +2802,8 @@ public class MetaDataClient {
isTransactionalProp = (Boolean)value;
} else if (propName.equals(UPDATE_CACHE_FREQUENCY)) {
updateCacheFrequencyProp = (Long)value;
+ } else if (propName.equals(GUIDE_POSTS_WIDTH)) {
+ guidePostWidth = (Long)value;
}
}
// if removeTableProps is true only add the property if it is not a HTable or Phoenix Table property
@@ -2857,6 +2870,9 @@ public class MetaDataClient {
changingPhoenixTableProperty = true;
}
}
+ if (guidePostWidth == null || guidePostWidth >= 0) {
+ changingPhoenixTableProperty = true;
+ }
Boolean storeNulls = null;
if (storeNullsProp != null) {
if (storeNullsProp.booleanValue() != table.getStoreNulls()) {
@@ -2996,7 +3012,8 @@ public class MetaDataClient {
}
long seqNum = table.getSequenceNumber();
if (changingPhoenixTableProperty || columnDefs.size() > 0) {
- seqNum = incrementTableSeqNum(table, tableType, columnDefs.size(), isTransactional, updateCacheFrequency, isImmutableRows, disableWAL, multiTenant, storeNulls);
+ seqNum = incrementTableSeqNum(table, tableType, columnDefs.size(), isTransactional, updateCacheFrequency, isImmutableRows,
+ disableWAL, multiTenant, storeNulls, guidePostWidth);
tableMetaData.addAll(connection.getMutationState().toMutations(timeStamp).next().getSecond());
connection.rollback();
}
@@ -3091,7 +3108,7 @@ public class MetaDataClient {
connection.setAutoCommit(true);
// Delete everything in the column. You'll still be able to do queries at earlier timestamps
long ts = (scn == null ? result.getMutationTime() : scn);
- MutationPlan plan = new PostDDLCompiler(connection).compile(Collections.singletonList(new TableRef(null, table, ts, false)), emptyCF, projectCF, null, ts);
+ MutationPlan plan = new PostDDLCompiler(connection).compile(Collections.singletonList(new TableRef(null, table, ts, false)), emptyCF, Collections.singletonList(projectCF), null, ts);
return connection.getQueryServices().updateData(plan);
}
return new MutationState(0,connection);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java
index d5d0b84..26a7718 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java
@@ -77,6 +77,13 @@ public enum TableProperty {
},
APPEND_ONLY_SCHEMA(PhoenixDatabaseMetaData.APPEND_ONLY_SCHEMA, COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY, true, true),
+ GUIDE_POSTS_WIDTH(PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH, true, false) {
+ @Override
+ public Object getValue(Object value) {
+ return value == null ? null : ((Number) value).longValue();
+ }
+
+ },
;
private final String propertyName;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java
index 3bb9898..0cf5ed8 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java
@@ -21,12 +21,17 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
@@ -35,11 +40,14 @@ import org.apache.hadoop.hbase.regionserver.Store;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.phoenix.coprocessor.MetaDataProtocol;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.SortOrder;
import org.apache.phoenix.schema.types.PInteger;
import org.apache.phoenix.schema.types.PLong;
+import org.apache.phoenix.util.MetaDataUtil;
+import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.TimeKeeper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -53,24 +61,21 @@ class DefaultStatisticsCollector implements StatisticsCollector {
private static final Logger logger = LoggerFactory.getLogger(DefaultStatisticsCollector.class);
private final Map<ImmutableBytesPtr, Pair<Long, GuidePostsInfoBuilder>> guidePostsInfoWriterMap = Maps.newHashMap();
private final StatisticsWriter statsWriter;
- private final Pair<Long, GuidePostsInfoBuilder> cachedGps;
+ private final Pair<Long, GuidePostsInfoBuilder> cachedGuidePosts;
+ private final byte[] guidePostWidthBytes;
+ private final byte[] guidePostPerRegionBytes;
+ // Where to look for GUIDE_POST_WIDTH in SYSTEM.CATALOG
+ private final byte[] ptableKey;
+ private final RegionCoprocessorEnvironment env;
- private long guidepostDepth;
+ private long guidePostDepth;
private long maxTimeStamp = MetaDataProtocol.MIN_TABLE_TIMESTAMP;
DefaultStatisticsCollector(RegionCoprocessorEnvironment env, String tableName, long clientTimeStamp, byte[] family,
byte[] gp_width_bytes, byte[] gp_per_region_bytes) throws IOException {
- Configuration config = env.getConfiguration();
- int guidepostPerRegion = gp_per_region_bytes == null
- ? config.getInt(QueryServices.STATS_GUIDEPOST_PER_REGION_ATTRIB,
- QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_PER_REGION)
- : PInteger.INSTANCE.getCodec().decodeInt(gp_per_region_bytes, 0, SortOrder.getDefault());
- long guidepostWidth = gp_width_bytes == null
- ? config.getLong(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB,
- QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_WIDTH_BYTES)
- : PLong.INSTANCE.getCodec().decodeInt(gp_width_bytes, 0, SortOrder.getDefault());
- this.guidepostDepth = StatisticsUtil.getGuidePostDepth(guidepostPerRegion, guidepostWidth,
- env.getRegion().getTableDesc());
+ this.env = env;
+ this.guidePostWidthBytes = gp_width_bytes;
+ this.guidePostPerRegionBytes = gp_per_region_bytes;
// Provides a means of clients controlling their timestamps to not use current time
// when background tasks are updating stats. Instead we track the max timestamp of
// the cells and use that.
@@ -80,16 +85,64 @@ class DefaultStatisticsCollector implements StatisticsCollector {
if (!useCurrentTime) {
clientTimeStamp = DefaultStatisticsCollector.NO_TIMESTAMP;
}
+ String pName = tableName;
+ // For view index, get GUIDE_POST_WIDTH from data physical table
+ // since there's no row representing those in SYSTEM.CATALOG.
+ if (MetaDataUtil.isViewIndex(tableName)) {
+ pName = MetaDataUtil.getViewIndexUserTableName(tableName);
+ }
+ ptableKey = SchemaUtil.getTableKeyFromFullName(pName);
// Get the stats table associated with the current table on which the CP is
// triggered
this.statsWriter = StatisticsWriter.newWriter(env, tableName, clientTimeStamp);
// in a compaction we know the one family ahead of time
if (family != null) {
ImmutableBytesPtr cfKey = new ImmutableBytesPtr(family);
- cachedGps = new Pair<Long, GuidePostsInfoBuilder>(0l, new GuidePostsInfoBuilder());
- guidePostsInfoWriterMap.put(cfKey, cachedGps);
+ cachedGuidePosts = new Pair<Long, GuidePostsInfoBuilder>(0l, new GuidePostsInfoBuilder());
+ guidePostsInfoWriterMap.put(cfKey, cachedGuidePosts);
+ } else {
+ cachedGuidePosts = null;
+ }
+ }
+
+ private void initGuidepostDepth() throws IOException {
+ // First check is if guidepost info set on statement itself
+ if (guidePostPerRegionBytes != null || guidePostWidthBytes != null) {
+ int guidepostPerRegion = 0;
+ long guidepostWidth = QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_WIDTH_BYTES;
+ if (guidePostPerRegionBytes != null) {
+ guidepostPerRegion = PInteger.INSTANCE.getCodec().decodeInt(guidePostPerRegionBytes, 0, SortOrder.getDefault());
+ }
+ if (guidePostWidthBytes != null) {
+ guidepostWidth = PLong.INSTANCE.getCodec().decodeInt(guidePostWidthBytes, 0, SortOrder.getDefault());
+ }
+ this.guidePostDepth = StatisticsUtil.getGuidePostDepth(guidepostPerRegion, guidepostWidth,
+ env.getRegion().getTableDesc());
} else {
- cachedGps = null;
+ // Next check for GUIDE_POST_WIDTH on table
+ HTableInterface htable = env.getTable(TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES));
+ Get get = new Get(ptableKey);
+ get.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH_BYTES);
+ Result result = htable.get(get);
+ long guidepostWidth = 0;
+ if (!result.isEmpty()) {
+ Cell cell = result.listCells().get(0);
+ guidepostWidth = PLong.INSTANCE.getCodec().decodeLong(cell.getValueArray(), cell.getValueOffset(), SortOrder.getDefault());
+ }
+ if (guidepostWidth > 0) {
+ this.guidePostDepth = guidepostWidth;
+ } else {
+ // Last use global config value
+ Configuration config = env.getConfiguration();
+ this.guidePostDepth = StatisticsUtil.getGuidePostDepth(
+ config.getInt(
+ QueryServices.STATS_GUIDEPOST_PER_REGION_ATTRIB,
+ QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_PER_REGION),
+ config.getLong(
+ QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB,
+ QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_WIDTH_BYTES),
+ env.getRegion().getTableDesc());
+ }
}
}
@@ -114,28 +167,32 @@ class DefaultStatisticsCollector implements StatisticsCollector {
commitStats(mutations);
} catch (IOException e) {
logger.error("Unable to commit new stats", e);
- } finally {
- clear();
}
}
private void writeStatsToStatsTable(final Region region, boolean delete, List<Mutation> mutations, long currentTime)
throws IOException {
try {
+ Set<ImmutableBytesPtr> fams = guidePostsInfoWriterMap.keySet();
// update the statistics table
// Delete statistics for a region if no guidepost is collected for that region during UPDATE STATISTICS
// This will not impact a stats collection of single column family during compaction as
// guidePostsInfoWriterMap cannot be empty in this case.
- if (cachedGps == null) {
+ if (cachedGuidePosts == null) {
+ boolean collectingForLocalIndex = !fams.isEmpty() && MetaDataUtil.isLocalIndexFamily(fams.iterator().next());
for (Store store : region.getStores()) {
ImmutableBytesPtr cfKey = new ImmutableBytesPtr(store.getFamily().getName());
+ boolean isLocalIndexStore = MetaDataUtil.isLocalIndexFamily(cfKey);
+ if (isLocalIndexStore != collectingForLocalIndex) {
+ continue;
+ }
if (!guidePostsInfoWriterMap.containsKey(cfKey)) {
Pair<Long, GuidePostsInfoBuilder> emptyGps = new Pair<Long, GuidePostsInfoBuilder>(0l, new GuidePostsInfoBuilder());
guidePostsInfoWriterMap.put(cfKey, emptyGps);
}
}
}
- for (ImmutableBytesPtr fam : guidePostsInfoWriterMap.keySet()) {
+ for (ImmutableBytesPtr fam : fams) {
if (delete) {
if (logger.isDebugEnabled()) {
logger.debug("Deleting the stats for the region " + region.getRegionInfo());
@@ -162,6 +219,7 @@ class DefaultStatisticsCollector implements StatisticsCollector {
*
* @param results
* next batch of {@link KeyValue}s
+ * @throws IOException
*/
@Override
public void collectStatistics(final List<Cell> results) {
@@ -171,7 +229,7 @@ class DefaultStatisticsCollector implements StatisticsCollector {
KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
maxTimeStamp = Math.max(maxTimeStamp, kv.getTimestamp());
Pair<Long, GuidePostsInfoBuilder> gps;
- if (cachedGps == null) {
+ if (cachedGuidePosts == null) {
ImmutableBytesPtr cfKey = new ImmutableBytesPtr(kv.getFamilyArray(), kv.getFamilyOffset(),
kv.getFamilyLength());
gps = guidePostsInfoWriterMap.get(cfKey);
@@ -185,16 +243,16 @@ class DefaultStatisticsCollector implements StatisticsCollector {
gps.getSecond().incrementRowCount();
}
} else {
- gps = cachedGps;
+ gps = cachedGuidePosts;
if (incrementRow) {
- cachedGps.getSecond().incrementRowCount();
+ cachedGuidePosts.getSecond().incrementRowCount();
incrementRow = false;
}
}
int kvLength = kv.getLength();
long byteCount = gps.getFirst() + kvLength;
gps.setFirst(byteCount);
- if (byteCount >= guidepostDepth) {
+ if (byteCount >= guidePostDepth) {
ImmutableBytesWritable row = new ImmutableBytesWritable(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength());
if (gps.getSecond().addGuidePosts(row, byteCount, gps.getSecond().getRowCount())) {
gps.setFirst(0l);
@@ -221,9 +279,10 @@ class DefaultStatisticsCollector implements StatisticsCollector {
}
@Override
- public void clear() {
+ public void init() throws IOException {
this.guidePostsInfoWriterMap.clear();
maxTimeStamp = MetaDataProtocol.MIN_TABLE_TIMESTAMP;
+ initGuidepostDepth();
}
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java
index 1063229..a16a48a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java
@@ -21,13 +21,10 @@ import java.io.IOException;
import java.util.List;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.Store;
-import org.apache.hadoop.hbase.util.Pair;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
/**
@@ -62,7 +59,8 @@ public class NoOpStatisticsCollector implements StatisticsCollector {
return delegate;
}
- @Override public void clear() {
+ @Override
+ public void init() {
// No-op
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
index 1dcab08..43ec6c7 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
@@ -23,7 +23,6 @@ import java.util.List;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.Store;
@@ -50,6 +49,7 @@ public interface StatisticsCollector extends Closeable {
/**
* Collect statistics for the given list of cells. This method can be called multiple times
* during collection of statistics.
+ * @throws IOException
*/
void collectStatistics(List<Cell> results);
@@ -60,9 +60,10 @@ public interface StatisticsCollector extends Closeable {
InternalScanner delegate) throws IOException;
/**
- * Clear all statistics information that has been collected.
+ * Called before beginning the collection of statistics through {@link #collectStatistics(List)}
+ * @throws IOException
*/
- void clear();
+ void init() throws IOException;
/**
* Retrieve the calculated guide post info for the given column family.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectorFactory.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectorFactory.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectorFactory.java
index bb21f1b..bbc7cd4 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectorFactory.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectorFactory.java
@@ -29,27 +29,28 @@ import org.apache.phoenix.query.QueryServices;
public class StatisticsCollectorFactory {
public static StatisticsCollector createStatisticsCollector(RegionCoprocessorEnvironment env,
- String tableName, long clientTimestamp, byte[] guidepostWidthBytes,
+ String tableName, long clientTimeStamp, byte[] guidepostWidthBytes,
byte[] guidepostsPerRegionBytes) throws IOException {
- if (statisticsEnabled(env)) {
- return new DefaultStatisticsCollector(env, tableName, clientTimestamp, null,
- guidepostWidthBytes, guidepostsPerRegionBytes);
- } else {
- return new NoOpStatisticsCollector();
- }
+ return createStatisticsCollector(env, tableName, clientTimeStamp, null, guidepostWidthBytes, guidepostsPerRegionBytes);
}
public static StatisticsCollector createStatisticsCollector(
RegionCoprocessorEnvironment env, String tableName, long clientTimeStamp,
byte[] storeName) throws IOException {
+ return createStatisticsCollector(env, tableName, clientTimeStamp, storeName, null, null);
+ }
+
+ public static StatisticsCollector createStatisticsCollector(
+ RegionCoprocessorEnvironment env, String tableName, long clientTimeStamp,
+ byte[] storeName, byte[] guidepostWidthBytes,
+ byte[] guidepostsPerRegionBytes) throws IOException {
if (statisticsEnabled(env)) {
return new DefaultStatisticsCollector(env, tableName, clientTimeStamp, storeName,
- null, null);
+ guidepostWidthBytes, guidepostsPerRegionBytes);
} else {
return new NoOpStatisticsCollector();
}
}
-
/**
* Determines if statistics are enabled (which is the default). This is done on the
http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java
index 736efc6..8306940 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java
@@ -83,8 +83,9 @@ public class StatisticsScanner implements InternalScanner {
*
* @param results
* next batch of {@link KeyValue}s
+ * @throws IOException
*/
- private void updateStats(final List<Cell> results) {
+ private void updateStats(final List<Cell> results) throws IOException {
if (!results.isEmpty()) {
tracker.collectStatistics(results);
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
index db31b69..685c24e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
@@ -20,7 +20,6 @@ import static org.apache.phoenix.util.SchemaUtil.getVarCharLength;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.List;
import java.util.Set;
import java.util.SortedMap;
@@ -30,7 +29,6 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Result;
@@ -99,7 +97,7 @@ public class StatisticsUtil {
return rowKey;
}
- public static byte[] getKey(byte[] table, ImmutableBytesPtr fam) {
+ private static byte[] getKey(byte[] table, ImmutableBytesPtr fam) {
// always starts with the source table and column family
byte[] rowKey = new byte[table.length + fam.getLength() + 1];
int offset = 0;
@@ -111,11 +109,7 @@ public class StatisticsUtil {
return rowKey;
}
- public static byte[] copyRow(KeyValue kv) {
- return Arrays.copyOfRange(kv.getRowArray(), kv.getRowOffset(), kv.getRowOffset() + kv.getRowLength());
- }
-
- public static byte[] getAdjustedKey(byte[] key, byte[] tableNameBytes, ImmutableBytesPtr cf, boolean nextKey) {
+ private static byte[] getAdjustedKey(byte[] key, byte[] tableNameBytes, ImmutableBytesPtr cf, boolean nextKey) {
if (Bytes.compareTo(key, ByteUtil.EMPTY_BYTE_ARRAY) != 0) { return getRowKey(tableNameBytes, cf, key); }
key = ByteUtil.concat(getKey(tableNameBytes, cf), QueryConstants.SEPARATOR_BYTE_ARRAY);
if (nextKey) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
index 9089b68..9622880 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
@@ -491,7 +491,7 @@ public class IndexUtil {
joinResult = dataRegion.get(get);
} else {
TableName dataTable =
- TableName.valueOf(MetaDataUtil.getUserTableName(c.getEnvironment()
+ TableName.valueOf(MetaDataUtil.getLocalIndexUserTableName(c.getEnvironment()
.getRegion().getTableDesc().getNameAsString()));
HTableInterface table = null;
try {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
index c72e404..52346a5 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
@@ -214,7 +215,7 @@ public class MetaDataUtil {
}
public static long getSequenceNumber(List<Mutation> tableMetaData) {
- return getSequenceNumber(getTableHeaderRow(tableMetaData));
+ return getSequenceNumber(getPutOnlyTableHeaderRow(tableMetaData));
}
public static PTableType getTableType(List<Mutation> tableMetaData, KeyValueBuilder builder,
@@ -366,7 +367,7 @@ public class MetaDataUtil {
return schemaName;
}
- public static String getUserTableName(String localIndexTableName) {
+ public static String getLocalIndexUserTableName(String localIndexTableName) {
if (localIndexTableName.contains(QueryConstants.NAMESPACE_SEPARATOR)) {
String schemaName = SchemaUtil.getSchemaNameFromFullName(localIndexTableName,
QueryConstants.NAMESPACE_SEPARATOR);
@@ -384,6 +385,24 @@ public class MetaDataUtil {
}
}
+ public static String getViewIndexUserTableName(String viewIndexTableName) {
+ if (viewIndexTableName.contains(QueryConstants.NAMESPACE_SEPARATOR)) {
+ String schemaName = SchemaUtil.getSchemaNameFromFullName(viewIndexTableName,
+ QueryConstants.NAMESPACE_SEPARATOR);
+ String tableName = SchemaUtil.getTableNameFromFullName(viewIndexTableName,
+ QueryConstants.NAMESPACE_SEPARATOR);
+ String userTableName = tableName.substring(VIEW_INDEX_TABLE_PREFIX.length());
+ return (schemaName + QueryConstants.NAMESPACE_SEPARATOR + userTableName);
+ } else {
+ String schemaName = SchemaUtil.getSchemaNameFromFullName(viewIndexTableName);
+ if (!schemaName.isEmpty()) schemaName = schemaName.substring(VIEW_INDEX_TABLE_PREFIX.length());
+ String tableName = viewIndexTableName.substring(
+ (schemaName.isEmpty() ? 0 : (schemaName.length() + QueryConstants.NAME_SEPARATOR.length()))
+ + VIEW_INDEX_TABLE_PREFIX.length());
+ return SchemaUtil.getTableName(schemaName, tableName);
+ }
+ }
+
public static String getViewIndexSequenceSchemaName(PName physicalName, boolean isNamespaceMapped) {
if (!isNamespaceMapped) { return VIEW_INDEX_SEQUENCE_PREFIX + physicalName.getString(); }
return SchemaUtil.getSchemaNameFromFullName(physicalName.toString());
@@ -461,6 +480,18 @@ public class MetaDataUtil {
return families;
}
+ public static List<byte[]> getLocalIndexColumnFamilies(PhoenixConnection conn, byte[] physicalTableName) throws SQLException {
+ HTableDescriptor desc = conn.getQueryServices().getTableDescriptor(physicalTableName);
+ if(desc == null ) return Collections.emptyList();
+ List<byte[]> families = new ArrayList<byte[]>(desc.getColumnFamilies().length / 2);
+ for (HColumnDescriptor cf : desc.getColumnFamilies()) {
+ if (cf.getNameAsString().startsWith(QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX)) {
+ families.add(cf.getName());
+ }
+ }
+ return families;
+ }
+
public static void deleteViewIndexSequences(PhoenixConnection connection, PName name, boolean isNamespaceMapped)
throws SQLException {
String schemaName = getViewIndexSequenceSchemaName(name, isNamespaceMapped);
@@ -603,4 +634,9 @@ public class MetaDataUtil {
public static boolean isHTableProperty(String propName) {
return !isHColumnProperty(propName) && !TableProperty.isPhoenixTableProperty(propName);
}
+
+ public static boolean isLocalIndexFamily(ImmutableBytesPtr cfPtr) {
+ return cfPtr.getLength() >= QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX_BYTES.length &&
+ Bytes.compareTo(cfPtr.get(), cfPtr.getOffset(), QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX_BYTES.length, QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX_BYTES, 0, QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX_BYTES.length) == 0;
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
index cddebb7..93b3f73 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
@@ -348,7 +348,7 @@ public class UpgradeUtil {
HTableDescriptor[] localIndexTables = admin.listTables(MetaDataUtil.LOCAL_INDEX_TABLE_PREFIX+".*");
String localIndexSplitter = LocalIndexSplitter.class.getName();
for (HTableDescriptor table : localIndexTables) {
- HTableDescriptor dataTableDesc = admin.getTableDescriptor(TableName.valueOf(MetaDataUtil.getUserTableName(table.getNameAsString())));
+ HTableDescriptor dataTableDesc = admin.getTableDescriptor(TableName.valueOf(MetaDataUtil.getLocalIndexUserTableName(table.getNameAsString())));
HColumnDescriptor[] columnFamilies = dataTableDesc.getColumnFamilies();
boolean modifyTable = false;
for(HColumnDescriptor cf : columnFamilies) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
index c91b855..7697d8c 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
@@ -2257,6 +2257,22 @@ public class QueryCompilerTest extends BaseConnectionlessQueryTest {
}
}
+ @Test
+ public void testNegativeGuidePostWidth() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);) {
+ try {
+ conn.createStatement().execute(
+ "CREATE TABLE t (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR) GUIDE_POST_WIDTH = -1");
+ fail();
+ } catch (SQLException e) {
+ assertEquals("Unexpected Exception",
+ SQLExceptionCode.PARSER_ERROR
+ .getErrorCode(), e.getErrorCode());
+ }
+ }
+ }
+
private static void assertFamilies(Scan s, String... families) {
assertEquals(families.length, s.getFamilyMap().size());
for (String fam : families) {