You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ja...@apache.org on 2014/01/27 20:23:07 UTC
[06/51] [partial] Initial commit
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/c5b80246/src/main/java/org/apache/phoenix/util/SchemaUtil.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/phoenix/util/SchemaUtil.java b/src/main/java/org/apache/phoenix/util/SchemaUtil.java
new file mode 100644
index 0000000..b960073
--- /dev/null
+++ b/src/main/java/org/apache/phoenix/util/SchemaUtil.java
@@ -0,0 +1,823 @@
+/*
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.util;
+
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_MODIFIER;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_NAME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DATA_TABLE_NAME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DATA_TYPE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_ROWS;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_STATE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NULLABLE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ORDINAL_POSITION;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SALT_BUCKETS;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_CAT_NAME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME_NAME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM_NAME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_SCHEMA_AND_TABLE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_TABLE_NAME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_TABLE_NAME_BYTES;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValue.Type;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.phoenix.coprocessor.MetaDataProtocol;
+import org.apache.phoenix.exception.SQLExceptionCode;
+import org.apache.phoenix.exception.SQLExceptionInfo;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
+import org.apache.phoenix.parse.HintNode.Hint;
+import org.apache.phoenix.query.ConnectionQueryServices;
+import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.schema.AmbiguousColumnException;
+import org.apache.phoenix.schema.ColumnFamilyNotFoundException;
+import org.apache.phoenix.schema.ColumnModifier;
+import org.apache.phoenix.schema.ColumnNotFoundException;
+import org.apache.phoenix.schema.PColumn;
+import org.apache.phoenix.schema.PColumnFamily;
+import org.apache.phoenix.schema.PDataType;
+import org.apache.phoenix.schema.PDatum;
+import org.apache.phoenix.schema.PMetaData;
+import org.apache.phoenix.schema.PName;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTableType;
+import org.apache.phoenix.schema.RowKeySchema;
+import org.apache.phoenix.schema.RowKeySchema.RowKeySchemaBuilder;
+import org.apache.phoenix.schema.SaltingUtil;
+
+
+
+/**
+ *
+ * Static class for various schema-related utilities
+ *
+ * @author jtaylor
+ * @since 0.1
+ */
+public class SchemaUtil {
+ private static final Logger logger = LoggerFactory.getLogger(SchemaUtil.class);
+ private static final int VAR_LENGTH_ESTIMATE = 10;
+
+ public static final DataBlockEncoding DEFAULT_DATA_BLOCK_ENCODING = DataBlockEncoding.FAST_DIFF;
+ public static final RowKeySchema VAR_BINARY_SCHEMA = new RowKeySchemaBuilder(1).addField(new PDatum() {
+
+ @Override
+ public boolean isNullable() {
+ return false;
+ }
+
+ @Override
+ public PDataType getDataType() {
+ return PDataType.VARBINARY;
+ }
+
+ @Override
+ public Integer getByteSize() {
+ return null;
+ }
+
+ @Override
+ public Integer getMaxLength() {
+ return null;
+ }
+
+ @Override
+ public Integer getScale() {
+ return null;
+ }
+
+ @Override
+ public ColumnModifier getColumnModifier() {
+ return null;
+ }
+
+ }, false, null).build();
+
+ /**
+ * May not be instantiated
+ */
+ private SchemaUtil() {
+ }
+
+ /**
+ * Join srcRow KeyValues to dstRow KeyValues. For single-column mode, in the case of multiple
+ * column families (which can happen for our conceptual PK to PK joins), we always use the
+ * column family from the first dstRow. TODO: we'll likely need a different coprocessor to
+ * handle the PK join case.
+ * @param srcRow the source list of KeyValues to join with dstRow KeyValues. The key will be
+ * changed upon joining to match the dstRow key. The list may not be empty and is left unchanged.
+ * @param dstRow the list of KeyValues to which the srcRows are joined. The list is modified in
+ * place and may not be empty.
+ */
+ public static void joinColumns(List<KeyValue> srcRow, List<KeyValue> dstRow) {
+ assert(!dstRow.isEmpty());
+ KeyValue dst = dstRow.get(0);
+ byte[] dstBuf = dst.getBuffer();
+ int dstKeyOffset = dst.getRowOffset();
+ int dstKeyLength = dst.getRowLength();
+ // Combine columns from both rows
+ // The key for the cached KeyValues are modified to match the other key.
+ for (KeyValue srcValue : srcRow) {
+ byte[] srcBuf = srcValue.getBuffer();
+ byte type = srcValue.getType();
+ KeyValue dstValue = new KeyValue(dstBuf, dstKeyOffset, dstKeyLength,
+ srcBuf, srcValue.getFamilyOffset(), srcValue.getFamilyLength(),
+ srcBuf, srcValue.getQualifierOffset(), srcValue.getQualifierLength(),
+ HConstants.LATEST_TIMESTAMP, KeyValue.Type.codeToType(type),
+ srcBuf, srcValue.getValueOffset(), srcValue.getValueLength());
+ dstRow.add(dstValue);
+ }
+ // Put KeyValues in proper sort order
+ // TODO: our tests need this, but otherwise would this be required?
+ Collections.sort(dstRow, KeyValue.COMPARATOR);
+ }
+
+ /**
+ * Get the column value of a row.
+ * @param result the Result return from iterating through scanner results.
+ * @param fam the column family
+ * @param col the column qualifier
+ * @param pos the column position
+ * @param value updated in place to the bytes representing the column value
+ * @return true if the column exists and value was set and false otherwise.
+ */
+ public static boolean getValue(Result result, byte[] fam, byte[] col, int pos, ImmutableBytesWritable value) {
+ KeyValue keyValue = ResultUtil.getColumnLatest(result, fam, col);
+ if (keyValue != null) {
+ value.set(keyValue.getBuffer(),keyValue.getValueOffset(),keyValue.getValueLength());
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Concatenate two PColumn arrays
+ * @param first first array
+ * @param second second array
+ * @return new combined array
+ */
+ public static PColumn[] concat(PColumn[] first, PColumn[] second) {
+ PColumn[] result = new PColumn[first.length + second.length];
+ System.arraycopy(first, 0, result, 0, first.length);
+ System.arraycopy(second, 0, result, first.length, second.length);
+ return result;
+ }
+
+ public static boolean isPKColumn(PColumn column) {
+ return column.getFamilyName() == null;
+ }
+
+ /**
+ * Estimate the max key length in bytes of the PK for a given table
+ * @param table the table
+ * @return the max PK length
+ */
+ public static int estimateKeyLength(PTable table) {
+ int maxKeyLength = 0;
+ // Calculate the max length of a key (each part must currently be of a fixed width)
+ int i = 0;
+ List<PColumn> columns = table.getPKColumns();
+ while (i < columns.size()) {
+ PColumn keyColumn = columns.get(i++);
+ Integer byteSize = keyColumn.getByteSize();
+ maxKeyLength += (byteSize == null) ? VAR_LENGTH_ESTIMATE : byteSize;
+ }
+ return maxKeyLength;
+ }
+
+ /**
+ * Normalize an identifier. If name is surrounded by double quotes,
+ * it is used as-is, otherwise the name is upper caased.
+ * @param name the parsed identifier
+ * @return the normalized identifier
+ */
+ public static String normalizeIdentifier(String name) {
+ if (name == null) {
+ return name;
+ }
+ if (isCaseSensitive(name)) {
+ // Don't upper case if in quotes
+ return name.substring(1, name.length()-1);
+ }
+ return name.toUpperCase();
+ }
+
+ public static boolean isCaseSensitive(String name) {
+ return name.length() > 0 && name.charAt(0)=='"';
+ }
+
+ public static <T> List<T> concat(List<T> l1, List<T> l2) {
+ int size1 = l1.size();
+ if (size1 == 0) {
+ return l2;
+ }
+ int size2 = l2.size();
+ if (size2 == 0) {
+ return l1;
+ }
+ List<T> l3 = new ArrayList<T>(size1 + size2);
+ l3.addAll(l1);
+ l3.addAll(l2);
+ return l3;
+ }
+
+ /**
+ * Get the key used in the Phoenix metadata row for a table definition
+ * @param schemaName
+ * @param tableName
+ */
+ public static byte[] getTableKey(byte[] schemaName, byte[] tableName) {
+ return ByteUtil.concat(schemaName, QueryConstants.SEPARATOR_BYTE_ARRAY, tableName);
+ }
+
+ public static byte[] getTableKey(String schemaName, String tableName) {
+ return ByteUtil.concat(schemaName == null ? ByteUtil.EMPTY_BYTE_ARRAY : Bytes.toBytes(schemaName), QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes(tableName));
+ }
+
+ public static String getTableName(String schemaName, String tableName) {
+ return getName(schemaName,tableName);
+ }
+
+ private static String getName(String optionalQualifier, String name) {
+ if (optionalQualifier == null || optionalQualifier.isEmpty()) {
+ return name;
+ }
+ return optionalQualifier + QueryConstants.NAME_SEPARATOR + name;
+ }
+
+ public static String getTableName(byte[] schemaName, byte[] tableName) {
+ return getName(schemaName, tableName);
+ }
+
+ public static String getColumnDisplayName(byte[] cf, byte[] cq) {
+ return getName(cf == null || cf.length == 0 || Bytes.compareTo(cf, QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES) == 0 ? ByteUtil.EMPTY_BYTE_ARRAY : cf, cq);
+ }
+
+ public static String getColumnDisplayName(String cf, String cq) {
+ return getName(cf == null || cf.isEmpty() || QueryConstants.DEFAULT_COLUMN_FAMILY.equals(cf) ? null : cf, cq);
+ }
+
+ public static String getMetaDataEntityName(String schemaName, String tableName, String familyName, String columnName) {
+ if ((schemaName == null || schemaName.isEmpty()) && (tableName == null || tableName.isEmpty())) {
+ return getName(familyName, columnName);
+ }
+ if ((familyName == null || familyName.isEmpty()) && (columnName == null || columnName.isEmpty())) {
+ return getName(schemaName, tableName);
+ }
+ return getName(getName(schemaName, tableName), getName(familyName, columnName));
+ }
+
+ public static String getColumnName(String familyName, String columnName) {
+ return getName(familyName, columnName);
+ }
+
+ public static byte[] getTableNameAsBytes(String schemaName, String tableName) {
+ if (schemaName == null || schemaName.length() == 0) {
+ return StringUtil.toBytes(tableName);
+ }
+ return getTableNameAsBytes(StringUtil.toBytes(schemaName),StringUtil.toBytes(tableName));
+ }
+
+ public static byte[] getTableNameAsBytes(byte[] schemaName, byte[] tableName) {
+ return getNameAsBytes(schemaName, tableName);
+ }
+
+ private static byte[] getNameAsBytes(byte[] nameOne, byte[] nameTwo) {
+ if (nameOne == null || nameOne.length == 0) {
+ return nameTwo;
+ } else if ((nameTwo == null || nameTwo.length == 0)) {
+ return nameOne;
+ } else {
+ return ByteUtil.concat(nameOne, QueryConstants.NAME_SEPARATOR_BYTES, nameTwo);
+ }
+ }
+
+ public static String getName(byte[] nameOne, byte[] nameTwo) {
+ return Bytes.toString(getNameAsBytes(nameOne,nameTwo));
+ }
+
+ public static int getVarCharLength(byte[] buf, int keyOffset, int maxLength) {
+ return getVarCharLength(buf, keyOffset, maxLength, 1);
+ }
+
+ public static int getVarCharLength(byte[] buf, int keyOffset, int maxLength, int skipCount) {
+ int length = 0;
+ for (int i=0; i<skipCount; i++) {
+ while (length < maxLength && buf[keyOffset+length] != QueryConstants.SEPARATOR_BYTE) {
+ length++;
+ }
+ if (i != skipCount-1) { // skip over the separator if it's not the last one.
+ length++;
+ }
+ }
+ return length;
+ }
+
+ public static int getVarChars(byte[] rowKey, byte[][] rowKeyMetadata) {
+ return getVarChars(rowKey, 0, rowKey.length, 0, rowKeyMetadata);
+ }
+
+ public static int getVarChars(byte[] rowKey, int colMetaDataLength, byte[][] colMetaData) {
+ return getVarChars(rowKey, 0, rowKey.length, 0, colMetaDataLength, colMetaData);
+ }
+
+ public static int getVarChars(byte[] rowKey, int keyOffset, int keyLength, int colMetaDataOffset, byte[][] colMetaData) {
+ return getVarChars(rowKey, keyOffset, keyLength, colMetaDataOffset, colMetaData.length, colMetaData);
+ }
+
+ public static int getVarChars(byte[] rowKey, int keyOffset, int keyLength, int colMetaDataOffset, int colMetaDataLength, byte[][] colMetaData) {
+ int i, offset = keyOffset;
+ for (i = colMetaDataOffset; i < colMetaDataLength && keyLength > 0; i++) {
+ int length = getVarCharLength(rowKey, offset, keyLength);
+ byte[] b = new byte[length];
+ System.arraycopy(rowKey, offset, b, 0, length);
+ offset += length + 1;
+ keyLength -= length + 1;
+ colMetaData[i] = b;
+ }
+ return i;
+ }
+
+ public static String findExistingColumn(PTable table, List<PColumn> columns) {
+ for (PColumn column : columns) {
+ PName familyName = column.getFamilyName();
+ if (familyName == null) {
+ try {
+ return table.getPKColumn(column.getName().getString()).getName().getString();
+ } catch (ColumnNotFoundException e) {
+ continue;
+ }
+ } else {
+ try {
+ return table.getColumnFamily(familyName.getString()).getColumn(column.getName().getString()).getName().getString();
+ } catch (ColumnFamilyNotFoundException e) {
+ continue; // Shouldn't happen
+ } catch (ColumnNotFoundException e) {
+ continue;
+ }
+ }
+ }
+ return null;
+ }
+
+ public static String toString(byte[][] values) {
+ if (values == null) {
+ return "null";
+ }
+ StringBuilder buf = new StringBuilder("[");
+ for (byte[] value : values) {
+ buf.append(Bytes.toStringBinary(value));
+ buf.append(',');
+ }
+ buf.setCharAt(buf.length()-1, ']');
+ return buf.toString();
+ }
+
+ public static String toString(PDataType type, byte[] value) {
+ boolean isString = type.isCoercibleTo(PDataType.VARCHAR);
+ return isString ? ("'" + type.toObject(value).toString() + "'") : type.toObject(value).toString();
+ }
+
+ public static byte[] getEmptyColumnFamily(List<PColumnFamily> families) {
+ return families.isEmpty() ? QueryConstants.EMPTY_COLUMN_BYTES : families.get(0).getName().getBytes();
+ }
+
+ public static boolean isMetaTable(byte[] tableName) {
+ return Bytes.compareTo(tableName, TYPE_TABLE_NAME_BYTES) == 0;
+ }
+
+ public static boolean isMetaTable(String schemaName, String tableName) {
+ return PhoenixDatabaseMetaData.TYPE_SCHEMA.equals(schemaName) && PhoenixDatabaseMetaData.TYPE_TABLE.equals(tableName);
+ }
+
+ // Given the splits and the rowKeySchema, find out the keys that
+ public static byte[][] processSplits(byte[][] splits, LinkedHashSet<PColumn> pkColumns, Integer saltBucketNum, boolean defaultRowKeyOrder) throws SQLException {
+ // FIXME: shouldn't this return if splits.length == 0?
+ if (splits == null) return null;
+ // We do not accept user specified splits if the table is salted and we specify defaultRowKeyOrder. In this case,
+ // throw an exception.
+ if (splits.length > 0 && saltBucketNum != null && defaultRowKeyOrder) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.NO_SPLITS_ON_SALTED_TABLE).build().buildException();
+ }
+ // If the splits are not specified and table is salted, pre-split the table.
+ if (splits.length == 0 && saltBucketNum != null) {
+ splits = SaltingUtil.getSalteByteSplitPoints(saltBucketNum);
+ }
+ byte[][] newSplits = new byte[splits.length][];
+ for (int i=0; i<splits.length; i++) {
+ newSplits[i] = processSplit(splits[i], pkColumns);
+ }
+ return newSplits;
+ }
+
+ // Go through each slot in the schema and try match it with the split byte array. If the split
+ // does not confer to the schema, extends its length to match the schema.
+ private static byte[] processSplit(byte[] split, LinkedHashSet<PColumn> pkColumns) {
+ int pos = 0, offset = 0, maxOffset = split.length;
+ Iterator<PColumn> iterator = pkColumns.iterator();
+ while (pos < pkColumns.size()) {
+ PColumn column = iterator.next();
+ if (column.getDataType().isFixedWidth()) { // Fixed width
+ int length = column.getByteSize();
+ if (maxOffset - offset < length) {
+ // The split truncates the field. Fill in the rest of the part and any fields that
+ // are missing after this field.
+ int fillInLength = length - (maxOffset - offset);
+ fillInLength += estimatePartLength(pos + 1, iterator);
+ return ByteUtil.fillKey(split, split.length + fillInLength);
+ }
+ // Account for this field, move to next position;
+ offset += length;
+ pos++;
+ } else { // Variable length
+ // If we are the last slot, then we are done. Nothing needs to be filled in.
+ if (pos == pkColumns.size() - 1) {
+ break;
+ }
+ while (offset < maxOffset && split[offset] != QueryConstants.SEPARATOR_BYTE) {
+ offset++;
+ }
+ if (offset == maxOffset) {
+ // The var-length field does not end with a separator and it's not the last field.
+ int fillInLength = 1; // SEPARATOR byte for the current var-length slot.
+ fillInLength += estimatePartLength(pos + 1, iterator);
+ return ByteUtil.fillKey(split, split.length + fillInLength);
+ }
+ // Move to the next position;
+ offset += 1; // skip separator;
+ pos++;
+ }
+ }
+ return split;
+ }
+
+ // Estimate the key length after pos slot for schema.
+ private static int estimatePartLength(int pos, Iterator<PColumn> iterator) {
+ int length = 0;
+ while (iterator.hasNext()) {
+ PColumn column = iterator.next();
+ if (column.getDataType().isFixedWidth()) {
+ length += column.getByteSize();
+ } else {
+ length += 1; // SEPARATOR byte.
+ }
+ }
+ return length;
+ }
+
+ public static final String UPGRADE_TO_2_0 = "UpgradeTo20";
+ public static final Integer SYSTEM_TABLE_NULLABLE_VAR_LENGTH_COLUMNS = 3;
+ public static final String UPGRADE_TO_2_1 = "UpgradeTo21";
+
+ public static boolean isUpgradeTo2Necessary(ConnectionQueryServices connServices) throws SQLException {
+ HTableInterface htable = connServices.getTable(PhoenixDatabaseMetaData.TYPE_TABLE_NAME_BYTES);
+ try {
+ return (htable.getTableDescriptor().getValue(SchemaUtil.UPGRADE_TO_2_0) == null);
+ } catch (IOException e) {
+ throw new SQLException(e);
+ }
+ }
+
+ public static void upgradeTo2IfNecessary(HRegion region, int nColumns) throws IOException {
+ Scan scan = new Scan();
+ scan.setRaw(true);
+ scan.setMaxVersions(MetaDataProtocol.DEFAULT_MAX_META_DATA_VERSIONS);
+ RegionScanner scanner = region.getScanner(scan);
+ int batchSizeBytes = 100 * 1024; // 100K chunks
+ int sizeBytes = 0;
+ List<Pair<Mutation,Integer>> mutations = Lists.newArrayListWithExpectedSize(10000);
+ MultiVersionConsistencyControl.setThreadReadPoint(scanner.getMvccReadPoint());
+ region.startRegionOperation();
+ try {
+ List<KeyValue> result;
+ do {
+ result = Lists.newArrayList();
+ scanner.nextRaw(result, null);
+ for (KeyValue keyValue : result) {
+ KeyValue newKeyValue = SchemaUtil.upgradeTo2IfNecessary(nColumns, keyValue);
+ if (newKeyValue != null) {
+ sizeBytes += newKeyValue.getLength();
+ if (Type.codeToType(newKeyValue.getType()) == Type.Put) {
+ // Delete old value
+ byte[] buf = keyValue.getBuffer();
+ Delete delete = new Delete(keyValue.getRow());
+ KeyValue deleteKeyValue = new KeyValue(buf, keyValue.getRowOffset(), keyValue.getRowLength(),
+ buf, keyValue.getFamilyOffset(), keyValue.getFamilyLength(),
+ buf, keyValue.getQualifierOffset(), keyValue.getQualifierLength(),
+ keyValue.getTimestamp(), Type.Delete,
+ ByteUtil.EMPTY_BYTE_ARRAY,0,0);
+ delete.addDeleteMarker(deleteKeyValue);
+ mutations.add(new Pair<Mutation,Integer>(delete,null));
+ sizeBytes += deleteKeyValue.getLength();
+ // Put new value
+ Put put = new Put(newKeyValue.getRow());
+ put.add(newKeyValue);
+ mutations.add(new Pair<Mutation,Integer>(put,null));
+ } else if (Type.codeToType(newKeyValue.getType()) == Type.Delete){
+ // Copy delete marker using new key so that it continues
+ // to delete the key value preceding it that will be updated
+ // as well.
+ Delete delete = new Delete(newKeyValue.getRow());
+ delete.addDeleteMarker(newKeyValue);
+ mutations.add(new Pair<Mutation,Integer>(delete,null));
+ }
+ if (sizeBytes >= batchSizeBytes) {
+ commitBatch(region, mutations);
+ mutations.clear();
+ sizeBytes = 0;
+ }
+
+ }
+ }
+ } while (!result.isEmpty());
+ commitBatch(region, mutations);
+ } finally {
+ region.closeRegionOperation();
+ }
+ }
+
+ private static void commitBatch(HRegion region, List<Pair<Mutation,Integer>> mutations) throws IOException {
+ if (mutations.isEmpty()) {
+ return;
+ }
+ @SuppressWarnings("unchecked")
+ Pair<Mutation,Integer>[] mutationArray = new Pair[mutations.size()];
+ // TODO: should we use the one that is all or none?
+ region.batchMutate(mutations.toArray(mutationArray));
+ }
+
+ private static KeyValue upgradeTo2IfNecessary(int maxSeparators, KeyValue keyValue) {
+ int originalLength = keyValue.getRowLength();
+ int length = originalLength;
+ int offset = keyValue.getRowOffset();
+ byte[] buf = keyValue.getBuffer();
+ while (originalLength - length < maxSeparators && buf[offset+length-1] == QueryConstants.SEPARATOR_BYTE) {
+ length--;
+ }
+ if (originalLength == length) {
+ return null;
+ }
+ return new KeyValue(buf, offset, length,
+ buf, keyValue.getFamilyOffset(), keyValue.getFamilyLength(),
+ buf, keyValue.getQualifierOffset(), keyValue.getQualifierLength(),
+ keyValue.getTimestamp(), Type.codeToType(keyValue.getType()),
+ buf, keyValue.getValueOffset(), keyValue.getValueLength());
+ }
+
+ public static int upgradeColumnCount(String url, Properties info) throws SQLException {
+ String upgradeStr = JDBCUtil.findProperty(url, info, UPGRADE_TO_2_0);
+ return (upgradeStr == null ? 0 : Integer.parseInt(upgradeStr));
+ }
+
+ public static boolean checkIfUpgradeTo2Necessary(ConnectionQueryServices connectionQueryServices, String url,
+ Properties info) throws SQLException {
+ boolean isUpgrade = upgradeColumnCount(url, info) > 0;
+ boolean isUpgradeNecessary = isUpgradeTo2Necessary(connectionQueryServices);
+ if (!isUpgrade && isUpgradeNecessary) {
+ throw new SQLException("Please run the upgrade script in bin/updateTo2.sh to ensure your data is converted correctly to the 2.0 format");
+ }
+ if (isUpgrade && !isUpgradeNecessary) {
+ info.remove(SchemaUtil.UPGRADE_TO_2_0); // Remove this property and ignore, since upgrade has already been done
+ return false;
+ }
+ return isUpgradeNecessary;
+ }
+
+ public static String getEscapedTableName(String schemaName, String tableName) {
+ if (schemaName == null || schemaName.length() == 0) {
+ return "\"" + tableName + "\"";
+ }
+ return "\"" + schemaName + "\"." + "\"" + tableName + "\"";
+ }
+
+ private static PhoenixConnection addMetaDataColumn(PhoenixConnection conn, long scn, String columnDef) throws SQLException {
+ String url = conn.getURL();
+ Properties props = conn.getClientInfo();
+ PMetaData metaData = conn.getPMetaData();
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(scn));
+ PhoenixConnection metaConnection = null;
+
+ Statement stmt = null;
+ try {
+ metaConnection = new PhoenixConnection(conn.getQueryServices(), url, props, metaData);
+ try {
+ stmt = metaConnection.createStatement();
+ stmt.executeUpdate("ALTER TABLE SYSTEM.\"TABLE\" ADD IF NOT EXISTS " + columnDef);
+ return metaConnection;
+ } finally {
+ if(stmt != null) {
+ stmt.close();
+ }
+ }
+ } finally {
+ if(metaConnection != null) {
+ metaConnection.close();
+ }
+ }
+ }
+
+ public static boolean columnExists(PTable table, String columnName) {
+ try {
+ table.getColumn(columnName);
+ return true;
+ } catch (ColumnNotFoundException e) {
+ return false;
+ } catch (AmbiguousColumnException e) {
+ return true;
+ }
+ }
+
+ public static void updateSystemTableTo2(PhoenixConnection metaConnection, PTable table) throws SQLException {
+ PTable metaTable = metaConnection.getPMetaData().getTable(TYPE_TABLE_NAME);
+ // Execute alter table statement for each column that was added if not already added
+ if (table.getTimeStamp() < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP - 1) {
+ // Causes row key of system table to be upgraded
+ if (checkIfUpgradeTo2Necessary(metaConnection.getQueryServices(), metaConnection.getURL(), metaConnection.getClientInfo())) {
+ metaConnection.createStatement().executeQuery("select count(*) from " + PhoenixDatabaseMetaData.TYPE_SCHEMA_AND_TABLE).next();
+ }
+
+ if (metaTable.getTimeStamp() < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP - 5 && !columnExists(table, COLUMN_MODIFIER)) {
+ metaConnection = addMetaDataColumn(metaConnection, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP - 5, COLUMN_MODIFIER + " INTEGER NULL");
+ }
+ if (metaTable.getTimeStamp() < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP - 4 && !columnExists(table, SALT_BUCKETS)) {
+ metaConnection = addMetaDataColumn(metaConnection, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP - 4, SALT_BUCKETS + " INTEGER NULL");
+ }
+ if (metaTable.getTimeStamp() < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP - 3 && !columnExists(table, DATA_TABLE_NAME)) {
+ metaConnection = addMetaDataColumn(metaConnection, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP - 3, DATA_TABLE_NAME + " VARCHAR NULL");
+ }
+ if (metaTable.getTimeStamp() < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP - 2 && !columnExists(table, INDEX_STATE)) {
+ metaConnection = addMetaDataColumn(metaConnection, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP - 2, INDEX_STATE + " VARCHAR NULL");
+ }
+ if (!columnExists(table, IMMUTABLE_ROWS)) {
+ addMetaDataColumn(metaConnection, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP - 1, IMMUTABLE_ROWS + " BOOLEAN NULL");
+ }
+ }
+ }
+
+ public static void upgradeTo2(PhoenixConnection conn) throws SQLException {
+ // Filter VIEWs from conversion
+ StringBuilder buf = new StringBuilder();
+ ResultSet rs = conn.getMetaData().getTables(null, null, null, new String[] {PTableType.VIEW.getSerializedValue()});
+ while (rs.next()) {
+ buf.append("(" + TABLE_SCHEM_NAME + " = " + rs.getString(2) + " and " + TABLE_NAME_NAME + " = " + rs.getString(3) + ") or ");
+ }
+ String filterViews = "";
+ if (buf.length() > 0) {
+ buf.setLength(buf.length() - " or ".length());
+ filterViews = " and not (" + buf.toString() + ")";
+ }
+ /*
+ * Our upgrade hack sets a property on the scan that gets activated by an ungrouped aggregate query.
+ * Our UngroupedAggregateRegionObserver coprocessors will perform the required upgrade.
+ * The SYSTEM.TABLE will already have been upgraded, so walk through each user table and upgrade
+ * any table with a nullable variable length column (VARCHAR or DECIMAL)
+ * at the end.
+ */
+ String query = "select /*+" + Hint.NO_INTRA_REGION_PARALLELIZATION + "*/ " +
+ TABLE_SCHEM_NAME + "," +
+ TABLE_NAME_NAME + " ," +
+ DATA_TYPE + "," +
+ NULLABLE +
+ " from " + TYPE_SCHEMA_AND_TABLE +
+ " where " + TABLE_CAT_NAME + " is null " +
+ " and " + COLUMN_NAME + " is not null " + filterViews +
+ " order by " + TABLE_SCHEM_NAME + "," + TABLE_NAME_NAME + "," + ORDINAL_POSITION + " DESC";
+ rs = conn.createStatement().executeQuery(query);
+ String currentTableName = null;
+ int nColumns = 0;
+ boolean skipToNext = false;
+ Map<String,Integer> tablesToUpgrade = Maps.newHashMap();
+ while (rs.next()) {
+ String tableName = getEscapedTableName(rs.getString(1), rs.getString(2));
+ if (currentTableName == null) {
+ currentTableName = tableName;
+ } else if (!currentTableName.equals(tableName)) {
+ if (nColumns > 0) {
+ tablesToUpgrade.put(currentTableName, nColumns);
+ nColumns = 0;
+ }
+ currentTableName = tableName;
+ skipToNext = false;
+ } else if (skipToNext) {
+ continue;
+ }
+ PDataType dataType = PDataType.fromSqlType(rs.getInt(3));
+ if (dataType.isFixedWidth() || rs.getInt(4) != DatabaseMetaData.attributeNullable) {
+ skipToNext = true;
+ } else {
+ nColumns++;
+ }
+ }
+
+ for (Map.Entry<String, Integer> entry : tablesToUpgrade.entrySet()) {
+ String msg = "Upgrading " + entry.getKey() + " for " + entry.getValue() + " columns";
+ logger.info(msg);
+ System.out.println(msg);
+ Properties props = new Properties(conn.getClientInfo());
+ props.setProperty(UPGRADE_TO_2_0, entry.getValue().toString());
+ Connection newConn = DriverManager.getConnection(conn.getURL(), props);
+ try {
+ rs = newConn.createStatement().executeQuery("select /*+" + Hint.NO_INTRA_REGION_PARALLELIZATION + "*/ count(*) from " + entry.getKey());
+ rs.next();
+ } finally {
+ newConn.close();
+ }
+ }
+
+ ConnectionQueryServices connServices = conn.getQueryServices();
+ HTableInterface htable = null;
+ HBaseAdmin admin = connServices.getAdmin();
+ try {
+ htable = connServices.getTable(PhoenixDatabaseMetaData.TYPE_TABLE_NAME_BYTES);
+ HTableDescriptor htd = new HTableDescriptor(htable.getTableDescriptor());
+ htd.setValue(SchemaUtil.UPGRADE_TO_2_0, Boolean.TRUE.toString());
+ admin.disableTable(PhoenixDatabaseMetaData.TYPE_TABLE_NAME_BYTES);
+ admin.modifyTable(PhoenixDatabaseMetaData.TYPE_TABLE_NAME_BYTES, htd);
+ admin.enableTable(PhoenixDatabaseMetaData.TYPE_TABLE_NAME_BYTES);
+ } catch (IOException e) {
+ throw new SQLException(e);
+ } finally {
+ try {
+ try {
+ admin.close();
+ } finally {
+ if(htable != null) {
+ htable.close();
+ }
+ }
+ } catch (IOException e) {
+ throw new SQLException(e);
+ }
+ }
+ }
+
+ public static String getSchemaNameFromFullName(String tableName) {
+ int index = tableName.indexOf(QueryConstants.NAME_SEPARATOR);
+ if (index < 0) {
+ return "";
+ }
+ return tableName.substring(0, index);
+ }
+
+ public static byte[] getTableKeyFromFullName(String fullTableName) {
+ int index = fullTableName.indexOf(QueryConstants.NAME_SEPARATOR);
+ if (index < 0) {
+ return getTableKey(null, fullTableName);
+ }
+ String schemaName = fullTableName.substring(0, index);
+ String tableName = fullTableName.substring(index+1);
+ return getTableKey(schemaName, tableName);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/c5b80246/src/main/java/org/apache/phoenix/util/ServerUtil.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/phoenix/util/ServerUtil.java b/src/main/java/org/apache/phoenix/util/ServerUtil.java
new file mode 100644
index 0000000..819bcc4
--- /dev/null
+++ b/src/main/java/org/apache/phoenix/util/ServerUtil.java
@@ -0,0 +1,123 @@
+/*
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.util;
+
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.hadoop.hbase.DoNotRetryIOException;
+
+import org.apache.phoenix.exception.PhoenixIOException;
+import org.apache.phoenix.exception.SQLExceptionCode;
+
+
+public class ServerUtil {
+
+ private static final String FORMAT = "ERROR %d (%s): %s";
+ private static final Pattern PATTERN = Pattern.compile("ERROR (\\d+) \\((\\w+)\\): (.*)");
+ private static final Map<Class<? extends Exception>, SQLExceptionCode> errorcodeMap
+ = new HashMap<Class<? extends Exception>, SQLExceptionCode>();
+ static {
+ // Map a normal exception into a corresponding SQLException.
+ errorcodeMap.put(ArithmeticException.class, SQLExceptionCode.SERVER_ARITHMETIC_ERROR);
+ }
+
+ public static void throwIOException(String msg, Throwable t) throws IOException {
+ // First unwrap SQLExceptions if it's root cause is an IOException.
+ if (t instanceof SQLException) {
+ Throwable cause = t.getCause();
+ if (cause instanceof IOException) {
+ t = cause;
+ }
+ }
+ // Throw immediately if DoNotRetryIOException
+ if (t instanceof DoNotRetryIOException) {
+ throw (DoNotRetryIOException)t;
+ } else if (t instanceof IOException) {
+ // If the IOException does not wrap any exception, then bubble it up.
+ Throwable cause = t.getCause();
+ if (cause == null || cause instanceof IOException) {
+ throw (IOException)t;
+ }
+ // Else assume it's been wrapped, so throw as DoNotRetryIOException to prevent client hanging while retrying
+ throw new DoNotRetryIOException(t.getMessage(), cause);
+ } else if (t instanceof SQLException) {
+ // If it's already an SQLException, construct an error message so we can parse and reconstruct on the client side.
+ throw new DoNotRetryIOException(constructSQLErrorMessage((SQLException) t, msg), t);
+ } else {
+ // Not a DoNotRetryIOException, IOException or SQLException. Map the exception type to a general SQLException
+ // and construct the error message so it can be reconstruct on the client side.
+ //
+ // If no mapping exists, rethrow it as a generic exception.
+ SQLExceptionCode code = errorcodeMap.get(t.getClass());
+ if (code == null) {
+ throw new DoNotRetryIOException(msg + ": " + t.getMessage(), t);
+ } else {
+ throw new DoNotRetryIOException(constructSQLErrorMessage(code, t, msg), t);
+ }
+ }
+ }
+
+ private static String constructSQLErrorMessage(SQLExceptionCode code, Throwable e, String message) {
+ return constructSQLErrorMessage(code.getErrorCode(), code.getSQLState(), code.getMessage() + " " + e.getMessage() + " " + message);
+ }
+
+ private static String constructSQLErrorMessage(SQLException e, String message) {
+ return constructSQLErrorMessage(e.getErrorCode(), e.getSQLState(), e.getMessage() + " " + message);
+ }
+
+ private static String constructSQLErrorMessage(int errorCode, String SQLState, String message) {
+ return String.format(FORMAT, errorCode, SQLState, message);
+ }
+
+ public static SQLException parseServerException(Throwable t) {
+ SQLException e = parseServerExceptionOrNull(t);
+ if (e != null) {
+ return e;
+ }
+ return new PhoenixIOException(t);
+ }
+
+ public static SQLException parseServerExceptionOrNull(Throwable t) {
+ while (t.getCause() != null) {
+ t = t.getCause();
+ }
+ return parseRemoteException(t);
+ }
+
+ private static SQLException parseRemoteException(Throwable t) {
+ String message = t.getLocalizedMessage();
+ if (message != null) {
+ // If the message matches the standard pattern, recover the SQLException and throw it.
+ Matcher matcher = PATTERN.matcher(t.getLocalizedMessage());
+ if (matcher.find()) {
+ int errorCode = Integer.parseInt(matcher.group(1));
+ String sqlState = matcher.group(2);
+ return new SQLException(matcher.group(), sqlState, errorCode, t);
+ }
+ }
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/c5b80246/src/main/java/org/apache/phoenix/util/SizedUtil.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/phoenix/util/SizedUtil.java b/src/main/java/org/apache/phoenix/util/SizedUtil.java
new file mode 100644
index 0000000..62f0816
--- /dev/null
+++ b/src/main/java/org/apache/phoenix/util/SizedUtil.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.util;
+
+
+/**
+ * Utilities for computing an object's size. All size measurements are in bytes.
+ * Note, all of the sizes here, but especially OBJECT_SIZE and ARRAY_SIZE are estimates and will
+ * depend on the JVM itself (and which JVM, 64bit vs. 32bit, etc).
+ * The current values are based on:
+ * Java HotSpot(TM) 64-Bit Server VM/14.2-b01
+ *
+ * (Uncomment out and run the main w/ appropriate object to test)
+ * Also, see this link:
+ * https://sites.google.com/a/salesforce.com/development/Home/old-wiki-home-page/i-wish-i-knew#TOC-How-to-measure-the-size-of-a-Java-O
+ * For another way to measure.
+ */
+public class SizedUtil {
+ public static final int POINTER_SIZE = 8; // 64 bit jvm.
+ public static final int OBJECT_SIZE = 16; // measured, see class comment.
+ public static final int ARRAY_SIZE = 24; // measured, see class comment.
+ public static final int CHAR_SIZE = 2;
+ public static final int INT_SIZE = 4;
+ public static final int LONG_SIZE = 8;
+
+ public static final int MAP_ENTRY_SIZE = OBJECT_SIZE + 3 * POINTER_SIZE + INT_SIZE;
+ public static final int IMMUTABLE_BYTES_WRITABLE_SIZE = OBJECT_SIZE + INT_SIZE * 2 + ARRAY_SIZE;
+ public static final int IMMUTABLE_BYTES_PTR_SIZE = IMMUTABLE_BYTES_WRITABLE_SIZE + INT_SIZE;// Extra is an int field which caches hashcode.
+ public static final int KEY_VALUE_SIZE = 2 * INT_SIZE + LONG_SIZE + 2 * ARRAY_SIZE;
+ public static final int RESULT_SIZE = OBJECT_SIZE + 3 * POINTER_SIZE + IMMUTABLE_BYTES_WRITABLE_SIZE;
+ public static final int INT_OBJECT_SIZE = INT_SIZE + OBJECT_SIZE;
+ public static final int LONG_OBJECT_SIZE = LONG_SIZE + OBJECT_SIZE;
+ public static final int BIG_DECIMAL_SIZE =
+ OBJECT_SIZE + 2 * INT_SIZE + LONG_SIZE + 2 * POINTER_SIZE +
+ OBJECT_SIZE /* BigInteger */ + 5 * INT_SIZE + ARRAY_SIZE /*mag[]*/ + 2 * INT_SIZE /* est mag[2] */;
+
+ private SizedUtil() {
+ }
+
+ public static int sizeOfMap(int nRows, int keySize, int valueSize) {
+ return nRows * (
+ SizedUtil.MAP_ENTRY_SIZE + // entry
+ keySize + // key size
+ valueSize); // value size
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/c5b80246/src/main/java/org/apache/phoenix/util/StringUtil.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/phoenix/util/StringUtil.java b/src/main/java/org/apache/phoenix/util/StringUtil.java
new file mode 100644
index 0000000..876bc50
--- /dev/null
+++ b/src/main/java/org/apache/phoenix/util/StringUtil.java
@@ -0,0 +1,245 @@
+/*
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.util;
+
+import java.io.UnsupportedEncodingException;
+import java.util.Arrays;
+
+import org.apache.hadoop.hbase.util.Bytes;
+
+import org.apache.phoenix.schema.ColumnModifier;
+
+
+public class StringUtil {
+ // Masks to determine how many bytes are in each character
+ // From http://tools.ietf.org/html/rfc3629#section-3
+ public static final byte SPACE_UTF8 = 0x20;
+ private static final int BYTES_1_MASK = 0xFF << 7; // 0xxxxxxx is a single byte char
+ private static final int BYTES_2_MASK = 0xFF << 5; // 110xxxxx is a double byte char
+ private static final int BYTES_3_MASK = 0xFF << 4; // 1110xxxx is a triple byte char
+ private static final int BYTES_4_MASK = 0xFF << 3; // 11110xxx is a quadruple byte char
+
+ public static final byte[] MOD_SPACE_UTF8 = new byte[ColumnModifier.values().length];
+ static {
+ for (ColumnModifier columnModifier : ColumnModifier.values()) {
+ MOD_SPACE_UTF8[columnModifier.ordinal()] = columnModifier.apply(new byte[] {SPACE_UTF8}, 0, new byte[1], 0, 1)[0];
+ }
+ }
+
+ public final static char SINGLE_CHAR_WILDCARD = '?';
+ public final static char SINGLE_CHAR_LIKE = '_';
+ public final static char MULTI_CHAR_WILDCARD = '*';
+ public final static char MULTI_CHAR_LIKE = '%';
+ public final static String[] LIKE_ESCAPE_SEQS = new String[]{"\\"+SINGLE_CHAR_LIKE, "\\"+MULTI_CHAR_LIKE};
+ public final static String[] LIKE_UNESCAPED_SEQS = new String[]{""+SINGLE_CHAR_LIKE, ""+MULTI_CHAR_LIKE};
+
+
+ private StringUtil() {
+ }
+
+ /** Replace instances of character ch in String value with String replacement */
+ public static String replaceChar(String value, char ch, CharSequence replacement) {
+ if (value == null)
+ return null;
+ int i = value.indexOf(ch);
+ if (i == -1)
+ return value; // nothing to do
+
+ // we've got at least one character to replace
+ StringBuilder buf = new StringBuilder(value.length() + 16); // some extra space
+ int j = 0;
+ while (i != -1) {
+ buf.append(value, j, i).append(replacement);
+ j = i + 1;
+ i = value.indexOf(ch, j);
+ }
+ if (j < value.length())
+ buf.append(value, j, value.length());
+ return buf.toString();
+ }
+
+ /**
+ * @return the replacement of all occurrences of src[i] with target[i] in s. Src and target are not regex's so this
+ * uses simple searching with indexOf()
+ */
+ public static String replace(String s, String[] src, String[] target) {
+ assert src != null && target != null && src.length > 0 && src.length == target.length;
+ if (src.length == 1 && src[0].length() == 1) {
+ return replaceChar(s, src[0].charAt(0), target[0]);
+ }
+ if (s == null)
+ return null;
+ StringBuilder sb = new StringBuilder(s.length());
+ int pos = 0;
+ int limit = s.length();
+ int lastMatch = 0;
+ while (pos < limit) {
+ boolean matched = false;
+ for (int i = 0; i < src.length; i++) {
+ if (s.startsWith(src[i], pos) && src[i].length() > 0) {
+ // we found a matching pattern - append the acculumation plus the replacement
+ sb.append(s.substring(lastMatch, pos)).append(target[i]);
+ pos += src[i].length();
+ lastMatch = pos;
+ matched = true;
+ break;
+ }
+ }
+ if (!matched) {
+ // we didn't match any patterns, so move forward 1 character
+ pos++;
+ }
+ }
+ // see if we found any matches
+ if (lastMatch == 0) {
+ // we didn't match anything, so return the source string
+ return s;
+ }
+
+ // apppend the trailing portion
+ sb.append(s.substring(lastMatch));
+
+ return sb.toString();
+ }
+
+ public static int getBytesInChar(byte b, ColumnModifier columnModifier) {
+ if (columnModifier != null) {
+ b = columnModifier.apply(b);
+ }
+ int c = b & 0xff;
+ if ((c & BYTES_1_MASK) == 0)
+ return 1;
+ if ((c & BYTES_2_MASK) == 0xC0)
+ return 2;
+ if ((c & BYTES_3_MASK) == 0xE0)
+ return 3;
+ if ((c & BYTES_4_MASK) == 0xF0)
+ return 4;
+ // Any thing else in the first byte is invalid
+ throw new RuntimeException("Undecodable byte: " + b);
+ }
+
+ public static int calculateUTF8Length(byte[] bytes, int offset, int length, ColumnModifier columnModifier) throws UnsupportedEncodingException {
+ int i = offset, endOffset = offset + length;
+ length = 0;
+ while (i < endOffset) {
+ int charLength = getBytesInChar(bytes[i], columnModifier);
+ i += charLength;
+ length++;
+ }
+ return length;
+ }
+
+ // Given an array of bytes containing encoding utf-8 encoded strings, the offset and a length
+ // parameter, return the actual index into the byte array which would represent a substring
+ // of <length> starting from the character at <offset>. We assume the <offset> is the start
+ // byte of an UTF-8 character.
+ public static int getByteLengthForUtf8SubStr(byte[] bytes, int offset, int length, ColumnModifier columnModifier) throws UnsupportedEncodingException {
+ int byteLength = 0;
+ while(length > 0 && offset + byteLength < bytes.length) {
+ int charLength = getBytesInChar(bytes[offset + byteLength], columnModifier);
+ byteLength += charLength;
+ length--;
+ }
+ return byteLength;
+ }
+
+ public static boolean hasMultiByteChars(String s) {
+ for (int i = 0; i < s.length(); i++) {
+ char c = s.charAt(i);
+ if (c > 0x007F) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public static int getFirstNonBlankCharIdxFromStart(byte[] string, int offset, int length, ColumnModifier columnModifier) {
+ int i = offset;
+ byte space = columnModifier == null ? SPACE_UTF8 : MOD_SPACE_UTF8[columnModifier.ordinal()];
+ for ( ; i < offset + length; i++) {
+ if (string[i] != space) {
+ break;
+ }
+ }
+ return i;
+ }
+
+ public static int getFirstNonBlankCharIdxFromEnd(byte[] string, int offset, int length, ColumnModifier columnModifier) {
+ int i = offset + length - 1;
+ byte space = columnModifier == null ? SPACE_UTF8 : MOD_SPACE_UTF8[columnModifier.ordinal()];
+ for ( ; i >= offset; i--) {
+ if (string[i] != space) {
+ break;
+ }
+ }
+ return i;
+ }
+
+ // A toBytes function backed up HBase's utility function, but would accept null input, in which
+ // case it returns an empty byte array.
+ public static byte[] toBytes(String input) {
+ if (input == null) {
+ return ByteUtil.EMPTY_BYTE_ARRAY;
+ }
+ return Bytes.toBytes(input);
+ }
+
+ public static String escapeLike(String s) {
+ return replace(s, LIKE_UNESCAPED_SEQS, LIKE_ESCAPE_SEQS);
+ }
+
+ public static int getUnpaddedCharLength(byte[] b, int offset, int length, ColumnModifier columnModifier) {
+ return getFirstNonBlankCharIdxFromEnd(b, offset, length, columnModifier) - offset + 1;
+ }
+
+ public static byte[] padChar(byte[] value, int offset, int length, int paddedLength) {
+ byte[] key = new byte[paddedLength];
+ System.arraycopy(value,offset, key, 0, length);
+ Arrays.fill(key, length, paddedLength, SPACE_UTF8);
+ return key;
+ }
+
+ public static byte[] padChar(byte[] value, Integer byteSize) {
+ byte[] newValue = Arrays.copyOf(value, byteSize);
+ if (newValue.length > value.length) {
+ Arrays.fill(newValue, value.length, newValue.length, SPACE_UTF8);
+ }
+ return newValue;
+ }
+
+ /**
+ * Lame - StringBuilder.equals is retarded.
+ * @param b1
+ * @param b2
+ * @return whether or not the two builders consist the same sequence of characters
+ */
+ public static boolean equals(StringBuilder b1, StringBuilder b2) {
+ if (b1.length() != b2.length()) {
+ return false;
+ }
+ for (int i = 0; i < b1.length(); i++) {
+ if (b1.charAt(i) != b2.charAt(i)) {
+ return false;
+ }
+ }
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/c5b80246/src/main/java/org/apache/phoenix/util/TrustedByteArrayOutputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/phoenix/util/TrustedByteArrayOutputStream.java b/src/main/java/org/apache/phoenix/util/TrustedByteArrayOutputStream.java
new file mode 100644
index 0000000..f503cb3
--- /dev/null
+++ b/src/main/java/org/apache/phoenix/util/TrustedByteArrayOutputStream.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.util;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+/**
+ *
+ * Derived version of {@link java.io.ByteArrayOutputStream} that provides access
+ * to underlying byte array buffer so that it doesn't have to be copied
+ *
+ * @author jtaylor
+ * @since 0.1
+ */
+public class TrustedByteArrayOutputStream extends ByteArrayOutputStream {
+ public TrustedByteArrayOutputStream(int initialSize) {
+ super(initialSize);
+ }
+ public byte[] getBuffer() {
+ return buf;
+ }
+ @Override
+ public byte[] toByteArray() {
+ if (buf.length == size()) {
+ return buf;
+ }
+ return super.toByteArray();
+ }
+ @Override
+ public void write(byte[] b) {
+ try {
+ super.write(b);
+ } catch (IOException e) {
+ throw new RuntimeException(e); // Impossible
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/c5b80246/src/main/java/org/apache/phoenix/util/TupleUtil.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/phoenix/util/TupleUtil.java b/src/main/java/org/apache/phoenix/util/TupleUtil.java
new file mode 100644
index 0000000..54aa5c7
--- /dev/null
+++ b/src/main/java/org/apache/phoenix/util/TupleUtil.java
@@ -0,0 +1,155 @@
+/*
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.util;
+
+import static org.apache.phoenix.query.QueryConstants.SINGLE_COLUMN;
+import static org.apache.phoenix.query.QueryConstants.SINGLE_COLUMN_FAMILY;
+
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.hbase.DoNotRetryIOException;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.WritableUtils;
+
+import org.apache.hbase.index.util.ImmutableBytesPtr;
+import org.apache.phoenix.expression.Expression;
+import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.schema.tuple.Tuple;
+
+
+/**
+ *
+ * Utilities for Tuple
+ *
+ * @author jtaylor
+ * @since 0.1
+ */
+public class TupleUtil {
+ private TupleUtil() {
+ }
+
+ public static boolean equals(Tuple t1, Tuple t2, ImmutableBytesWritable ptr) {
+ t1.getKey(ptr);
+ byte[] buf = ptr.get();
+ int offset = ptr.getOffset();
+ int length = ptr.getLength();
+ t2.getKey(ptr);
+ return Bytes.compareTo(buf, offset, length, ptr.get(), ptr.getOffset(), ptr.getLength()) == 0;
+ }
+
+ public static int compare(Tuple t1, Tuple t2, ImmutableBytesWritable ptr) {
+ return compare(t1, t2, ptr, 0);
+ }
+
+ public static int compare(Tuple t1, Tuple t2, ImmutableBytesWritable ptr, int keyOffset) {
+ t1.getKey(ptr);
+ byte[] buf = ptr.get();
+ int offset = ptr.getOffset() + keyOffset;
+ int length = ptr.getLength() - keyOffset;
+ t2.getKey(ptr);
+ return Bytes.compareTo(buf, offset, length, ptr.get(), ptr.getOffset() + keyOffset, ptr.getLength() - keyOffset);
+ }
+
+ /**
+ * Set ptr to point to the value contained in the first KeyValue without
+ * exploding Result into KeyValue array.
+ * @param r
+ * @param ptr
+ */
+ public static void getAggregateValue(Tuple r, ImmutableBytesWritable ptr) {
+ if (r.size() == 1) {
+ KeyValue kv = r.getValue(0); // Just one KV for aggregation
+ if (Bytes.compareTo(SINGLE_COLUMN_FAMILY, 0, SINGLE_COLUMN_FAMILY.length, kv.getBuffer(), kv.getFamilyOffset(), kv.getFamilyLength()) == 0) {
+ if (Bytes.compareTo(SINGLE_COLUMN, 0, SINGLE_COLUMN.length, kv.getBuffer(), kv.getQualifierOffset(), kv.getQualifierLength()) == 0) {
+ ptr.set(kv.getBuffer(), kv.getValueOffset(), kv.getValueLength());
+ return;
+ }
+ }
+ }
+ throw new IllegalStateException("Expected single, aggregated KeyValue from coprocessor, but instead received " + r + ". Ensure aggregating coprocessors are loaded correctly on server");
+ }
+
+ /** Concatenate results evaluated against a list of expressions
+ *
+ * @param result the tuple for expression evaluation
+ * @param expressions
+ * @return the concatenated byte array as ImmutableBytesWritable
+ * @throws IOException
+ */
+ public static ImmutableBytesPtr getConcatenatedValue(Tuple result, List<Expression> expressions) throws IOException {
+ ImmutableBytesPtr value = new ImmutableBytesPtr(ByteUtil.EMPTY_BYTE_ARRAY);
+ Expression expression = expressions.get(0);
+ boolean evaluated = expression.evaluate(result, value);
+
+ if (expressions.size() == 1) {
+ if (!evaluated) {
+ value.set(ByteUtil.EMPTY_BYTE_ARRAY);
+ }
+ return value;
+ } else {
+ TrustedByteArrayOutputStream output = new TrustedByteArrayOutputStream(value.getLength() * expressions.size());
+ try {
+ if (evaluated) {
+ output.write(value.get(), value.getOffset(), value.getLength());
+ }
+ for (int i = 1; i < expressions.size(); i++) {
+ if (!expression.getDataType().isFixedWidth()) {
+ output.write(QueryConstants.SEPARATOR_BYTE);
+ }
+ expression = expressions.get(i);
+ // TODO: should we track trailing null values and omit the separator bytes?
+ if (expression.evaluate(result, value)) {
+ output.write(value.get(), value.getOffset(), value.getLength());
+ } else if (i < expressions.size()-1 && expression.getDataType().isFixedWidth()) {
+ // This should never happen, because any non terminating nullable fixed width type (i.e. INT or LONG) is
+ // converted to a variable length type (i.e. DECIMAL) to allow an empty byte array to represent null.
+ throw new DoNotRetryIOException("Non terminating null value found for fixed width expression (" + expression + ") in row: " + result);
+ }
+ }
+ byte[] outputBytes = output.getBuffer();
+ value.set(outputBytes, 0, output.size());
+ return value;
+ } finally {
+ output.close();
+ }
+ }
+ }
+
+ public static int write(Tuple result, DataOutput out) throws IOException {
+ int size = 0;
+ for(int i = 0; i < result.size(); i++) {
+ KeyValue kv = result.getValue(i);
+ size += kv.getLength();
+ size += Bytes.SIZEOF_INT; // kv.getLength
+ }
+
+ WritableUtils.writeVInt(out, size);
+ for(int i = 0; i < result.size(); i++) {
+ KeyValue kv = result.getValue(i);
+ out.writeInt(kv.getLength());
+ out.write(kv.getBuffer(), kv.getOffset(), kv.getLength());
+ }
+ return size;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/c5b80246/src/main/java/overview.html
----------------------------------------------------------------------
diff --git a/src/main/java/overview.html b/src/main/java/overview.html
new file mode 100644
index 0000000..b58fa45
--- /dev/null
+++ b/src/main/java/overview.html
@@ -0,0 +1,25 @@
+<html>
+<body>
+This document is for the JDBC implementation of Phoenix, targeting those who are interested in extending the level of SQL support.
+For users of Phoenix, the programmatic API is <a href="http://docs.oracle.com/javase/1.4.2/docs/api/java/sql/package-summary.html">JDBC</a>.
+For example, the following snippet demonstrates how to query a table:
+
+<pre>
+ Properties props = new Properties();
+ // Ensure the driver is on classpath
+ // Connect through Phoenix to the zookeeper quorum with a host name of myServer
+ Connection conn = DriverManager.getConnection("jdbc:phoenix:myServer", props);
+ try {
+ PreparedStatement statement = conn.prepareStatement("SELECT count(1) FROM product_metrics WHERE organization_id=?");
+ statement.setString(1, orgId);
+ ResultSet rs = statement.executeQuery();
+ if (rs.next()) {
+ System.out.println("Row count of orgId='" + orgId + "' is " + rs.getLong(1));
+ }
+ } finally {
+ conn.close();
+ }
+</pre>
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/c5b80246/src/main/resources/java.sql.Driver
----------------------------------------------------------------------
diff --git a/src/main/resources/java.sql.Driver b/src/main/resources/java.sql.Driver
new file mode 100644
index 0000000..fc51a04
--- /dev/null
+++ b/src/main/resources/java.sql.Driver
@@ -0,0 +1 @@
+org.apache.phoenix.jdbc.PhoenixDriver
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/c5b80246/src/site/bin/merge.sh
----------------------------------------------------------------------
diff --git a/src/site/bin/merge.sh b/src/site/bin/merge.sh
new file mode 100755
index 0000000..5ac0641
--- /dev/null
+++ b/src/site/bin/merge.sh
@@ -0,0 +1,10 @@
+current_dir=$(cd $(dirname $0);pwd)
+cd $current_dir
+SITE_TARGET="../../../target/site"
+java -jar merge.jar ../language_reference_source/index.html $SITE_TARGET/language/index.html
+java -jar merge.jar ../language_reference_source/functions.html $SITE_TARGET/language/functions.html
+java -jar merge.jar ../language_reference_source/datatypes.html $SITE_TARGET/language/datatypes.html
+cd $SITE_TARGET
+
+grep -rl class=\"nav-collapse\" . | xargs sed -i 's/class=\"nav-collapse\"/class=\"nav-collapse collapse\"/g';grep -rl class=\"active\" . | xargs sed -i 's/class=\"active\"/class=\"divider\"/g'
+grep -rl "dropdown active" . | xargs sed -i 's/dropdown active/dropdown/g'
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/c5b80246/src/site/language_reference_source/datatypes.html
----------------------------------------------------------------------
diff --git a/src/site/language_reference_source/datatypes.html b/src/site/language_reference_source/datatypes.html
new file mode 100644
index 0000000..9efca10
--- /dev/null
+++ b/src/site/language_reference_source/datatypes.html
@@ -0,0 +1,493 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en">
+<head><meta http-equiv="Content-Type" content="text/html;charset=utf-8" /><title>
+Data Types
+</title><link rel="stylesheet" type="text/css" href="stylesheet.css" />
+<!-- [search] { -->
+</head>
+<body>
+
+<!-- } -->
+<h2 id="dataTypes">Index</h2>
+<!-- syntax-start
+<p class="notranslate">
+
+ <a href="#integer_type" >INTEGER Type</a><br />
+
+ <a href="#unsigned_int_type" >UNSIGNED_INT Type</a><br />
+
+ <a href="#bigint_type" >BIGINT Type</a><br />
+
+ <a href="#unsigned_long_type" >UNSIGNED_LONG Type</a><br />
+
+ <a href="#tinyint_type" >TINYINT Type</a><br />
+
+ <a href="#unsigned_tinyint_type" >UNSIGNED_TINYINT Type</a><br />
+
+ <a href="#smallint_type" >SMALLINT Type</a><br />
+
+ <a href="#unsigned_smallint_type" >UNSIGNED_SMALLINT Type</a><br />
+
+ <a href="#float_type" >FLOAT Type</a><br />
+
+ <a href="#unsigned_float_type" >UNSIGNED_FLOAT Type</a><br />
+
+ <a href="#double_type" >DOUBLE Type</a><br />
+
+ <a href="#unsigned_double_type" >UNSIGNED_DOUBLE Type</a><br />
+
+ <a href="#decimal_type" >DECIMAL Type</a><br />
+
+ <a href="#boolean_type" >BOOLEAN Type</a><br />
+
+ <a href="#time_type" >TIME Type</a><br />
+
+ <a href="#date_type" >DATE Type</a><br />
+
+ <a href="#timestamp_type" >TIMESTAMP Type</a><br />
+
+ <a href="#varchar_type" >VARCHAR Type</a><br />
+
+ <a href="#char_type" >CHAR Type</a><br />
+
+ <a href="#binary_type" >BINARY Type</a><br />
+
+ <a href="#varbinary_type" >VARBINARY Type</a><br />
+</p>
+syntax-end -->
+<!-- railroad-start -->
+<table class="notranslate index">
+ <tr>
+ <td class="index">
+
+ <a href="#integer_type" >INTEGER Type</a><br />
+
+ <a href="#unsigned_int_type" >UNSIGNED_INT Type</a><br />
+
+ <a href="#bigint_type" >BIGINT Type</a><br />
+
+ <a href="#unsigned_long_type" >UNSIGNED_LONG Type</a><br />
+
+ <a href="#tinyint_type" >TINYINT Type</a><br />
+
+ <a href="#unsigned_tinyint_type" >UNSIGNED_TINYINT Type</a><br />
+
+ <a href="#smallint_type" >SMALLINT Type</a><br />
+ </td><td class="index">
+
+ <a href="#unsigned_smallint_type" >UNSIGNED_SMALLINT Type</a><br />
+
+ <a href="#float_type" >FLOAT Type</a><br />
+
+ <a href="#unsigned_float_type" >UNSIGNED_FLOAT Type</a><br />
+
+ <a href="#double_type" >DOUBLE Type</a><br />
+
+ <a href="#unsigned_double_type" >UNSIGNED_DOUBLE Type</a><br />
+
+ <a href="#decimal_type" >DECIMAL Type</a><br />
+
+ <a href="#boolean_type" >BOOLEAN Type</a><br />
+ </td><td class="index">
+
+ <a href="#time_type" >TIME Type</a><br />
+
+ <a href="#date_type" >DATE Type</a><br />
+
+ <a href="#timestamp_type" >TIMESTAMP Type</a><br />
+
+ <a href="#varchar_type" >VARCHAR Type</a><br />
+
+ <a href="#char_type" >CHAR Type</a><br />
+
+ <a href="#binary_type" >BINARY Type</a><br />
+
+ <a href="#varbinary_type" >VARBINARY Type</a><br />
+ </td>
+ </tr>
+</table>
+<!-- railroad-end -->
+
+
+<h3 id="integer_type" class="notranslate">INTEGER Type</h3>
+<!-- railroad-start -->
+<pre name="bnf" style="display: none">
+INTEGER
+</pre>
+<div name="railroad">
+<code class="c">INTEGER</code>
+</div>
+<!-- railroad-end -->
+<!-- syntax-start
+<pre>
+INTEGER
+</pre>
+syntax-end -->
+<p>Possible values: -2147483648 to 2147483647.</p><p>Mapped to <code>java.lang.Integer</code>. The binary representation is a 4 byte integer with the sign bit flipped (so that negative values sorts before positive values).</p>
+<p>Example:</p>
+<p class="notranslate">INTEGER</p>
+
+<h3 id="unsigned_int_type" class="notranslate">UNSIGNED_INT Type</h3>
+<!-- railroad-start -->
+<pre name="bnf" style="display: none">
+UNSIGNED_INT
+</pre>
+<div name="railroad">
+<code class="c">UNSIGNED_INT</code>
+</div>
+<!-- railroad-end -->
+<!-- syntax-start
+<pre>
+UNSIGNED_INT
+</pre>
+syntax-end -->
+<p>Possible values: 0 to 2147483647. Mapped to <code>java.lang.Integer</code>. The binary representation is a 4 byte integer, matching the <code>HBase</code> Bytes.toBytes(int) method. The purpose of this type is to map to existing <code>HBase</code> data that was serialized using this <code>HBase</code> utility method. If that is not the case, use the regular signed type instead.</p>
+<p>Example:</p>
+<p class="notranslate">UNSIGNED_INT</p>
+
+<h3 id="bigint_type" class="notranslate">BIGINT Type</h3>
+<!-- railroad-start -->
+<pre name="bnf" style="display: none">
+BIGINT
+</pre>
+<div name="railroad">
+<code class="c">BIGINT</code>
+</div>
+<!-- railroad-end -->
+<!-- syntax-start
+<pre>
+BIGINT
+</pre>
+syntax-end -->
+<p>Possible values: -9223372036854775807 to 9223372036854775807. Mapped to <code>java.lang.Long</code>. The binary representation is an 8 byte long with the sign bit flipped (so that negative values sorts before positive values).</p>
+<p>Example:</p>
+<p class="notranslate">BIGINT</p>
+
+<h3 id="unsigned_long_type" class="notranslate">UNSIGNED_LONG Type</h3>
+<!-- railroad-start -->
+<pre name="bnf" style="display: none">
+UNSIGNED_LONG
+</pre>
+<div name="railroad">
+<code class="c">UNSIGNED_LONG</code>
+</div>
+<!-- railroad-end -->
+<!-- syntax-start
+<pre>
+UNSIGNED_LONG
+</pre>
+syntax-end -->
+<p>Possible values: 0 to 9223372036854775807. Mapped to <code>java.lang.Long</code>. The binary representation is an 8 byte integer, matching the <code>HBase</code> Bytes.toBytes(long) method. The purpose of this type is to map to existing <code>HBase</code> data that was serialized using this <code>HBase</code> utility method. If that is not the case, use the regular signed type instead.</p>
+<p>Example:</p>
+<p class="notranslate">UNSIGNED_LONG</p>
+
+<h3 id="tinyint_type" class="notranslate">TINYINT Type</h3>
+<!-- railroad-start -->
+<pre name="bnf" style="display: none">
+TINYINT
+</pre>
+<div name="railroad">
+<code class="c">TINYINT</code>
+</div>
+<!-- railroad-end -->
+<!-- syntax-start
+<pre>
+TINYINT
+</pre>
+syntax-end -->
+<p>Possible values: -128 to 127. Mapped to <code>java.lang.Byte</code>. The binary representation is a single byte, with the sign bit flipped (so that negative values sorts before positive values).</p>
+<p>Example:</p>
+<p class="notranslate">TINYINT</p>
+
+<h3 id="unsigned_tinyint_type" class="notranslate">UNSIGNED_TINYINT Type</h3>
+<!-- railroad-start -->
+<pre name="bnf" style="display: none">
+UNSIGNED_TINYINT
+</pre>
+<div name="railroad">
+<code class="c">UNSIGNED_TINYINT</code>
+</div>
+<!-- railroad-end -->
+<!-- syntax-start
+<pre>
+UNSIGNED_TINYINT
+</pre>
+syntax-end -->
+<p>Possible values: 0 to 127. Mapped to <code>java.lang.Byte</code>. The binary representation is a single byte, matching the <code>HBase</code> Bytes.toBytes(byte) method. The purpose of this type is to map to existing <code>HBase</code> data that was serialized using this <code>HBase</code> utility method. If that is not the case, use the regular signed type instead.</p>
+<p>Example:</p>
+<p class="notranslate">UNSIGNED_TINYINT</p>
+
+<h3 id="smallint_type" class="notranslate">SMALLINT Type</h3>
+<!-- railroad-start -->
+<pre name="bnf" style="display: none">
+SMALLINT
+</pre>
+<div name="railroad">
+<code class="c">SMALLINT</code>
+</div>
+<!-- railroad-end -->
+<!-- syntax-start
+<pre>
+SMALLINT
+</pre>
+syntax-end -->
+<p>Possible values: -32768 to 32767. Mapped to <code>java.lang.Short</code>. The binary representation is a 2 byte short with the sign bit flipped (so that negative values sort before positive values).</p>
+<p>Example:</p>
+<p class="notranslate">SMALLINT</p>
+
+<h3 id="unsigned_smallint_type" class="notranslate">UNSIGNED_SMALLINT Type</h3>
+<!-- railroad-start -->
+<pre name="bnf" style="display: none">
+UNSIGNED_SMALLINT
+</pre>
+<div name="railroad">
+<code class="c">UNSIGNED_SMALLINT</code>
+</div>
+<!-- railroad-end -->
+<!-- syntax-start
+<pre>
+UNSIGNED_SMALLINT
+</pre>
+syntax-end -->
+<p>Possible values: 0 to 32767. Mapped to <code>java.lang.Short</code>. The binary representation is an 2 byte integer, matching the <code>HBase</code> Bytes.toBytes(short) method. The purpose of this type is to map to existing <code>HBase</code> data that was serialized using this <code>HBase</code> utility method. If that is not the case, use the regular signed type instead.</p>
+<p>Example:</p>
+<p class="notranslate">UNSIGNED_SMALLINT</p>
+
+<h3 id="float_type" class="notranslate">FLOAT Type</h3>
+<!-- railroad-start -->
+<pre name="bnf" style="display: none">
+FLOAT
+</pre>
+<div name="railroad">
+<code class="c">FLOAT</code>
+</div>
+<!-- railroad-end -->
+<!-- syntax-start
+<pre>
+FLOAT
+</pre>
+syntax-end -->
+<p>Possible values: -3.402823466 E + 38 to 3.402823466 E + 38. Mapped to <code>java.lang.Float</code>. The binary representation is an 4 byte float with the sign bit flipped (so that negative values sort before positive values).</p>
+<p>Example:</p>
+<p class="notranslate">FLOAT</p>
+
+<h3 id="unsigned_float_type" class="notranslate">UNSIGNED_FLOAT Type</h3>
+<!-- railroad-start -->
+<pre name="bnf" style="display: none">
+UNSIGNED_FLOAT
+</pre>
+<div name="railroad">
+<code class="c">UNSIGNED_FLOAT</code>
+</div>
+<!-- railroad-end -->
+<!-- syntax-start
+<pre>
+UNSIGNED_FLOAT
+</pre>
+syntax-end -->
+<p>Possible values: 0 to 3.402823466 E + 38. Mapped to <code>java.lang.Float</code>. The binary representation is an 4 byte float matching the <code>HBase</code> Bytes.toBytes(float) method. The purpose of this type is to map to existing <code>HBase</code> data that was serialized using this <code>HBase</code> utility method. If that is not the case, use the regular signed type instead.</p>
+<p>Example:</p>
+<p class="notranslate">UNSIGNED_FLOAT</p>
+
+<h3 id="double_type" class="notranslate">DOUBLE Type</h3>
+<!-- railroad-start -->
+<pre name="bnf" style="display: none">
+DOUBLE
+</pre>
+<div name="railroad">
+<code class="c">DOUBLE</code>
+</div>
+<!-- railroad-end -->
+<!-- syntax-start
+<pre>
+DOUBLE
+</pre>
+syntax-end -->
+<p>Possible values: -1.7976931348623158 E + 308 to 1.7976931348623158 E + 308. Mapped to <code>java.lang.Double</code>. The binary representation is an 8 byte double with the sign bit flipped (so that negative values sort before positive value).</p>
+<p>Example:</p>
+<p class="notranslate">DOUBLE</p>
+
+<h3 id="unsigned_double_type" class="notranslate">UNSIGNED_DOUBLE Type</h3>
+<!-- railroad-start -->
+<pre name="bnf" style="display: none">
+UNSIGNED_DOUBLE
+</pre>
+<div name="railroad">
+<code class="c">UNSIGNED_DOUBLE</code>
+</div>
+<!-- railroad-end -->
+<!-- syntax-start
+<pre>
+UNSIGNED_DOUBLE
+</pre>
+syntax-end -->
+<p>Possible values: 0 to 1.7976931348623158 E + 308. Mapped to <code>java.lang.Double</code>. The binary representation is an 8 byte double matching the <code>HBase</code> Bytes.toBytes(double) method. The purpose of this type is to map to existing <code>HBase</code> data that was serialized using this <code>HBase</code> utility method. If that is not the case, use the regular signed type instead.</p>
+<p>Example:</p>
+<p class="notranslate">UNSIGNED_DOUBLE</p>
+
+<h3 id="decimal_type" class="notranslate">DECIMAL Type</h3>
+<!-- railroad-start -->
+<pre name="bnf" style="display: none">
+DECIMAL
+</pre>
+<div name="railroad">
+<code class="c">DECIMAL</code>
+</div>
+<!-- railroad-end -->
+<!-- syntax-start
+<pre>
+DECIMAL
+</pre>
+syntax-end -->
+<p>Data type with fixed precision and scale. The maximum precision is 18 digits. Mapped to <code>java.math.BigDecimal</code>. The binary representation is binary comparable, variable length format. When used in a row key, it is terminated with a null byte unless it is the last column.</p>
+<p>Example:</p>
+<p class="notranslate">DECIMAL</p>
+
+<h3 id="boolean_type" class="notranslate">BOOLEAN Type</h3>
+<!-- railroad-start -->
+<pre name="bnf" style="display: none">
+BOOLEAN
+</pre>
+<div name="railroad">
+<code class="c">BOOLEAN</code>
+</div>
+<!-- railroad-end -->
+<!-- syntax-start
+<pre>
+BOOLEAN
+</pre>
+syntax-end -->
+<p>Possible values: <code>TRUE</code> and <code>FALSE</code>.</p><p>Mapped to <code>java.lang.Boolean</code>. The binary representation is a single byte with 0 for false and 1 for true</p>
+<p>Example:</p>
+<p class="notranslate">BOOLEAN</p>
+
+<h3 id="time_type" class="notranslate">TIME Type</h3>
+<!-- railroad-start -->
+<pre name="bnf" style="display: none">
+TIME
+</pre>
+<div name="railroad">
+<code class="c">TIME</code>
+</div>
+<!-- railroad-end -->
+<!-- syntax-start
+<pre>
+TIME
+</pre>
+syntax-end -->
+<p>The time data type. The format is yyyy-<code>MM</code>-dd hh:mm:ss, with both the date and time parts maintained. Mapped to <code>java.sql.Time</code>. The binary representation is an 8 byte long (the number of milliseconds from the epoch).</p>
+<p>Example:</p>
+<p class="notranslate">TIME</p>
+
+<h3 id="date_type" class="notranslate">DATE Type</h3>
+<!-- railroad-start -->
+<pre name="bnf" style="display: none">
+DATE
+</pre>
+<div name="railroad">
+<code class="c">DATE</code>
+</div>
+<!-- railroad-end -->
+<!-- syntax-start
+<pre>
+DATE
+</pre>
+syntax-end -->
+<p>The date data type. The format is yyyy-<code>MM</code>-dd hh:mm:ss, with both the date and time parts maintained to a millisecond accuracy. Mapped to <code>java.sql.Date</code>. The binary representation is an 8 byte long (the number of milliseconds from the epoch).</p>
+<p>Example:</p>
+<p class="notranslate">DATE</p>
+
+<h3 id="timestamp_type" class="notranslate">TIMESTAMP Type</h3>
+<!-- railroad-start -->
+<pre name="bnf" style="display: none">
+TIMESTAMP
+</pre>
+<div name="railroad">
+<code class="c">TIMESTAMP</code>
+</div>
+<!-- railroad-end -->
+<!-- syntax-start
+<pre>
+TIMESTAMP
+</pre>
+syntax-end -->
+<p>The timestamp data type. The format is yyyy-<code>MM</code>-dd hh:mm:ss[.nnnnnnnnn]. Mapped to <code>java.sql.Timestamp</code> with an internal representation of the number of nanos from the epoch. The binary representation is 12 bytes: an 8 byte long for the epoch time plus a 4 byte integer for the nanos.</p>
+<p>Example:</p>
+<p class="notranslate">TIMESTAMP</p>
+
+<h3 id="varchar_type" class="notranslate">VARCHAR Type</h3>
+<!-- railroad-start -->
+<pre name="bnf" style="display: none">
+VARCHAR [ ( <a href="index.html#int">precisionInt</a> ) ]
+</pre>
+<div name="railroad">
+<table class="railroad"><tr class="railroad"><td class="d"><code class="c">VARCHAR</code></td><td class="d"><table class="railroad"><tr class="railroad"><td class="ts"></td><td class="d"> </td><td class="te"></td></tr><tr class="railroad"><td class="ls"></td><td class="d"><table class="railroad"><tr class="railroad"><td class="d"><code class="c">( <a href="index.html#int">precisionInt</a> )</code></td></tr></table></td><td class="le"></td></tr></table></td></tr></table>
+</div>
+<!-- railroad-end -->
+<!-- syntax-start
+<pre>
+VARCHAR [ ( <a href="index.html#int">precisionInt</a> ) ]
+</pre>
+syntax-end -->
+<p>A variable length String with an optional max byte length. The binary representation is <code>UTF8</code> matching the <code>HBase</code> Bytes.toBytes(String) method. When used in a row key, it is terminated with a null byte unless it is the last column.</p><p>Mapped to <code>java.lang.String</code>.</p>
+<p>Example:</p>
+<p class="notranslate">VARCHAR<br />VARCHAR(255)</p>
+
+<h3 id="char_type" class="notranslate">CHAR Type</h3>
+<!-- railroad-start -->
+<pre name="bnf" style="display: none">
+CHAR ( <a href="index.html#int">precisionInt</a> )
+</pre>
+<div name="railroad">
+<table class="railroad"><tr class="railroad"><td class="d"><code class="c">CHAR ( <a href="index.html#int">precisionInt</a> )</code></td></tr></table>
+</div>
+<!-- railroad-end -->
+<!-- syntax-start
+<pre>
+CHAR ( <a href="index.html#int">precisionInt</a> )
+</pre>
+syntax-end -->
+<p>A fixed length String with single-byte characters. The binary representation is <code>UTF8</code> matching the <code>HBase</code> Bytes.toBytes(String) method.</p><p>Mapped to <code>java.lang.String</code>.</p>
+<p>Example:</p>
+<p class="notranslate">CHAR(10)</p>
+
+<h3 id="binary_type" class="notranslate">BINARY Type</h3>
+<!-- railroad-start -->
+<pre name="bnf" style="display: none">
+BINARY ( <a href="index.html#int">precisionInt</a> )
+</pre>
+<div name="railroad">
+<table class="railroad"><tr class="railroad"><td class="d"><code class="c">BINARY ( <a href="index.html#int">precisionInt</a> )</code></td></tr></table>
+</div>
+<!-- railroad-end -->
+<!-- syntax-start
+<pre>
+BINARY ( <a href="index.html#int">precisionInt</a> )
+</pre>
+syntax-end -->
+<p>Raw fixed length byte array.</p><p>Mapped to <code>byte[]</code>.</p>
+<p>Example:</p>
+<p class="notranslate">BINARY</p>
+
+<h3 id="varbinary_type" class="notranslate">VARBINARY Type</h3>
+<!-- railroad-start -->
+<pre name="bnf" style="display: none">
+VARBINARY
+</pre>
+<div name="railroad">
+<code class="c">VARBINARY</code>
+</div>
+<!-- railroad-end -->
+<!-- syntax-start
+<pre>
+VARBINARY
+</pre>
+syntax-end -->
+<p>Raw variable length byte array.</p><p>Mapped to <code>byte[]</code>.</p>
+<p>Example:</p>
+<p class="notranslate">VARBINARY</p>
+
+<!-- [close] { -->
+
+<!-- } --></body></html>
+