You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by da...@apache.org on 2016/09/01 18:18:09 UTC

kudu git commit: [java-client] Add ScanToken.stringifySerializedToken

Repository: kudu
Updated Branches:
  refs/heads/master 1f660ee9f -> 4fd0572ee


[java-client] Add ScanToken.stringifySerializedToken

stringifySerializedToken takes a serialized scan token, and returns a
String suitable for debug printing. The string contains information
sufficient to determine which range of tablet(s) the scan token will
cover, and which rows within the tablets. Namely, the range partition
bounds and primary key bounds. Example output, wrapped for readability
(a, b, and c are column names with type STRING):

ScanToken{table=org.apache.kudu.client.TestKuduClient-1472595465767,
          lower-bound-primary-key=(string a=1, string b=3, string c=5),
          upper-bound-primary-key=(string a=2, string b=4, string c=),
          hash-partition-buckets: [2],
          range-partition: [(string a=0, string b=0, string c=0),
                            (string a=3, string b=5, string c=6))}

The Java client did not have any method of deserializing encoded primary
or partition keys, so most of the work in this commit is introducing
utility methods for that purpose. I haven't added tests of the specific
format of the strings, but I have added the printing to many of the
existing ScanToken tests in order to make sure that the formatting code
itself doesn't fail. I've also verified that the output looks good. The
format doesn't include information like predicates or consistency, but
that could be added in the future if so desired.

Change-Id: I42014da270e66c370cc6d6ed68fb38331130ff6d
Reviewed-on: http://gerrit.cloudera.org:8080/4173
Tested-by: Kudu Jenkins
Reviewed-by: Dan Burkert <da...@cloudera.com>


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/4fd0572e
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/4fd0572e
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/4fd0572e

Branch: refs/heads/master
Commit: 4fd0572eef60b163f9fb02742a3f5661117e8c8b
Parents: 1f660ee
Author: Dan Burkert <da...@cloudera.com>
Authored: Mon Aug 29 17:48:31 2016 -0700
Committer: Dan Burkert <da...@cloudera.com>
Committed: Thu Sep 1 18:17:45 2016 +0000

----------------------------------------------------------------------
 .../org/apache/kudu/client/DeadlineTracker.java |   2 +-
 .../java/org/apache/kudu/client/KeyEncoder.java | 216 +++++++++++++++++++
 .../org/apache/kudu/client/KuduScanToken.java   |  46 +++-
 .../java/org/apache/kudu/client/PartialRow.java | 136 ++++++++----
 .../java/org/apache/kudu/client/RowResult.java  |  18 +-
 .../kudu/client/TestFlexiblePartitioning.java   |  51 ++++-
 .../org/apache/kudu/client/TestKeyEncoding.java |  37 +++-
 .../org/apache/kudu/client/TestKuduClient.java  |  13 +-
 .../org/apache/kudu/client/TestOperation.java   |   7 +-
 .../kudu/mapreduce/KuduTableInputFormat.java    |   9 +-
 10 files changed, 454 insertions(+), 81 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/4fd0572e/java/kudu-client/src/main/java/org/apache/kudu/client/DeadlineTracker.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/DeadlineTracker.java b/java/kudu-client/src/main/java/org/apache/kudu/client/DeadlineTracker.java
index 226952e..d878d89 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/DeadlineTracker.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/DeadlineTracker.java
@@ -148,7 +148,7 @@ public class DeadlineTracker {
   }
 
   public String toString() {
-    StringBuffer buf = new StringBuffer("DeadlineTracker(timeout=");
+    StringBuilder buf = new StringBuilder("DeadlineTracker(timeout=");
     buf.append(deadline);
     buf.append(", elapsed=").append(stopwatch.elapsed(TimeUnit.MILLISECONDS));
     buf.append(")");

http://git-wip-us.apache.org/repos/asf/kudu/blob/4fd0572e/java/kudu-client/src/main/java/org/apache/kudu/client/KeyEncoder.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/KeyEncoder.java b/java/kudu-client/src/main/java/org/apache/kudu/client/KeyEncoder.java
index 49a0a72..96e36ac 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/KeyEncoder.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/KeyEncoder.java
@@ -23,10 +23,14 @@ import org.apache.kudu.Schema;
 import org.apache.kudu.Type;
 import org.apache.kudu.annotations.InterfaceAudience;
 import org.apache.kudu.client.PartitionSchema.HashBucketSchema;
+import org.apache.kudu.util.Pair;
 
 import java.io.ByteArrayOutputStream;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
 import java.util.List;
 
 /**
@@ -190,4 +194,216 @@ class KeyEncoder {
     buf.reset();
     return bytes;
   }
+
+  /**
+   * Decodes a primary key into a row
+   *
+   * @param schema the table schema
+   * @param key the encoded key
+   * @return the decoded primary key as a row
+   */
+  public static PartialRow decodePrimaryKey(Schema schema, byte[] key) {
+    PartialRow row = schema.newPartialRow();
+    ByteBuffer buf = ByteBuffer.wrap(key);
+    buf.order(ByteOrder.BIG_ENDIAN);
+
+    for (int idx = 0; idx < schema.getPrimaryKeyColumnCount(); idx++) {
+      decodeColumn(buf, row, idx, idx + 1 == schema.getPrimaryKeyColumnCount());
+    }
+
+    if (buf.hasRemaining()) {
+      throw new IllegalArgumentException("Unable to decode all primary key bytes");
+    }
+    return row;
+  }
+
+  /**
+   * Decodes a partition key into a list of hash buckets and range key
+   *
+   * @param schema the schema of the table
+   * @param partitionSchema the partition schema of the table
+   * @param key the encoded partition key
+   * @return the decoded buckets and range key
+   */
+  public static Pair<List<Integer>, PartialRow> decodePartitionKey(Schema schema,
+                                                                   PartitionSchema partitionSchema,
+                                                                   byte[] key) {
+    ByteBuffer buf = ByteBuffer.wrap(key);
+    buf.order(ByteOrder.BIG_ENDIAN);
+
+    List<Integer> buckets = new ArrayList<>();
+    PartialRow row = schema.newPartialRow();
+
+    for (HashBucketSchema hashSchema : partitionSchema.getHashBucketSchemas()) {
+      if (buf.hasRemaining()) {
+        buckets.add(buf.getInt());
+      } else {
+        buckets.add(0);
+      }
+    }
+
+    Iterator<Integer> rangeIds = partitionSchema.getRangeSchema().getColumns().iterator();
+    while (rangeIds.hasNext()) {
+      int idx = schema.getColumnIndex(rangeIds.next());
+      if (buf.hasRemaining()) {
+        decodeColumn(buf, row, idx, !rangeIds.hasNext());
+      } else {
+        row.setMin(idx);
+      }
+    }
+
+    if (buf.hasRemaining()) {
+      throw new IllegalArgumentException("Unable to decode all partition key bytes");
+    }
+
+    return new Pair<>(buckets, row);
+  }
+
+  /**
+   * Decoded a key-encoded column into a row.
+   *
+   * @param buf the buffer containing the column
+   * @param row the row to set the column value in
+   * @param idx the index of the column to decode
+   * @param isLast whether the column is the last column in the key
+   */
+  private static void decodeColumn(ByteBuffer buf, PartialRow row, int idx, boolean isLast) {
+    Schema schema = row.getSchema();
+    switch (schema.getColumnByIndex(idx).getType()) {
+      case INT8: row.addByte(idx, (byte) (buf.get() ^ Byte.MIN_VALUE)); break;
+      case INT16: row.addShort(idx, (short) (buf.getShort() ^ Short.MIN_VALUE)); break;
+      case INT32: row.addInt(idx, buf.getInt() ^ Integer.MIN_VALUE); break;
+      case INT64:
+      case TIMESTAMP: row.addLong(idx, buf.getLong() ^ Long.MIN_VALUE); break;
+      case BINARY: {
+        byte[] binary = decodeBinaryColumn(buf, isLast);
+        row.addBinary(idx, binary);
+        break;
+      }
+      case STRING: {
+        byte[] binary = decodeBinaryColumn(buf, isLast);
+        row.addStringUtf8(idx, binary);
+        break;
+      }
+      default:
+        throw new IllegalArgumentException(String.format(
+            "The column type %s is not a valid key component type",
+            schema.getColumnByIndex(idx).getType()));
+    }
+  }
+
+  /**
+   * Decode a binary key column.
+   *
+   * @param key the key bytes
+   * @param isLast whether the column is the final column in the key.
+   * @return the binary value.
+   */
+  private static byte[] decodeBinaryColumn(ByteBuffer key, boolean isLast) {
+    if (isLast) {
+      byte[] bytes = Arrays.copyOfRange(key.array(),
+                                        key.arrayOffset() + key.position(),
+                                        key.arrayOffset() + key.limit());
+      key.position(key.limit());
+      return bytes;
+    }
+
+    // When encoding a binary column that is not the final column in the key, a
+    // 0x0000 separator is used to retain lexicographic comparability. Null
+    // bytes in the input are escaped as 0x0001.
+    ByteArrayOutputStream baos = new ByteArrayOutputStream(key.remaining());
+    for (int i = key.position(); i < key.limit(); i++) {
+      if (key.get(i) == 0) {
+        switch (key.get(i + 1)) {
+          case 0: {
+            baos.write(key.array(),
+                       key.arrayOffset() + key.position(),
+                       i - key.position());
+            key.position(i + 2);
+            return baos.toByteArray();
+          }
+          case 1: {
+            baos.write(key.array(),
+                       key.arrayOffset() + key.position(),
+                       i + 1 - key.position());
+            i++;
+            key.position(i + 1);
+            break;
+          }
+          default: throw new IllegalArgumentException("Unexpected binary sequence");
+        }
+      }
+    }
+
+    baos.write(key.array(),
+               key.arrayOffset() + key.position(),
+               key.remaining());
+    key.position(key.limit());
+    return baos.toByteArray();
+  }
+
+  /**
+   * Debug formats a partition key range.
+   *
+   * @param schema the table schema
+   * @param partitionSchema the table partition schema
+   * @param lowerBound the lower bound encoded partition key
+   * @param upperBound the upper bound encoded partition key
+   * @return a debug string which describes the partition key range.
+   */
+  public static String formatPartitionKeyRange(Schema schema,
+                                               PartitionSchema partitionSchema,
+                                               byte[] lowerBound,
+                                               byte[] upperBound) {
+    if (partitionSchema.getRangeSchema().getColumns().isEmpty() &&
+        partitionSchema.getHashBucketSchemas().isEmpty()) {
+      assert lowerBound.length == 0 && upperBound.length == 0;
+      return "<no-partitioning>";
+    }
+
+    // Even though we parse hash buckets for the upper and lower bound partition
+    // keys, we only use the lower bound set. Upper bound partition keys are
+    // exclusive, so the upper bound partition key may include an incremented
+    // bucket in the last position.
+    Pair<List<Integer>, PartialRow> lower = decodePartitionKey(schema, partitionSchema, lowerBound);
+    Pair<List<Integer>, PartialRow> upper = decodePartitionKey(schema, partitionSchema, upperBound);
+
+    StringBuilder sb = new StringBuilder();
+
+    List<Integer> hashBuckets = lower.getFirst();
+    if (!hashBuckets.isEmpty()) {
+      sb.append("hash-partition-buckets: ");
+      sb.append(hashBuckets);
+    }
+
+    if (partitionSchema.getRangeSchema().getColumns().size() > 0) {
+      if (!hashBuckets.isEmpty()) {
+        sb.append(", ");
+      }
+
+      List<Integer> idxs = new ArrayList<>();
+      for (int id : partitionSchema.getRangeSchema().getColumns()) {
+        idxs.add(schema.getColumnIndex(id));
+      }
+
+      sb.append("range-partition: [");
+      if (lowerBound.length > 4 * hashBuckets.size()) {
+        sb.append('(');
+        lower.getSecond().appendDebugString(idxs, sb);
+        sb.append(')');
+      } else {
+        sb.append("<start>");
+      }
+      sb.append(", ");
+      if (upperBound.length > 4 * hashBuckets.size()) {
+        sb.append('(');
+        upper.getSecond().appendDebugString(idxs, sb);
+        sb.append(')');
+      } else {
+        sb.append("<end>");
+      }
+      sb.append(')');
+    }
+    return sb.toString();
+  }
 }

http://git-wip-us.apache.org/repos/asf/kudu/blob/4fd0572e/java/kudu-client/src/main/java/org/apache/kudu/client/KuduScanToken.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduScanToken.java b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduScanToken.java
index 6b121b2..a5f5081 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduScanToken.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduScanToken.java
@@ -17,6 +17,7 @@
 
 package org.apache.kudu.client;
 
+import com.google.common.base.MoreObjects;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
 import com.google.protobuf.CodedInputStream;
@@ -97,14 +98,49 @@ public class KuduScanToken implements Comparable<KuduScanToken> {
    * @param buf a byte array containing the serialized scan token.
    * @param client a Kudu client for the cluster
    * @return a scanner for the serialized scan token
-   * @throws Exception
    */
-  public static KuduScanner deserializeIntoScanner(byte[] buf, KuduClient client) throws Exception {
+  public static KuduScanner deserializeIntoScanner(byte[] buf, KuduClient client) throws IOException {
     return pbIntoScanner(ScanTokenPB.parseFrom(CodedInputStream.newInstance(buf)), client);
   }
 
+  /**
+   * Formats the serialized token for debug printing.
+   *
+   * @param buf the serialized token
+   * @param client a Kudu client for the cluster to which the token belongs
+   * @return a debug string
+   */
+  public static String stringifySerializedToken(byte[] buf, KuduClient client) throws IOException {
+    ScanTokenPB token = ScanTokenPB.parseFrom(CodedInputStream.newInstance(buf));
+    KuduTable table = client.openTable(token.getTableName());
+
+    MoreObjects.ToStringHelper helper = MoreObjects.toStringHelper("ScanToken")
+                                                   .add("table", token.getTableName());
+
+    if (token.hasLowerBoundPrimaryKey() && !token.getLowerBoundPrimaryKey().isEmpty()) {
+      helper.add("lower-bound-primary-key",
+                 KeyEncoder.decodePrimaryKey(table.getSchema(),
+                                             token.getLowerBoundPrimaryKey().toByteArray())
+                           .stringifyRowKey());
+    }
+
+    if (token.hasUpperBoundPrimaryKey() && !token.getUpperBoundPrimaryKey().isEmpty()) {
+      helper.add("upper-bound-primary-key",
+                 KeyEncoder.decodePrimaryKey(table.getSchema(),
+                                             token.getUpperBoundPrimaryKey().toByteArray())
+                           .stringifyRowKey());
+    }
+
+    helper.addValue(KeyEncoder.formatPartitionKeyRange(table.getSchema(),
+                                                       table.getPartitionSchema(),
+                                                       token.getLowerBoundPartitionKey().toByteArray(),
+                                                       token.getUpperBoundPartitionKey().toByteArray()));
+
+    return helper.toString();
+  }
+
   private static KuduScanner pbIntoScanner(ScanTokenPB message,
-                                           KuduClient client) throws Exception {
+                                           KuduClient client) throws KuduException {
     Preconditions.checkArgument(
         !message.getFeatureFlagsList().contains(ScanTokenPB.Feature.Unknown),
         "Scan token requires an unsupported feature. This Kudu client must be updated.");
@@ -299,9 +335,9 @@ public class KuduScanToken implements Comparable<KuduScanToken> {
         for (LocatedTablet tablet : tablets) {
           Client.ScanTokenPB.Builder builder = proto.clone();
           builder.setLowerBoundPartitionKey(
-              ZeroCopyLiteralByteString.wrap(tablet.getPartition().partitionKeyStart));
+              ZeroCopyLiteralByteString.wrap(tablet.getPartition().getPartitionKeyStart()));
           builder.setUpperBoundPartitionKey(
-              ZeroCopyLiteralByteString.wrap(tablet.getPartition().partitionKeyEnd));
+              ZeroCopyLiteralByteString.wrap(tablet.getPartition().getPartitionKeyEnd()));
           tokens.add(new KuduScanToken(tablet, builder.build()));
         }
         return tokens;

http://git-wip-us.apache.org/repos/asf/kudu/blob/4fd0572e/java/kudu-client/src/main/java/org/apache/kudu/client/PartialRow.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/PartialRow.java b/java/kudu-client/src/main/java/org/apache/kudu/client/PartialRow.java
index 0deee84..31a7596 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/PartialRow.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/PartialRow.java
@@ -17,6 +17,7 @@
 package org.apache.kudu.client;
 
 import java.nio.ByteBuffer;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.BitSet;
 import java.util.List;
@@ -520,59 +521,106 @@ public class PartialRow {
    */
   public String stringifyRowKey() {
     int numRowKeys = schema.getPrimaryKeyColumnCount();
+    List<Integer> idxs = new ArrayList<>(numRowKeys);
+    for (int i = 0; i < numRowKeys; i++) {
+      idxs.add(i);
+    }
+
     StringBuilder sb = new StringBuilder();
     sb.append("(");
-    for (int i = 0; i < numRowKeys; i++) {
-      if (i > 0) {
+    appendDebugString(idxs, sb);
+    sb.append(")");
+    return sb.toString();
+  }
+
+  /**
+   * Appends a debug string for the provided columns in the row.
+   *
+   * @param idxs the column indexes.
+   * @param sb the string builder to append to
+   */
+  void appendDebugString(List<Integer> idxs, StringBuilder sb) {
+    boolean first = true;
+    for (int idx : idxs) {
+      if (first) {
+        first = false;
+      } else {
         sb.append(", ");
       }
 
-      ColumnSchema col = schema.getColumnByIndex(i);
-      assert !col.isNullable();
-      Preconditions.checkState(columnsBitSet.get(i),
-          "Full row key not specified, missing at least col: " + col.getName());
-      Type type = col.getType();
-      sb.append(type.getName());
-      sb.append(" ");
+      ColumnSchema col = schema.getColumnByIndex(idx);
+
+      sb.append(col.getType().getName());
+      sb.append(' ');
       sb.append(col.getName());
-      sb.append("=");
-
-      if (type == Type.STRING || type == Type.BINARY) {
-        ByteBuffer value = getVarLengthData().get(i).duplicate();
-        value.reset(); // Make sure we start at the beginning.
-        byte[] data = new byte[value.limit()];
-        value.get(data);
-        if (type == Type.STRING) {
-          sb.append(Bytes.getString(data));
-        } else {
-          sb.append(Bytes.pretty(data));
-        }
-      } else {
-        switch (type) {
-          case INT8:
-            sb.append(Bytes.getByte(rowAlloc, schema.getColumnOffset(i)));
-            break;
-          case INT16:
-            sb.append(Bytes.getShort(rowAlloc, schema.getColumnOffset(i)));
-            break;
-          case INT32:
-            sb.append(Bytes.getInt(rowAlloc, schema.getColumnOffset(i)));
-            break;
-          case INT64:
-            sb.append(Bytes.getLong(rowAlloc, schema.getColumnOffset(i)));
-            break;
-          case TIMESTAMP:
-            sb.append(Bytes.getLong(rowAlloc, schema.getColumnOffset(i)));
-            break;
-          default:
-            throw new IllegalArgumentException(String.format(
-                "The column type %s is not a valid key component type", type));
-        }
+      sb.append('=');
+
+      Preconditions.checkState(columnsBitSet.get(idx), "Column %s is not set", col.getName());
+
+      if (nullsBitSet != null && nullsBitSet.get(idx)) {
+        sb.append("NULL");
+        continue;
+      }
+
+      switch (col.getType()) {
+        case BOOL:
+          sb.append(Bytes.getBoolean(rowAlloc, schema.getColumnOffset(idx)));
+          break;
+        case INT8:
+          sb.append(Bytes.getByte(rowAlloc, schema.getColumnOffset(idx)));
+          break;
+        case INT16:
+          sb.append(Bytes.getShort(rowAlloc, schema.getColumnOffset(idx)));
+          break;
+        case INT32:
+          sb.append(Bytes.getInt(rowAlloc, schema.getColumnOffset(idx)));
+          break;
+        case INT64:
+          sb.append(Bytes.getLong(rowAlloc, schema.getColumnOffset(idx)));
+          break;
+        case TIMESTAMP:
+          sb.append(RowResult.timestampToString(
+              Bytes.getLong(rowAlloc, schema.getColumnOffset(idx))));
+          break;
+        case FLOAT:
+          sb.append(Bytes.getFloat(rowAlloc, schema.getColumnOffset(idx)));
+          break;
+        case DOUBLE:
+          sb.append(Bytes.getDouble(rowAlloc, schema.getColumnOffset(idx)));
+          break;
+        case BINARY:
+        case STRING:
+          ByteBuffer value = getVarLengthData().get(idx).duplicate();
+          value.reset(); // Make sure we start at the beginning.
+          byte[] data = new byte[value.limit()];
+          value.get(data);
+          if (col.getType() == Type.STRING) {
+            sb.append(Bytes.getString(data));
+          } else {
+            sb.append(Bytes.pretty(data));
+          }
       }
     }
-    sb.append(")");
+  }
 
-    return sb.toString();
+  /**
+   * Sets the column to the minimum possible value for the column's type.
+   * @param index the index of the column to set to the minimum
+   */
+  void setMin(int index) {
+    Type type = schema.getColumnByIndex(index).getType();
+    switch (type) {
+      case BOOL: addBoolean(index, false); break;
+      case INT8: addByte(index, Byte.MIN_VALUE); break;
+      case INT16: addShort(index, Short.MIN_VALUE); break;
+      case INT32: addInt(index, Integer.MIN_VALUE); break;
+      case INT64:
+      case TIMESTAMP: addLong(index, Integer.MIN_VALUE); break;
+      case FLOAT: addFloat(index, -Float.MAX_VALUE); break;
+      case DOUBLE: addDouble(index, -Double.MAX_VALUE); break;
+      case STRING: addStringUtf8(index, AsyncKuduClient.EMPTY_ARRAY); break;
+      case BINARY: addBinary(index, AsyncKuduClient.EMPTY_ARRAY); break;
+    }
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/kudu/blob/4fd0572e/java/kudu-client/src/main/java/org/apache/kudu/client/RowResult.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/RowResult.java b/java/kudu-client/src/main/java/org/apache/kudu/client/RowResult.java
index 6238608..7afc1b4 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/RowResult.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/RowResult.java
@@ -25,6 +25,7 @@ import org.apache.kudu.util.Slice;
 
 import java.nio.ByteBuffer;
 import java.text.DateFormat;
+import java.text.FieldPosition;
 import java.text.SimpleDateFormat;
 import java.util.BitSet;
 import java.util.Date;
@@ -38,9 +39,9 @@ import java.util.TimeZone;
 public class RowResult {
 
   private static final int INDEX_RESET_LOCATION = -1;
-  private static final DateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
-  {
-    DATE_FORMAT.setTimeZone(TimeZone.getTimeZone("GMT"));
+  private static final DateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
+  static {
+    DATE_FORMAT.setTimeZone(TimeZone.getTimeZone("UTC"));
   }
   private static final long MS_IN_S = 1000L;
   private static final long US_IN_S = 1000L * 1000L;
@@ -506,14 +507,14 @@ public class RowResult {
    * Transforms a timestamp into a string, whose formatting and timezone is consistent
    * across kudu.
    * @param timestamp the timestamp, in microseconds
-   * @return a string, in the format: YYYY-MM-DD HH:MM:SS.ssssss GMT
+   * @return a string, in the format: YYYY-MM-DDTHH:MM:SS.ssssssZ
    */
   static String timestampToString(long timestamp) {
     long tsMillis = timestamp / MS_IN_S;
     long tsMicros = timestamp % US_IN_S;
     StringBuffer formattedTs = new StringBuffer();
-    formattedTs.append(DATE_FORMAT.format(new Date(tsMillis)));
-    formattedTs.append(String.format(".%06d GMT", tsMicros));
+    DATE_FORMAT.format(new Date(tsMillis), formattedTs, new FieldPosition(0));
+    formattedTs.append(String.format(".%06dZ", tsMicros));
     return formattedTs.toString();
   }
 
@@ -522,7 +523,7 @@ public class RowResult {
    * form.
    */
   public String rowToString() {
-    StringBuffer buf = new StringBuffer();
+    StringBuilder buf = new StringBuilder();
     for (int i = 0; i < schema.getColumnCount(); i++) {
       ColumnSchema col = schema.getColumnByIndex(i);
       if (i != 0) {
@@ -559,12 +560,11 @@ public class RowResult {
    * the iterator as well as its data.
    */
   public String toStringLongFormat() {
-    StringBuffer buf = new StringBuffer(this.rowSize); // super rough estimation
+    StringBuilder buf = new StringBuilder(this.rowSize); // super rough estimation
     buf.append(this.toString());
     buf.append("{");
     buf.append(rowToString());
     buf.append("}");
     return buf.toString();
   }
-
 }

http://git-wip-us.apache.org/repos/asf/kudu/blob/4fd0572e/java/kudu-client/src/test/java/org/apache/kudu/client/TestFlexiblePartitioning.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestFlexiblePartitioning.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestFlexiblePartitioning.java
index fe78ce6..d4fe444 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestFlexiblePartitioning.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestFlexiblePartitioning.java
@@ -21,11 +21,14 @@ import com.google.common.base.Predicates;
 import com.google.common.collect.ComparisonChain;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Sets;
+
 import org.junit.Before;
 import org.junit.Test;
 import org.apache.kudu.ColumnSchema;
 import org.apache.kudu.Schema;
 import org.apache.kudu.Type;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.util.ArrayList;
 import java.util.HashSet;
@@ -34,9 +37,9 @@ import java.util.Objects;
 import java.util.Set;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
 
 public class TestFlexiblePartitioning extends BaseKuduTest {
+  private static final Logger LOG = LoggerFactory.getLogger(TestKuduClient.class);
   private String tableName;
 
   @Before
@@ -52,7 +55,7 @@ public class TestFlexiblePartitioning extends BaseKuduTest {
     return new Schema(columns);
   }
 
-  private static Set<Row> rows() throws Exception {
+  private static Set<Row> rows() throws KuduException {
     Set<Row> rows = new HashSet<>();
     for (int a = 0; a < 6; a++) {
       for (int b = 0; b < 6; b++) {
@@ -66,7 +69,7 @@ public class TestFlexiblePartitioning extends BaseKuduTest {
     return rows;
   }
 
-  private void insertRows(KuduTable table, Set<Row> rows) throws Exception {
+  private void insertRows(KuduTable table, Set<Row> rows) throws KuduException {
     KuduSession session = syncClient.newSession();
     try {
       for (Row row : rows) {
@@ -80,7 +83,7 @@ public class TestFlexiblePartitioning extends BaseKuduTest {
     }
   }
 
-  private Set<Row> collectRows(KuduScanner scanner) throws Exception {
+  private Set<Row> collectRows(KuduScanner scanner) throws KuduException {
     Set<Row> rows = new HashSet<>();
     while (scanner.hasMoreRows()) {
       for (RowResult result : scanner.nextRows()) {
@@ -90,6 +93,26 @@ public class TestFlexiblePartitioning extends BaseKuduTest {
     return rows;
   }
 
+  /**
+   * Collects the rows from a set of scan tokens.
+   *
+   * @param scanTokens the scan token builder
+   * @return the rows
+   */
+  private Set<Row> collectRows(KuduScanToken.KuduScanTokenBuilder scanTokens) throws Exception {
+    Set<Row> rows = new HashSet<>();
+    for (KuduScanToken token : scanTokens.build()) {
+      LOG.debug("Scanning token: {}", KuduScanToken.stringifySerializedToken(token.serialize(),
+                                                                             syncClient));
+
+      int existingCount = rows.size();
+      Set<Row> newRows = collectRows(token.intoScanner(syncClient));
+      rows.addAll(newRows);
+      assertEquals(existingCount + newRows.size(), rows.size());
+    }
+    return rows;
+  }
+
   private void testPartitionSchema(CreateTableOptions tableBuilder) throws Exception {
     Schema schema = createSchema();
 
@@ -112,8 +135,12 @@ public class TestFlexiblePartitioning extends BaseKuduTest {
 
       KuduScanner scanner = syncClient.newScannerBuilder(table).lowerBound(lowerBound).build();
       Set<Row> results = collectRows(scanner);
-
       assertEquals(expected, results);
+
+      KuduScanToken.KuduScanTokenBuilder scanTokens =
+          syncClient.newScanTokenBuilder(table).lowerBound(lowerBound);
+      Set<Row> tokenResults = collectRows(scanTokens);
+      assertEquals(expected, tokenResults);
     }
 
     { // Upper bound
@@ -127,8 +154,12 @@ public class TestFlexiblePartitioning extends BaseKuduTest {
                                       .exclusiveUpperBound(upperBound)
                                       .build();
       Set<Row> results = collectRows(scanner);
-
       assertEquals(expected, results);
+
+      KuduScanToken.KuduScanTokenBuilder scanTokens =
+          syncClient.newScanTokenBuilder(table).exclusiveUpperBound(upperBound);
+      Set<Row> tokenResults = collectRows(scanTokens);
+      assertEquals(expected, tokenResults);
     }
 
     { // Lower & Upper bounds
@@ -146,8 +177,14 @@ public class TestFlexiblePartitioning extends BaseKuduTest {
                                       .exclusiveUpperBound(upperBound)
                                       .build();
       Set<Row> results = collectRows(scanner);
-
       assertEquals(expected, results);
+
+      KuduScanToken.KuduScanTokenBuilder scanTokens =
+          syncClient.newScanTokenBuilder(table)
+                    .lowerBound(lowerBound)
+                    .exclusiveUpperBound(upperBound);
+      Set<Row> tokenResults = collectRows(scanTokens);
+      assertEquals(expected, tokenResults);
     }
 
     List<LocatedTablet> tablets = table.getTabletsLocations(TestTimeouts.DEFAULT_SLEEP);

http://git-wip-us.apache.org/repos/asf/kudu/blob/4fd0572e/java/kudu-client/src/test/java/org/apache/kudu/client/TestKeyEncoding.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKeyEncoding.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKeyEncoding.java
index 9b0ed96..37ddabc 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKeyEncoding.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKeyEncoding.java
@@ -16,6 +16,7 @@
 // under the License.
 package org.apache.kudu.client;
 
+import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
@@ -48,7 +49,7 @@ public class TestKeyEncoding {
   }
 
   private static void assertBytesEquals(byte[] actual, byte[] expected) {
-    assertTrue(String.format("expected: '%s', got '%s'",
+    assertTrue(String.format("expected: '%s', got: '%s'",
                              Bytes.pretty(expected),
                              Bytes.pretty(actual)),
                Bytes.equals(expected, actual));
@@ -159,13 +160,16 @@ public class TestKeyEncoding {
     rowA.addString("string", "");
     rowA.addBinary("binary", "".getBytes(Charsets.UTF_8));
 
-    assertBytesEquals(rowA.encodePrimaryKey(),
+    byte[] rowAEncoded = rowA.encodePrimaryKey();
+    assertBytesEquals(rowAEncoded,
                       "\0"
                     + "\0\0"
                     + "\0\0\0\0"
                     + "\0\0\0\0\0\0\0\0"
                     + "\0\0"
                     + "");
+    assertEquals(rowA.stringifyRowKey(),
+                 KeyEncoder.decodePrimaryKey(schema, rowAEncoded).stringifyRowKey());
 
     PartialRow rowB = schema.newPartialRow();
     rowB.addByte("int8", Byte.MAX_VALUE);
@@ -175,6 +179,7 @@ public class TestKeyEncoding {
     rowB.addString("string", "abc\1\0def");
     rowB.addBinary("binary", "\0\1binary".getBytes(Charsets.UTF_8));
 
+    byte[] rowBEncoded = rowB.encodePrimaryKey();
     assertBytesEquals(rowB.encodePrimaryKey(),
                       new byte[] {
                           -1,
@@ -184,6 +189,8 @@ public class TestKeyEncoding {
                           'a', 'b', 'c', 1, 0, 1, 'd', 'e', 'f', 0, 0,
                           0, 1, 'b', 'i', 'n', 'a', 'r', 'y',
                       });
+    assertEquals(rowB.stringifyRowKey(),
+                 KeyEncoder.decodePrimaryKey(schema, rowBEncoded).stringifyRowKey());
 
     PartialRow rowC = schema.newPartialRow();
     rowC.addByte("int8", (byte) 1);
@@ -193,7 +200,8 @@ public class TestKeyEncoding {
     rowC.addString("string", "abc\n123");
     rowC.addBinary("binary", "\0\1\2\3\4\5".getBytes(Charsets.UTF_8));
 
-    assertBytesEquals(rowC.encodePrimaryKey(),
+    byte[] rowCEncoded = rowC.encodePrimaryKey();
+    assertBytesEquals(rowCEncoded,
                       new byte[] {
                           (byte) 0x81,
                           (byte) 0x80, 2,
@@ -202,6 +210,29 @@ public class TestKeyEncoding {
                           'a', 'b', 'c', '\n', '1', '2', '3', 0, 0,
                           0, 1, 2, 3, 4, 5,
                       });
+    assertEquals(rowC.stringifyRowKey(),
+                 KeyEncoder.decodePrimaryKey(schema, rowCEncoded).stringifyRowKey());
+
+    PartialRow rowD = schema.newPartialRow();
+    rowD.addByte("int8", (byte) -1);
+    rowD.addShort("int16", (short) -2);
+    rowD.addInt("int32", -3);
+    rowD.addLong("int64", -4);
+    rowD.addString("string", "\0abc\n\1\1\0 123\1\0");
+    rowD.addBinary("binary", "\0\1\2\3\4\5\0".getBytes(Charsets.UTF_8));
+
+    byte[] rowDEncoded = rowD.encodePrimaryKey();
+    assertBytesEquals(rowDEncoded,
+                      new byte[] {
+                          (byte) 127,
+                          (byte) 127, -2,
+                          (byte) 127, -1, -1, -3,
+                          (byte) 127, -1, -1, -1, -1, -1, -1, -4,
+                          0, 1, 'a', 'b', 'c', '\n', 1, 1, 0, 1, ' ', '1', '2', '3', 1, 0, 1, 0, 0,
+                          0, 1, 2, 3, 4, 5, 0,
+                      });
+    assertEquals(rowD.stringifyRowKey(),
+                 KeyEncoder.decodePrimaryKey(schema, rowDEncoded).stringifyRowKey());
   }
 
   @Test

http://git-wip-us.apache.org/repos/asf/kudu/blob/4fd0572e/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java
index b65a81b..20d801f 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java
@@ -28,10 +28,8 @@ import static org.apache.kudu.client.RowResult.timestampToString;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterators;
-import com.google.common.collect.Lists;
 
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.Executors;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -409,7 +407,11 @@ public class TestKuduClient extends BaseKuduTest {
     tokenBuilder.setProjectedColumnIndexes(ImmutableList.<Integer>of());
     List<KuduScanToken> tokens = tokenBuilder.build();
     assertEquals(16, tokens.size());
-    assertEquals(100, countScanTokenRows(tokens));
+
+    for (KuduScanToken token : tokens) {
+      // Sanity check to make sure the debug printing does not throw.
+      LOG.debug(KuduScanToken.stringifySerializedToken(token.serialize(), syncClient));
+    }
   }
 
   /**
@@ -465,6 +467,11 @@ public class TestKuduClient extends BaseKuduTest {
     List<KuduScanToken> tokens = tokenBuilder.build();
     assertEquals(6, tokens.size());
     assertEquals('f' - 'a' + 'z' - 'h', countScanTokenRows(tokens));
+
+    for (KuduScanToken token : tokens) {
+      // Sanity check to make sure the debug printing does not throw.
+      LOG.debug(KuduScanToken.stringifySerializedToken(token.serialize(), syncClient));
+    }
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/kudu/blob/4fd0572e/java/kudu-client/src/test/java/org/apache/kudu/client/TestOperation.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestOperation.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestOperation.java
index 161283b..77baf81 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestOperation.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestOperation.java
@@ -148,9 +148,10 @@ public class TestOperation {
     row.addString("c5", "c5_val");
     row.addBinary("c6", Bytes.fromString("c6_val"));
 
-    assertEquals("(int8 c0=1, int16 c1=2, int32 c2=3, int64 c3=4, timestamp c4=5, string" +
-            " c5=c5_val, binary c6=\"c6_val\")",
-        insert.getRow().stringifyRowKey());
+    assertEquals("(int8 c0=1, int16 c1=2, int32 c2=3, int64 c3=4, " +
+                     "timestamp c4=1970-01-01T00:00:00.000005Z, string c5=c5_val, " +
+                     "binary c6=\"c6_val\")",
+                 insert.getRow().stringifyRowKey());
 
     // Test an incomplete row key.
     insert = new Insert(table);

http://git-wip-us.apache.org/repos/asf/kudu/blob/4fd0572e/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableInputFormat.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableInputFormat.java b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableInputFormat.java
index 0772833..6a15cc1 100644
--- a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableInputFormat.java
+++ b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableInputFormat.java
@@ -376,12 +376,9 @@ public class KuduTableInputFormat extends InputFormat<NullWritable, RowResult>
       }
 
       split = (TableSplit) inputSplit;
-
-      try {
-        scanner = KuduScanToken.deserializeIntoScanner(split.getScanToken(), client);
-      } catch (Exception e) {
-        throw new IOException(e);
-      }
+      LOG.debug("Creating scanner for token: {}",
+                KuduScanToken.stringifySerializedToken(split.getScanToken(), client));
+      scanner = KuduScanToken.deserializeIntoScanner(split.getScanToken(), client);
 
       // Calling this now to set iterator.
       tryRefreshIterator();