You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by mm...@apache.org on 2019/07/09 16:55:42 UTC
[accumulo-examples] branch master updated: Refactor out
DefaultFormatter (#35) (#56)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git
The following commit(s) were added to refs/heads/master by this push:
new 09f8d64 Refactor out DefaultFormatter (#35) (#56)
09f8d64 is described below
commit 09f8d64bbb6d0374ed7858d41353bcc41235e0e5
Author: lbschanno <52...@users.noreply.github.com>
AuthorDate: Tue Jul 9 12:55:38 2019 -0400
Refactor out DefaultFormatter (#35) (#56)
* Refactor out usage of DefaultFormatter. It is an internal-only unstable
API that is not recommended for users, and as such is unsuitable to
include in examples.
---
contrib/import-control.xml | 1 -
.../examples/filedata/ChunkInputFormat.java | 17 ++--
.../accumulo/examples/mapreduce/TableToFile.java | 4 +-
.../apache/accumulo/examples/util/FormatUtil.java | 97 ++++++++++++++++++++++
4 files changed, 110 insertions(+), 9 deletions(-)
diff --git a/contrib/import-control.xml b/contrib/import-control.xml
index 0c0f647..f4628ff 100644
--- a/contrib/import-control.xml
+++ b/contrib/import-control.xml
@@ -36,7 +36,6 @@
<!-- TODO refactor code to remove the following exceptions -->
<allow class="org.apache.accumulo.tracer.TraceDump"/>
<allow class="org.apache.accumulo.core.trace.DistributedTrace"/>
- <allow class="org.apache.accumulo.core.util.format.DefaultFormatter"/>
<!-- End TODO section -->
<!-- disallow everything else coming from accumulo -->
diff --git a/src/main/java/org/apache/accumulo/examples/filedata/ChunkInputFormat.java b/src/main/java/org/apache/accumulo/examples/filedata/ChunkInputFormat.java
index 50c385b..49f592d 100644
--- a/src/main/java/org/apache/accumulo/examples/filedata/ChunkInputFormat.java
+++ b/src/main/java/org/apache/accumulo/examples/filedata/ChunkInputFormat.java
@@ -25,7 +25,7 @@ import java.util.Map.Entry;
import org.apache.accumulo.core.client.mapreduce.InputFormatBase;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.util.format.DefaultFormatter;
+import org.apache.accumulo.examples.util.FormatUtil;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
@@ -40,7 +40,7 @@ import com.google.common.collect.PeekingIterator;
public class ChunkInputFormat extends InputFormatBase<List<Entry<Key,Value>>,InputStream> {
@Override
public RecordReader<List<Entry<Key,Value>>,InputStream> createRecordReader(InputSplit split,
- TaskAttemptContext context) throws IOException, InterruptedException {
+ TaskAttemptContext context) {
return new RecordReaderBase<List<Entry<Key,Value>>,InputStream>() {
private PeekingIterator<Entry<Key,Value>> peekingScannerIterator;
@@ -53,7 +53,9 @@ public class ChunkInputFormat extends InputFormatBase<List<Entry<Key,Value>>,Inp
}
@Override
- public boolean nextKeyValue() throws IOException, InterruptedException {
+ public boolean nextKeyValue() throws IOException {
+ log.debug("nextKeyValue called");
+
currentK.clear();
if (peekingScannerIterator.hasNext()) {
++numKeysRead;
@@ -61,14 +63,17 @@ public class ChunkInputFormat extends InputFormatBase<List<Entry<Key,Value>>,Inp
while (!entry.getKey().getColumnFamily().equals(FileDataIngest.CHUNK_CF)) {
currentK.add(entry);
peekingScannerIterator.next();
- if (!peekingScannerIterator.hasNext())
+ if (!peekingScannerIterator.hasNext()) {
return true;
+ }
entry = peekingScannerIterator.peek();
}
currentKey = entry.getKey();
((ChunkInputStream) currentV).setSource(peekingScannerIterator);
- if (log.isTraceEnabled())
- log.trace("Processing key/value pair: " + DefaultFormatter.formatEntry(entry, true));
+ if (log.isTraceEnabled()) {
+ log.trace("Processing key/value pair: " + FormatUtil.formatTableEntry(entry, true));
+ }
+
return true;
}
return false;
diff --git a/src/main/java/org/apache/accumulo/examples/mapreduce/TableToFile.java b/src/main/java/org/apache/accumulo/examples/mapreduce/TableToFile.java
index 9621821..456546a 100644
--- a/src/main/java/org/apache/accumulo/examples/mapreduce/TableToFile.java
+++ b/src/main/java/org/apache/accumulo/examples/mapreduce/TableToFile.java
@@ -25,8 +25,8 @@ import java.util.Map;
import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.util.format.DefaultFormatter;
import org.apache.accumulo.examples.cli.ClientOpts;
+import org.apache.accumulo.examples.util.FormatUtil;
import org.apache.accumulo.hadoop.mapreduce.AccumuloInputFormat;
import org.apache.accumulo.hadoop.mapreduce.InputFormatBuilder;
import org.apache.hadoop.fs.Path;
@@ -59,7 +59,7 @@ public class TableToFile {
@Override
public void map(Key row, Value data, Context context) throws IOException, InterruptedException {
Map.Entry<Key,Value> entry = new SimpleImmutableEntry<>(row, data);
- context.write(NullWritable.get(), new Text(DefaultFormatter.formatEntry(entry, false)));
+ context.write(NullWritable.get(), new Text(FormatUtil.formatTableEntry(entry, false)));
context.setStatus("Outputed Value");
}
}
diff --git a/src/main/java/org/apache/accumulo/examples/util/FormatUtil.java b/src/main/java/org/apache/accumulo/examples/util/FormatUtil.java
new file mode 100644
index 0000000..72a64b5
--- /dev/null
+++ b/src/main/java/org/apache/accumulo/examples/util/FormatUtil.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.examples.util;
+
+import java.util.Map;
+
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.hadoop.io.Text;
+
+public final class FormatUtil {
+
+ /**
+ * Format and return the specified table entry as a human-readable String suitable for logging.
+ * <br/>
+ * If {@code includeTimestamp} is true, the entry will be formatted as: <br/>
+ * {@literal <row> <columnFamily>:<columnQualifier> <columnVisibility> <timestamp>\t<value>} <br/>
+ * If false, the entry will be formatted as: <br/>
+ * {@literal <row> <columnFamily>:<columnQualifier> <columnVisibility>\t<value>} <br/>
+ * Examples: <br/>
+ * {@literal a ~chunk:\x00\x00\x00d\x00\x00\x00\x00 [A&B] 9223372036854775807 asdfjkl;}
+ * {@literal a ~chunk:\x00\x00\x00d\x00\x00\x00\x00 [A&B] asdfjkl;}
+ *
+ * @param entry
+ * the table entry to format
+ * @param includeTimestamp
+ * if true, include the timestamp in the returned result
+ * @return the specified entry as a formatted String, or null if the entry is null
+ */
+ public static String formatTableEntry(final Map.Entry<Key,Value> entry,
+ final boolean includeTimestamp) {
+ if (entry == null) {
+ return null;
+ }
+
+ Key key = entry.getKey();
+ StringBuilder sb = new StringBuilder();
+ Text buffer = new Text();
+
+ // Append row.
+ appendBytes(sb, key.getRow(buffer).getBytes()).append(" ");
+
+ // Append column family.
+ appendBytes(sb, key.getColumnFamily().getBytes()).append(":");
+
+ // Append column qualifier.
+ appendBytes(sb, key.getColumnQualifier().getBytes()).append(" ");
+
+ // Append visibility and timestamp.
+ sb.append(new ColumnVisibility(key.getColumnVisibility(buffer)));
+
+ if (includeTimestamp) {
+ sb.append(" ").append(entry.getKey().getTimestamp());
+ }
+
+ // Append value.
+ Value value = entry.getValue();
+ if (value != null && value.getSize() > 0) {
+ sb.append("\t");
+ appendBytes(sb, value.get());
+ }
+ return sb.toString();
+ }
+
+ private static StringBuilder appendBytes(final StringBuilder sb, final byte[] ba) {
+ for (byte b : ba) {
+ int c = 0xff & b;
+ if (c == '\\') {
+ sb.append("\\\\");
+ } else if (c >= 32 && c <= 126) {
+ sb.append((char) c);
+ } else {
+ sb.append("\\x").append(String.format("%02X", c));
+ }
+ }
+ return sb;
+ }
+
+ private FormatUtil() {
+ throw new UnsupportedOperationException();
+ }
+}