You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by mm...@apache.org on 2019/09/19 22:34:41 UTC
[accumulo-testing] branch master updated: Update
ContinuousInputFormat to use col vis
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git
The following commit(s) were added to refs/heads/master by this push:
new 24f4898 Update ContinuousInputFormat to use col vis
24f4898 is described below
commit 24f489805f8232860078a55ce464033bd4783570
Author: Mike Miller <mm...@apache.org>
AuthorDate: Thu Sep 19 18:33:26 2019 -0400
Update ContinuousInputFormat to use col vis
---
.../testing/continuous/ContinuousIngest.java | 23 +++++++++++++---------
.../testing/continuous/ContinuousInputFormat.java | 10 ++++++++--
2 files changed, 22 insertions(+), 11 deletions(-)
diff --git a/src/main/java/org/apache/accumulo/testing/continuous/ContinuousIngest.java b/src/main/java/org/apache/accumulo/testing/continuous/ContinuousIngest.java
index c5229f7..0f6b04d 100644
--- a/src/main/java/org/apache/accumulo/testing/continuous/ContinuousIngest.java
+++ b/src/main/java/org/apache/accumulo/testing/continuous/ContinuousIngest.java
@@ -97,15 +97,7 @@ public class ContinuousIngest {
try (ContinuousEnv env = new ContinuousEnv(args)) {
- String vis = env.getTestProperty(TestProps.CI_INGEST_VISIBILITIES);
- if (vis == null) {
- visibilities = Collections.singletonList(new ColumnVisibility());
- } else {
- visibilities = new ArrayList<>();
- for (String v : vis.split(",")) {
- visibilities.add(new ColumnVisibility(v.trim()));
- }
- }
+ visibilities = parseVisibilities(env.getTestProperty(TestProps.CI_INGEST_VISIBILITIES));
long rowMin = env.getRowMin();
long rowMax = env.getRowMax();
@@ -215,6 +207,19 @@ public class ContinuousIngest {
}
}
+ public static List<ColumnVisibility> parseVisibilities(String visString) {
+ List<ColumnVisibility> vis;
+ if (visString == null) {
+ vis = Collections.singletonList(new ColumnVisibility());
+ } else {
+ vis = new ArrayList<>();
+ for (String v : visString.split(",")) {
+ vis.add(new ColumnVisibility(v.trim()));
+ }
+ }
+ return vis;
+ }
+
private static long flush(BatchWriter bw, long count, final int flushInterval, long lastFlushTime)
throws MutationsRejectedException {
long t1 = System.currentTimeMillis();
diff --git a/src/main/java/org/apache/accumulo/testing/continuous/ContinuousInputFormat.java b/src/main/java/org/apache/accumulo/testing/continuous/ContinuousInputFormat.java
index ecf8172..4e17bb7 100644
--- a/src/main/java/org/apache/accumulo/testing/continuous/ContinuousInputFormat.java
+++ b/src/main/java/org/apache/accumulo/testing/continuous/ContinuousInputFormat.java
@@ -34,6 +34,7 @@ import java.util.zip.Checksum;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.accumulo.testing.TestProps;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Writable;
@@ -57,6 +58,7 @@ public class ContinuousInputFormat extends InputFormat<Key,Value> {
private static final String PROP_FAM_MAX = "mrbulk.fam.max";
private static final String PROP_QUAL_MAX = "mrbulk.qual.max";
private static final String PROP_CHECKSUM = "mrbulk.checksum";
+ private static final String PROP_VIS = "mrbulk.vis";
private static class RandomSplit extends InputSplit implements Writable {
@Override
@@ -96,6 +98,7 @@ public class ContinuousInputFormat extends InputFormat<Key,Value> {
conf.setInt(PROP_QUAL_MAX, env.getMaxColQ());
conf.setBoolean(PROP_CHECKSUM,
Boolean.parseBoolean(env.getTestProperty(TestProps.CI_INGEST_CHECKSUM)));
+ conf.set(PROP_VIS, env.getTestProperty(TestProps.CI_INGEST_VISIBILITIES));
}
@Override
@@ -112,6 +115,7 @@ public class ContinuousInputFormat extends InputFormat<Key,Value> {
long maxRow;
int maxFam;
int maxQual;
+ List<ColumnVisibility> visibilities;
boolean checksum;
Key prevKey;
@@ -128,6 +132,7 @@ public class ContinuousInputFormat extends InputFormat<Key,Value> {
maxFam = job.getConfiguration().getInt(PROP_FAM_MAX, Short.MAX_VALUE);
maxQual = job.getConfiguration().getInt(PROP_QUAL_MAX, Short.MAX_VALUE);
checksum = job.getConfiguration().getBoolean(PROP_CHECKSUM, false);
+ visibilities = ContinuousIngest.parseVisibilities(job.getConfiguration().get(PROP_VIS));
random = new Random(new SecureRandom().nextLong());
@@ -140,15 +145,16 @@ public class ContinuousInputFormat extends InputFormat<Key,Value> {
byte[] fam = genCol(random.nextInt(maxFam));
byte[] qual = genCol(random.nextInt(maxQual));
+ byte[] cv = visibilities.get(random.nextInt(visibilities.size())).flatten();
if (cksum != null) {
cksum.update(row);
cksum.update(fam);
cksum.update(qual);
- cksum.update(new byte[0]); // TODO col vis
+ cksum.update(cv);
}
- return new Key(row, fam, qual);
+ return new Key(row, fam, qual, cv);
}
private byte[] createValue(byte[] ingestInstanceId, byte[] prevRow, Checksum cksum) {