You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2013/11/15 03:56:41 UTC
[1/5] git commit: ACCUMULO-1614 use zipfian dist in conditional RW so
there are collisions as data size increases
Updated Branches:
refs/heads/master 209cc076b -> da199d9cb
ACCUMULO-1614 use zipfian dist in conditional RW so there are collisions as data size increases
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/484053a2
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/484053a2
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/484053a2
Branch: refs/heads/master
Commit: 484053a2f0823706e0a386202da75838f19d22d1
Parents: feb0f31
Author: Keith Turner <kt...@apache.org>
Authored: Thu Nov 14 19:37:12 2013 -0500
Committer: Keith Turner <kt...@apache.org>
Committed: Thu Nov 14 19:37:12 2013 -0500
----------------------------------------------------------------------
.../test/randomwalk/conditional/Init.java | 28 +++++++++++++++++---
.../test/randomwalk/conditional/Setup.java | 10 ++++---
.../test/randomwalk/conditional/Transfer.java | 15 ++++++++---
.../randomwalk/conf/modules/Conditional.xml | 14 +++++-----
4 files changed, 49 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/484053a2/test/src/main/java/org/apache/accumulo/test/randomwalk/conditional/Init.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/conditional/Init.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/conditional/Init.java
index c336e97..e3de7d8 100644
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/conditional/Init.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/conditional/Init.java
@@ -16,13 +16,19 @@
*/
package org.apache.accumulo.test.randomwalk.conditional;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.Properties;
+import java.util.Random;
+import java.util.TreeSet;
import org.apache.accumulo.core.client.ConditionalWriter;
+import org.apache.accumulo.core.client.ConditionalWriter.Status;
import org.apache.accumulo.core.data.Condition;
import org.apache.accumulo.core.data.ConditionalMutation;
import org.apache.accumulo.test.randomwalk.State;
import org.apache.accumulo.test.randomwalk.Test;
+import org.apache.hadoop.io.Text;
/**
*
@@ -34,10 +40,25 @@ public class Init extends Test {
int numBanks = (Integer) state.get("numBanks");
int numAccts = (Integer) state.get("numAccts");
+
+ // add some splits to spread ingest out a little
+ TreeSet<Text> splits = new TreeSet<Text>();
+ for (int i = 1; i < 10; i++)
+ splits.add(new Text(Utils.getBank((int) (numBanks * .1 * i))));
+ state.getConnector().tableOperations().addSplits((String) state.get("tableName"), splits);
+ log.debug("Added splits " + splits);
+
+ ArrayList<Integer> banks = new ArrayList<Integer>();
+ for (int i = 0; i < numBanks; i++)
+ banks.add(i);
+ // shuffle for case when multiple threads are adding banks
+ Collections.shuffle(banks, (Random) state.get("rand"));
+
ConditionalWriter cw = (ConditionalWriter) state.get("cw");
- for (int i = 0; i < numBanks; i++) {
+ for (int i : banks) {
ConditionalMutation m = null;
+ int acceptedCount = 0;
for (int j = 0; j < numAccts; j++) {
String cf = Utils.getAccount(j);
if (m == null) {
@@ -49,7 +70,8 @@ public class Init extends Test {
m.put(cf, "seq", Utils.getSeq(0));
if (j % 1000 == 0) {
- cw.write(m);
+ if (cw.write(m).getStatus() == Status.ACCEPTED)
+ acceptedCount++;
m = null;
}
@@ -57,7 +79,7 @@ public class Init extends Test {
if (m != null)
cw.write(m);
- log.debug("Added bank " + Utils.getBank(i));
+ log.debug("Added bank " + Utils.getBank(i) + " " + acceptedCount);
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/484053a2/test/src/main/java/org/apache/accumulo/test/randomwalk/conditional/Setup.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/conditional/Setup.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/conditional/Setup.java
index 2f39e29..0aa36c4 100644
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/conditional/Setup.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/conditional/Setup.java
@@ -22,6 +22,7 @@ import java.util.Random;
import org.apache.accumulo.core.client.ConditionalWriter;
import org.apache.accumulo.core.client.ConditionalWriterConfig;
import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.test.randomwalk.State;
import org.apache.accumulo.test.randomwalk.Test;
@@ -32,11 +33,11 @@ public class Setup extends Test {
Random rand = new Random();
state.set("rand", rand);
- int numBanks = Integer.parseInt(props.getProperty("numBanks", "10"));
+ int numBanks = Integer.parseInt(props.getProperty("numBanks", "1000"));
log.debug("numBanks = " + numBanks);
state.set("numBanks", numBanks);
- int numAccts = Integer.parseInt(props.getProperty("numAccts", "1000"));
+ int numAccts = Integer.parseInt(props.getProperty("numAccts", "10000"));
log.debug("numAccts = " + numAccts);
state.set("numAccts", numAccts);
@@ -46,11 +47,12 @@ public class Setup extends Test {
try {
state.getConnector().tableOperations().create(tableName);
log.debug("created table " + tableName);
+ boolean blockCache = rand.nextBoolean();
+ state.getConnector().tableOperations().setProperty(tableName, Property.TABLE_BLOCKCACHE_ENABLED.getKey(), blockCache + "");
+ log.debug("set " + Property.TABLE_BLOCKCACHE_ENABLED.getKey() + " " + blockCache);
} catch (TableExistsException tee) {}
-
-
ConditionalWriter cw = state.getConnector()
.createConditionalWriter(tableName, new ConditionalWriterConfig().setMaxWriteThreads(1));
state.set("cw", cw);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/484053a2/test/src/main/java/org/apache/accumulo/test/randomwalk/conditional/Transfer.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/conditional/Transfer.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/conditional/Transfer.java
index 70aa3dd..93f0d55 100644
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/conditional/Transfer.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/conditional/Transfer.java
@@ -33,6 +33,7 @@ import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.test.randomwalk.State;
import org.apache.accumulo.test.randomwalk.Test;
+import org.apache.commons.math.distribution.ZipfDistributionImpl;
import org.apache.hadoop.io.Text;
/**
@@ -64,11 +65,17 @@ public class Transfer extends Test {
Connector conn = state.getConnector();
int numAccts = (Integer) state.get("numAccts");
- String bank = Utils.getBank(rand.nextInt((Integer) state.get("numBanks")));
- String acct1 = Utils.getAccount(rand.nextInt(numAccts));
- String acct2 = Utils.getAccount(rand.nextInt(numAccts));
- while (acct2.equals(acct1))
+ // note: non integer exponents are slow
+
+ ZipfDistributionImpl zdiBanks = new ZipfDistributionImpl((Integer) state.get("numBanks"), 1);
+ String bank = Utils.getBank(zdiBanks.inverseCumulativeProbability(rand.nextDouble()));
+ ZipfDistributionImpl zdiAccts = new ZipfDistributionImpl(numAccts, 1);
+ String acct1 = Utils.getAccount(zdiAccts.inverseCumulativeProbability(rand.nextDouble()));
+ String acct2 = Utils.getAccount(zdiAccts.inverseCumulativeProbability(rand.nextDouble()));
+ while (acct2.equals(acct1)) {
+ // intentionally not using zipf distribution to pick on retry
acct2 = Utils.getAccount(rand.nextInt(numAccts));
+ }
// TODO document how data should be read when using ConditionalWriter
Scanner scanner = new IsolatedScanner(conn.createScanner(table, Authorizations.EMPTY));
http://git-wip-us.apache.org/repos/asf/accumulo/blob/484053a2/test/system/randomwalk/conf/modules/Conditional.xml
----------------------------------------------------------------------
diff --git a/test/system/randomwalk/conf/modules/Conditional.xml b/test/system/randomwalk/conf/modules/Conditional.xml
index d33d36c..54ff7ab 100644
--- a/test/system/randomwalk/conf/modules/Conditional.xml
+++ b/test/system/randomwalk/conf/modules/Conditional.xml
@@ -23,17 +23,17 @@
<init id="ct.Setup"/>
<node id="dummy.ToAll">
- <edge id="ct.Compact" weight="100"/>
- <edge id="ct.Flush" weight="100"/>
- <edge id="ct.Merge" weight="100"/>
- <edge id="ct.Split" weight="100"/>
+ <edge id="ct.Compact" weight="1"/>
+ <edge id="ct.Flush" weight="1"/>
+ <edge id="ct.Merge" weight="1"/>
+ <edge id="ct.Split" weight="1"/>
<edge id="ct.Transfer" weight="100000"/>
- <edge id="ct.Verify" weight="500"/>
+ <edge id="ct.Verify" weight="2"/>
</node>
<node id="ct.Setup">
- <property key="numAccts" value="1000"/>
- <property key="numBanks" value="10"/>
+ <property key="numAccts" value="10000"/>
+ <property key="numBanks" value="1000"/>
<edge id="ct.Init" weight="1"/>
</node>
[3/5] git commit: Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Posted by ct...@apache.org.
Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Conflicts:
core/src/main/java/org/apache/accumulo/core/client/AccumuloSecurityException.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/ac20fe06
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/ac20fe06
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/ac20fe06
Branch: refs/heads/master
Commit: ac20fe06cf7627bc7f25d455d4bff7b4fa057a34
Parents: 7551b55 268028f
Author: Christopher Tubbs <ct...@apache.org>
Authored: Thu Nov 14 21:51:29 2013 -0500
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Thu Nov 14 21:51:29 2013 -0500
----------------------------------------------------------------------
.../org/apache/accumulo/core/client/AccumuloSecurityException.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ac20fe06/core/src/main/java/org/apache/accumulo/core/client/AccumuloSecurityException.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/AccumuloSecurityException.java
index 890db36,0000000..ea73640
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/AccumuloSecurityException.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/AccumuloSecurityException.java
@@@ -1,135 -1,0 +1,135 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client;
+
+import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+
+/**
+ * An Accumulo Exception for security violations, authentication failures, authorization failures, etc.
+ *
+ */
+public class AccumuloSecurityException extends Exception {
+ private static final long serialVersionUID = 1L;
+
+ private static String getDefaultErrorMessage(final SecurityErrorCode errorcode) {
- switch (errorcode) {
++ switch (errorcode == null ? SecurityErrorCode.DEFAULT_SECURITY_ERROR : errorcode) {
+ case BAD_CREDENTIALS:
+ return "Username or Password is Invalid";
+ case CONNECTION_ERROR:
+ return "Connection Error Occurred";
+ case PERMISSION_DENIED:
+ return "User does not have permission to perform this action";
+ case USER_DOESNT_EXIST:
+ return "The user does not exist";
+ case USER_EXISTS:
+ return "The user exists";
+ case GRANT_INVALID:
+ return "The GRANT permission cannot be granted or revoked";
+ case BAD_AUTHORIZATIONS:
+ return "The user does not have the specified authorizations assigned";
+ case UNSUPPORTED_OPERATION:
+ return "The configured security handler does not support this operation";
+ case INVALID_TOKEN:
+ return "The configured authenticator does not accept this type of token";
+ case AUTHENTICATOR_FAILED:
+ return "The configured authenticator failed for some reason";
+ case AUTHORIZOR_FAILED:
+ return "The configured authorizor failed for some reason";
+ case PERMISSIONHANDLER_FAILED:
+ return "The configured permission handler failed for some reason";
+ case TOKEN_EXPIRED:
+ return "The supplied token expired, please update and try again";
+ case INSUFFICIENT_PROPERTIES:
+ return "The login properties supplied are not sufficient for authentication. Please check the requested properties and try again";
+ case DEFAULT_SECURITY_ERROR:
+ default:
+ return "Unknown security exception";
+ }
+ }
+
+ private String user;
+ private SecurityErrorCode errorCode;
+
+ /**
+ * @return this exception as a thrift exception
+ */
+ public ThriftSecurityException asThriftException() {
+ return new ThriftSecurityException(user, errorCode);
+ }
+
+ /**
+ * @param user
+ * the relevant user for the security violation
+ * @param errorcode
+ * the specific reason for this exception
+ * @param cause
+ * the exception that caused this violation
+ */
+ public AccumuloSecurityException(final String user, final SecurityErrorCode errorcode, final Throwable cause) {
+ super(getDefaultErrorMessage(errorcode), cause);
+ this.user = user;
+ this.errorCode = errorcode == null ? SecurityErrorCode.DEFAULT_SECURITY_ERROR : errorcode;
+ }
+
+ /**
+ * @param user
+ * the relevant user for the security violation
+ * @param errorcode
+ * the specific reason for this exception
+ */
+ public AccumuloSecurityException(final String user, final SecurityErrorCode errorcode) {
+ super(getDefaultErrorMessage(errorcode));
+ this.user = user;
+ this.errorCode = errorcode == null ? SecurityErrorCode.DEFAULT_SECURITY_ERROR : errorcode;
+ }
+
+ /**
+ * @return the relevant user for the security violation
+ */
+ public String getUser() {
+ return user;
+ }
+
+ public void setUser(String s) {
+ this.user = s;
+ }
+
+ /**
+ * @return the specific reason for this exception
+ * @since 1.5.0
+ */
+
+ public org.apache.accumulo.core.client.security.SecurityErrorCode getSecurityErrorCode() {
+ return org.apache.accumulo.core.client.security.SecurityErrorCode.valueOf(errorCode.name());
+ }
+
+ /**
+ * @return the specific reason for this exception
+ *
+ * @deprecated since 1.5.0; Use {@link #getSecurityErrorCode()} instead.
+ */
+ @Deprecated
+ public org.apache.accumulo.core.security.thrift.SecurityErrorCode getErrorCode() {
+ return org.apache.accumulo.core.security.thrift.SecurityErrorCode.valueOf(errorCode.name());
+ }
+
+ @Override
+ public String getMessage() {
+ return "Error " + errorCode + " for user " + user + " - " + super.getMessage();
+ }
+}
[4/5] git commit: Merge branch '1.5.1-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by ct...@apache.org.
Merge branch '1.5.1-SNAPSHOT' into 1.6.0-SNAPSHOT
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/28bbb56f
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/28bbb56f
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/28bbb56f
Branch: refs/heads/master
Commit: 28bbb56f32a49a2d2dd0a7730ec90b7455a0c9d0
Parents: 484053a ac20fe0
Author: Christopher Tubbs <ct...@apache.org>
Authored: Thu Nov 14 21:52:39 2013 -0500
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Thu Nov 14 21:52:39 2013 -0500
----------------------------------------------------------------------
.../org/apache/accumulo/core/client/AccumuloSecurityException.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/28bbb56f/core/src/main/java/org/apache/accumulo/core/client/AccumuloSecurityException.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/AccumuloSecurityException.java
index f626f4d,ea73640..06b148d
--- a/core/src/main/java/org/apache/accumulo/core/client/AccumuloSecurityException.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/AccumuloSecurityException.java
@@@ -26,9 -25,9 +26,9 @@@ import org.apache.commons.lang.StringUt
*/
public class AccumuloSecurityException extends Exception {
private static final long serialVersionUID = 1L;
-
+
private static String getDefaultErrorMessage(final SecurityErrorCode errorcode) {
- switch (errorcode) {
+ switch (errorcode == null ? SecurityErrorCode.DEFAULT_SECURITY_ERROR : errorcode) {
case BAD_CREDENTIALS:
return "Username or Password is Invalid";
case CONNECTION_ERROR:
[2/5] git commit: ACCUMULO-1891 sets errorcode to default if it is
null during message lookup.
Posted by ct...@apache.org.
ACCUMULO-1891 sets errorcode to default if it is null during message lookup.
Signed-off-by: Christopher Tubbs <ct...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/268028f8
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/268028f8
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/268028f8
Branch: refs/heads/master
Commit: 268028f8c17118208d45119ec26fc3e929a7f179
Parents: 1e96138
Author: Sean Busbey <bu...@clouderagovt.com>
Authored: Wed Nov 13 16:15:10 2013 -0600
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Thu Nov 14 21:48:34 2013 -0500
----------------------------------------------------------------------
.../apache/accumulo/core/client/AccumuloSecurityException.java | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/268028f8/src/core/src/main/java/org/apache/accumulo/core/client/AccumuloSecurityException.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/AccumuloSecurityException.java b/src/core/src/main/java/org/apache/accumulo/core/client/AccumuloSecurityException.java
index d1bb0cb..a8c3187 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/client/AccumuloSecurityException.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/client/AccumuloSecurityException.java
@@ -27,6 +27,9 @@ public class AccumuloSecurityException extends Exception {
private static final long serialVersionUID = 1L;
private static String getDefaultErrorMessage(SecurityErrorCode errorcode) {
+ if (null == errorcode) {
+ errorcode = SecurityErrorCode.DEFAULT_SECURITY_ERROR;
+ }
switch (errorcode) {
case BAD_CREDENTIALS:
return "Username or Password is Invalid";
[5/5] git commit: Merge branch '1.6.0-SNAPSHOT'
Posted by ct...@apache.org.
Merge branch '1.6.0-SNAPSHOT'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/da199d9c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/da199d9c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/da199d9c
Branch: refs/heads/master
Commit: da199d9cbfd6db8ff535d088c74783428a6feb40
Parents: 209cc07 28bbb56
Author: Christopher Tubbs <ct...@apache.org>
Authored: Thu Nov 14 21:54:07 2013 -0500
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Thu Nov 14 21:54:07 2013 -0500
----------------------------------------------------------------------
.../core/client/AccumuloSecurityException.java | 2 +-
.../test/randomwalk/conditional/Init.java | 28 +++++++++++++++++---
.../test/randomwalk/conditional/Setup.java | 10 ++++---
.../test/randomwalk/conditional/Transfer.java | 15 ++++++++---
.../randomwalk/conf/modules/Conditional.xml | 14 +++++-----
5 files changed, 50 insertions(+), 19 deletions(-)
----------------------------------------------------------------------