You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2013/07/31 03:59:21 UTC
[01/50] [abbrv] git commit: ACCUMULO-1341 configure logger for port
search in miniAcc
Updated Branches:
refs/heads/1.5.1-SNAPSHOT 1c44069a3 -> 8622b4d67
ACCUMULO-1341 configure logger for port search in miniAcc
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.4@1471672 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/bb71618b
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/bb71618b
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/bb71618b
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: bb71618be3ddb6f31d865d9b618397c5939ff8d0
Parents: b823578
Author: Keith Turner <kt...@apache.org>
Authored: Wed Apr 24 21:00:01 2013 +0000
Committer: Keith Turner <kt...@apache.org>
Committed: Wed Apr 24 21:00:01 2013 +0000
----------------------------------------------------------------------
.../main/java/org/apache/accumulo/test/MiniAccumuloCluster.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/bb71618b/src/test/src/main/java/org/apache/accumulo/test/MiniAccumuloCluster.java
----------------------------------------------------------------------
diff --git a/src/test/src/main/java/org/apache/accumulo/test/MiniAccumuloCluster.java b/src/test/src/main/java/org/apache/accumulo/test/MiniAccumuloCluster.java
index d07b430..eb6feb2 100644
--- a/src/test/src/main/java/org/apache/accumulo/test/MiniAccumuloCluster.java
+++ b/src/test/src/main/java/org/apache/accumulo/test/MiniAccumuloCluster.java
@@ -23,7 +23,6 @@ import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
-import java.lang.RuntimeException;
import java.net.ServerSocket;
import java.util.ArrayList;
import java.util.Arrays;
@@ -268,6 +267,7 @@ public class MiniAccumuloCluster {
appendProp(fileWriter, Property.TSERV_NATIVEMAP_ENABLED, "false", siteConfig);
appendProp(fileWriter, Property.TRACE_PORT, "" + getRandomFreePort(), siteConfig);
appendProp(fileWriter, Property.LOGGER_SORT_BUFFER_SIZE, "50M", siteConfig);
+ appendProp(fileWriter, Property.LOGGER_PORTSEARCH, "true", siteConfig);
// since there is a small amount of memory, check more frequently for majc... setting may not be needed in 1.5
appendProp(fileWriter, Property.TSERV_MAJC_DELAY, "3", siteConfig);
[46/50] [abbrv] git commit: ACCUMULO-1603 Undo some unnecessary
gitignore entries and add some for javadoc and nativeMap shared objects
Posted by el...@apache.org.
ACCUMULO-1603 Undo some unnecessary gitignore entries and add some for javadoc
and nativeMap shared objects
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/726a5a8a
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/726a5a8a
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/726a5a8a
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 726a5a8a332e0ed777a61ca4ca0957faeb39a337
Parents: f5b2efa
Author: Josh Elser <el...@apache.org>
Authored: Tue Jul 23 22:27:19 2013 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Tue Jul 23 22:27:19 2013 -0400
----------------------------------------------------------------------
.gitignore | 14 ++------------
1 file changed, 2 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/726a5a8a/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 7afd1e5..05fc69f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -50,6 +50,7 @@ target/
# /docs/
/docs/config.html
/docs/accumulo_user_manual.pdf
+/docs/apidocs
# /docs/src/user_manual/
/docs/src/user_manual/accumulo_user_manual.toc
@@ -65,6 +66,7 @@ target/
# /server/src/main/c++/nativeMap/
/server/src/main/c++/nativeMap/*.jnilib
+/server/src/main/c++/nativeMap/*.so
# /test/system/auto/
/test/system/auto/fake_disk_failure.so
@@ -73,15 +75,3 @@ target/
/test/system/continuous/continuous-env.sh
/test/system/continuous/walkers.txt
/test/system/continuous/ingesters.txt
-
-# src/minicluster
-minicluster/.classpath
-minicluster/.project
-minicluster/.settings
-minicluster/target
-
-# src/proxy
-proxy/.classpath
-proxy/.project
-proxy/.settings
-proxy/target
[49/50] [abbrv] git commit: Merge branch '1.4.4-SNAPSHOT' into
1.5.1-SNAPSHOT-1.4.4-SNAPSHOT-merged
Posted by el...@apache.org.
Merge branch '1.4.4-SNAPSHOT' into 1.5.1-SNAPSHOT-1.4.4-SNAPSHOT-merged
Conflicts:
trace/src/main/java/org/apache/accumulo/trace/instrument/impl/MilliSpan.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/8e1e2a57
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/8e1e2a57
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/8e1e2a57
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 8e1e2a57d150923574ace0e92945fdca14de4fc8
Parents: cb5a543 3e97167
Author: Josh Elser <el...@apache.org>
Authored: Tue Jul 30 20:03:26 2013 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Tue Jul 30 20:03:26 2013 -0400
----------------------------------------------------------------------
.../apache/accumulo/trace/instrument/impl/MilliSpan.java | 10 +++++-----
.../accumulo/trace/instrument/impl/RootMilliSpan.java | 2 +-
2 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e1e2a57/trace/src/main/java/org/apache/accumulo/trace/instrument/impl/MilliSpan.java
----------------------------------------------------------------------
diff --cc trace/src/main/java/org/apache/accumulo/trace/instrument/impl/MilliSpan.java
index 8f020f0,0000000..b641a2c
mode 100644,000000..100644
--- a/trace/src/main/java/org/apache/accumulo/trace/instrument/impl/MilliSpan.java
+++ b/trace/src/main/java/org/apache/accumulo/trace/instrument/impl/MilliSpan.java
@@@ -1,141 -1,0 +1,141 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.trace.instrument.impl;
+
+import java.security.SecureRandom;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.accumulo.trace.instrument.Span;
+import org.apache.accumulo.trace.instrument.Tracer;
+
+
+/**
+ * A Span implementation that stores its information in milliseconds since the epoch.
+ */
+public class MilliSpan implements Span {
+
+ private static final Random next = new SecureRandom();
+ private long start;
+ private long stop;
+ final private Span parent;
+ final private String description;
+ final private long spanId;
++ final private long traceId;
+ private Map<String,String> traceInfo = null;
+
+ public Span child(String description) {
- return new MilliSpan(description, next.nextLong(), this);
++ return new MilliSpan(description, next.nextLong(), traceId, this);
+ }
+
- public MilliSpan(String description, long id, Span parent) {
++ public MilliSpan(String description, long id, long traceId, Span parent) {
+ this.description = description;
+ this.spanId = id;
++ this.traceId = traceId;
+ this.parent = parent;
+ this.start = 0;
+ this.stop = 0;
+ }
+
+ public synchronized void start() {
+ if (start > 0)
+ throw new IllegalStateException("Span for " + description + " has already been started");
+ start = System.currentTimeMillis();
+ }
+
+ public synchronized void stop() {
+ if (start == 0)
+ throw new IllegalStateException("Span for " + description + " has not been started");
+ stop = System.currentTimeMillis();
+ Tracer.getInstance().pop(this);
+ }
+
+ protected long currentTimeMillis() {
+ return System.currentTimeMillis();
+ }
+
+ public synchronized boolean running() {
+ return start != 0 && stop == 0;
+ }
+
+ public synchronized long accumulatedMillis() {
+ if (start == 0)
+ return 0;
+ if (stop > 0)
+ return stop - start;
+ return currentTimeMillis() - start;
+ }
+
+ public String toString() {
+ long parentId = parentId();
+ return ("\"" + description() + "\" trace:" + Long.toHexString(traceId()) + " span:" + spanId + (parentId > 0 ? " parent:" + parentId : "") + " start:"
+ + start + " ms: " + Long.toString(accumulatedMillis()) + (running() ? "..." : ""));
+
+ }
+
+ public String description() {
+ return description;
+ }
+
+ @Override
+ public long spanId() {
+ return spanId;
+ }
+
+ @Override
+ public Span parent() {
+ return parent;
+ }
+
+ @Override
+ public long parentId() {
+ if (parent == null)
+ return -1;
+ return parent.spanId();
+ }
+
+ @Override
+ public long traceId() {
- if (parent == this)
- throw new RuntimeException("loop found in trace!");
- return parent.traceId();
++ return traceId;
+ }
+
+ @Override
+ public long getStartTimeMillis() {
+ return start;
+ }
+
+ @Override
+ public long getStopTimeMillis() {
+ return stop;
+ }
+
+ @Override
+ public void data(String key, String value) {
+ if (traceInfo == null)
+ traceInfo = new HashMap<String,String>();
+ traceInfo.put(key, value);
+ }
+
+ @Override
+ public Map<String,String> getData() {
+ if (traceInfo == null)
+ return Collections.emptyMap();
+ return Collections.unmodifiableMap(traceInfo);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e1e2a57/trace/src/main/java/org/apache/accumulo/trace/instrument/impl/RootMilliSpan.java
----------------------------------------------------------------------
diff --cc trace/src/main/java/org/apache/accumulo/trace/instrument/impl/RootMilliSpan.java
index eb31dcc,0000000..c25e644
mode 100644,000000..100644
--- a/trace/src/main/java/org/apache/accumulo/trace/instrument/impl/RootMilliSpan.java
+++ b/trace/src/main/java/org/apache/accumulo/trace/instrument/impl/RootMilliSpan.java
@@@ -1,43 -1,0 +1,43 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.trace.instrument.impl;
+
+/**
+ * Span that roots the span tree in a process, but perhaps not the whole trace.
+ *
+ */
+public class RootMilliSpan extends MilliSpan {
+
+ final long traceId;
+ final long parentId;
+
+ @Override
+ public long traceId() {
+ return traceId;
+ }
+
+ public RootMilliSpan(String description, long traceId, long spanId, long parentId) {
- super(description, spanId, null);
++ super(description, spanId, traceId, null);
+ this.traceId = traceId;
+ this.parentId = parentId;
+ }
+
+ public long parentId() {
+ return parentId;
+ }
+
+}
[10/50] [abbrv] git commit: ACCUMULO-1380 set svn eol property
Posted by el...@apache.org.
ACCUMULO-1380 set svn eol property
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.4@1482364 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/452bffc4
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/452bffc4
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/452bffc4
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 452bffc4ea61553a334803e16e542e0560e0b990
Parents: f04bdfe
Author: Keith Turner <kt...@apache.org>
Authored: Tue May 14 14:52:15 2013 +0000
Committer: Keith Turner <kt...@apache.org>
Committed: Tue May 14 14:52:15 2013 +0000
----------------------------------------------------------------------
.../chapters/development_clients.tex | 124 ++++++++++++++++++-
1 file changed, 123 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/452bffc4/docs/src/user_manual/chapters/development_clients.tex
----------------------------------------------------------------------
diff --git a/docs/src/user_manual/chapters/development_clients.tex b/docs/src/user_manual/chapters/development_clients.tex
index 3edb88d..7482a23 100644
--- a/docs/src/user_manual/chapters/development_clients.tex
+++ b/docs/src/user_manual/chapters/development_clients.tex
@@ -1 +1,123 @@
-
% Licensed to the Apache Software Foundation (ASF) under one or more
% contributor license agreements. See the NOTICE file distributed with
% this work for additional information regarding copyright ownership.
% The ASF licenses this file to You under the Apache License, Version 2.0
% (the "License"); you may not use this file except in compliance with
% the License. You may obtain a copy of the License at
%
% http://www.apache.org/licenses/LICENSE-2.0
%
% Unless required by applicable law or agreed to in writing, software
% distributed under the License is distributed on an "AS IS" BASIS,
% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
% See the License for the specific language governing permissions and
% limitations under the License.
\chapter{Development Clients}
Normally, Accumulo consists of lots of moving parts. Even a stand-alone version of
Accumulo requires Hadoop, Zookeeper, the Accumulo master, a tablet server, etc. If
you want to writ
e a unit test that uses Accumulo, you need a lot of infrastructure
in place before your test can run.
\section{Mock Accumulo}
Mock Accumulo supplies mock implementations for much of the client API. It presently
does not enforce users, logins, permissions, etc. It does support Iterators and Combiners.
Note that MockAccumulo holds all data in memory, and will not retain any data or
settings between runs.
While normal interaction with the Accumulo client looks like this:
\small
\begin{verbatim}
Instance instance = new ZooKeeperInstance(...);
Connector conn = instance.getConnector(user, passwd);
\end{verbatim}
\normalsize
To interact with the MockAccumulo, just replace the ZooKeeperInstance with MockInstance:
\small
\begin{verbatim}
Instance instance = new MockInstance();
\end{verbatim}
\normalsize
In fact, you can use the "--fake" option to the Accumulo shell and interact with
MockAccumulo:
\small
\begin{verbatim}
$ ./bin/accumulo shell --fake -u root -p nonsense
Shell - Apach
e Accumulo Interactive Shell
-
- version: 1.4.4
- instance name: mock-instance
- instance id: mock-instance-id
-
- type 'help' for a list of available commands
-
root@mock-instance> createtable test
root@mock-instance test> insert row1 cf cq value
root@mock-instance test> insert row2 cf cq value2
root@mock-instance test> insert row3 cf cq value3
root@mock-instance test> scan
row1 cf:cq [] value
row2 cf:cq [] value2
row3 cf:cq [] value3
root@mock-instance test> scan -b row2 -e row2
row2 cf:cq [] value2
root@mock-instance test>
\end{verbatim}
\normalsize
When testing Map Reduce jobs, you can also set the Mock Accumulo on the AccumuloInputFormat
and AccumuloOutputFormat classes:
\small
\begin{verbatim}
// ... set up job configuration
AccumuloInputFormat.setMockInstance(job, "mockInstance");
AccumuloOutputFormat.setMockInstance(job, "mockInstance");
\end{verbatim}
\normalsize
\section{Mini Accumulo Cluster}
While the Mock Accumulo provides a lightweight implementation of
the client API for unit
testing, it is often necessary to write more realistic end-to-end integration tests that
take advantage of the entire ecosystem. The Mini Accumulo Cluster makes this possible by
configuring and starting Zookeeper, initializing Accumulo, and starting the Master as well
as some Tablet Servers. It runs against the local filesystem instead of having to start
up HDFS.
To start it up, you will need to supply an empty directory and a root password as arguments:
\small
\begin{verbatim}
File tempDirectory = // JUnit and Guava supply mechanisms for creating temp directories
MiniAccumuloCluster accumulo = new MiniAccumuloCluster(tempDirectory, "password");
accumulo.start();
\end{verbatim}
\normalsize
Once we have our mini cluster running, we will want to interact with the Accumulo client API:
\small
\begin{verbatim}
Instance instance = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers());
Connector conn = instance.getConnector("root", "passwo
rd");
\end{verbatim}
\normalsize
Upon completion of our development code, we will want to shutdown our MiniAccumuloCluster:
\small
\begin{verbatim}
accumulo.stop()
// delete your temporary folder
\end{verbatim}
\normalsize
\ No newline at end of file
+
+% Licensed to the Apache Software Foundation (ASF) under one or more
+% contributor license agreements. See the NOTICE file distributed with
+% this work for additional information regarding copyright ownership.
+% The ASF licenses this file to You under the Apache License, Version 2.0
+% (the "License"); you may not use this file except in compliance with
+% the License. You may obtain a copy of the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS,
+% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+% See the License for the specific language governing permissions and
+% limitations under the License.
+
+\chapter{Development Clients}
+
+Normally, Accumulo consists of lots of moving parts. Even a stand-alone version of
+Accumulo requires Hadoop, Zookeeper, the Accumulo master, a tablet server, etc. If
+you want to write a unit test that uses Accumulo, you need a lot of infrastructure
+in place before your test can run.
+
+\section{Mock Accumulo}
+
+Mock Accumulo supplies mock implementations for much of the client API. It presently
+does not enforce users, logins, permissions, etc. It does support Iterators and Combiners.
+Note that MockAccumulo holds all data in memory, and will not retain any data or
+settings between runs.
+
+While normal interaction with the Accumulo client looks like this:
+
+\small
+\begin{verbatim}
+Instance instance = new ZooKeeperInstance(...);
+Connector conn = instance.getConnector(user, passwd);
+\end{verbatim}
+\normalsize
+
+To interact with the MockAccumulo, just replace the ZooKeeperInstance with MockInstance:
+
+\small
+\begin{verbatim}
+Instance instance = new MockInstance();
+\end{verbatim}
+\normalsize
+
+In fact, you can use the "--fake" option to the Accumulo shell and interact with
+MockAccumulo:
+
+\small
+\begin{verbatim}
+$ ./bin/accumulo shell --fake -u root -p nonsense
+
+Shell - Apache Accumulo Interactive Shell
+-
+- version: 1.4.4
+- instance name: mock-instance
+- instance id: mock-instance-id
+-
+- type 'help' for a list of available commands
+-
+root@mock-instance> createtable test
+root@mock-instance test> insert row1 cf cq value
+root@mock-instance test> insert row2 cf cq value2
+root@mock-instance test> insert row3 cf cq value3
+root@mock-instance test> scan
+row1 cf:cq [] value
+row2 cf:cq [] value2
+row3 cf:cq [] value3
+root@mock-instance test> scan -b row2 -e row2
+row2 cf:cq [] value2
+root@mock-instance test>
+\end{verbatim}
+\normalsize
+
+When testing Map Reduce jobs, you can also set the Mock Accumulo on the AccumuloInputFormat
+and AccumuloOutputFormat classes:
+
+\small
+\begin{verbatim}
+// ... set up job configuration
+AccumuloInputFormat.setMockInstance(job, "mockInstance");
+AccumuloOutputFormat.setMockInstance(job, "mockInstance");
+\end{verbatim}
+\normalsize
+
+\section{Mini Accumulo Cluster}
+
+While the Mock Accumulo provides a lightweight implementation of the client API for unit
+testing, it is often necessary to write more realistic end-to-end integration tests that
+take advantage of the entire ecosystem. The Mini Accumulo Cluster makes this possible by
+configuring and starting Zookeeper, initializing Accumulo, and starting the Master as well
+as some Tablet Servers. It runs against the local filesystem instead of having to start
+up HDFS.
+
+To start it up, you will need to supply an empty directory and a root password as arguments:
+
+\small
+\begin{verbatim}
+File tempDirectory = // JUnit and Guava supply mechanisms for creating temp directories
+MiniAccumuloCluster accumulo = new MiniAccumuloCluster(tempDirectory, "password");
+accumulo.start();
+\end{verbatim}
+\normalsize
+
+Once we have our mini cluster running, we will want to interact with the Accumulo client API:
+
+\small
+\begin{verbatim}
+Instance instance = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers());
+Connector conn = instance.getConnector("root", "password");
+\end{verbatim}
+\normalsize
+
+Upon completion of our development code, we will want to shutdown our MiniAccumuloCluster:
+
+\small
+\begin{verbatim}
+accumulo.stop()
+// delete your temporary folder
+\end{verbatim}
+\normalsize
[13/50] [abbrv] git commit: ACCUMULO-1394 backport to 1.4 branch
Posted by el...@apache.org.
ACCUMULO-1394 backport to 1.4 branch
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.4@1483484 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/55a97ceb
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/55a97ceb
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/55a97ceb
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 55a97cebfc6e0034494ffef130866a1bd91fec11
Parents: 3d886c2
Author: Christopher Tubbs <ct...@apache.org>
Authored: Thu May 16 18:02:18 2013 +0000
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Thu May 16 18:02:18 2013 +0000
----------------------------------------------------------------------
.../src/main/java/org/apache/accumulo/core/iterators/Filter.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/55a97ceb/src/core/src/main/java/org/apache/accumulo/core/iterators/Filter.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/iterators/Filter.java b/src/core/src/main/java/org/apache/accumulo/core/iterators/Filter.java
index a9ed76c..8eaf788 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/iterators/Filter.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/iterators/Filter.java
@@ -50,7 +50,7 @@ public abstract class Filter extends WrappingIterator implements OptionDescriber
return newInstance;
}
- private static final String NEGATE = "negate";
+ protected static final String NEGATE = "negate";
boolean negate = false;
@Override
[15/50] [abbrv] git commit: ACCUMULO-1438 Moving MiniAccumuloCluster
into its own package
Posted by el...@apache.org.
ACCUMULO-1438 Moving MiniAccumuloCluster into its own package
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.4@1484632 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/4f1ffc8c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/4f1ffc8c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/4f1ffc8c
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 4f1ffc8c55410333b6fc031f75bcfc02a4f54e67
Parents: a3d5dc6
Author: Corey J. Nolet <cj...@apache.org>
Authored: Tue May 21 01:13:58 2013 +0000
Committer: Corey J. Nolet <cj...@apache.org>
Committed: Tue May 21 01:13:58 2013 +0000
----------------------------------------------------------------------
pom.xml | 1 +
src/proxy/pom.xml | 2 +-
.../java/org/apache/accumulo/proxy/Proxy.java | 15 +-
.../org/apache/accumulo/proxy/SimpleTest.java | 8 +-
.../server/mini/MiniAccumuloCluster.java | 364 -------------------
.../server/mini/MiniAccumuloConfig.java | 89 -----
.../server/mini/MiniAccumuloClusterTest.java | 150 --------
7 files changed, 13 insertions(+), 616 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f1ffc8c/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 5dcc0c2..c6ab32a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -37,6 +37,7 @@
<module>src/start</module>
<module>src/examples</module>
<module>src/proxy</module>
+ <module>src/mini</module>
</modules>
<build>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f1ffc8c/src/proxy/pom.xml
----------------------------------------------------------------------
diff --git a/src/proxy/pom.xml b/src/proxy/pom.xml
index 5cfde7b..c03d022 100644
--- a/src/proxy/pom.xml
+++ b/src/proxy/pom.xml
@@ -115,7 +115,7 @@
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-server</artifactId>
+ <artifactId>accumulo-mini</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f1ffc8c/src/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java
----------------------------------------------------------------------
diff --git a/src/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java b/src/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java
index 7b106dd..ae788af 100644
--- a/src/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java
+++ b/src/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java
@@ -16,14 +16,9 @@
*/
package org.apache.accumulo.proxy;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.lang.reflect.Constructor;
-import java.util.Properties;
-
+import com.google.common.io.Files;
+import org.apache.accumulo.mini.MiniAccumuloCluster;
import org.apache.accumulo.proxy.thrift.AccumuloProxy;
-import org.apache.accumulo.server.mini.MiniAccumuloCluster;
import org.apache.log4j.Logger;
import org.apache.thrift.TProcessor;
import org.apache.thrift.protocol.TCompactProtocol;
@@ -33,7 +28,11 @@ import org.apache.thrift.server.TServer;
import org.apache.thrift.transport.TFramedTransport;
import org.apache.thrift.transport.TNonblockingServerSocket;
-import com.google.common.io.Files;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.util.Properties;
public class Proxy {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f1ffc8c/src/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
----------------------------------------------------------------------
diff --git a/src/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java b/src/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
index 8a898f4..ce49bfb 100644
--- a/src/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
+++ b/src/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
@@ -49,6 +49,7 @@ import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.iterators.user.SummingCombiner;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.examples.simple.constraints.NumericValueConstraint;
+import org.apache.accumulo.mini.MiniAccumuloCluster;
import org.apache.accumulo.proxy.thrift.AccumuloException;
import org.apache.accumulo.proxy.thrift.AccumuloProxy.Client;
import org.apache.accumulo.proxy.thrift.AccumuloSecurityException;
@@ -74,7 +75,6 @@ import org.apache.accumulo.proxy.thrift.TimeType;
import org.apache.accumulo.proxy.thrift.UnknownScanner;
import org.apache.accumulo.proxy.thrift.UnknownWriter;
import org.apache.accumulo.proxy.thrift.WriterOptions;
-import org.apache.accumulo.server.mini.MiniAccumuloCluster;
import org.apache.accumulo.server.test.functional.SlowIterator;
import org.apache.accumulo.server.util.PortUtils;
import org.apache.hadoop.conf.Configuration;
@@ -133,7 +133,7 @@ public class SimpleTest {
Properties props = new Properties();
props.put("instance", accumulo.getInstanceName());
props.put("zookeepers", accumulo.getZooKeepers());
-
+
protocolClass = getRandomProtocol();
System.out.println(protocolClass.getName());
@@ -325,7 +325,7 @@ public class SimpleTest {
client.hasNext(scanner);
fail("exception not thrown");
} catch (UnknownScanner us) {}
-
+
try {
client.closeScanner(scanner);
fail("exception not thrown");
@@ -365,7 +365,7 @@ public class SimpleTest {
// this is a oneway call, so it does not throw exceptions
client.update(writer, mutation("row2", "cf", "cq", "value2"));
-
+
try {
client.flush(writer);
fail("exception not thrown");
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f1ffc8c/src/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloCluster.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloCluster.java b/src/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloCluster.java
deleted file mode 100644
index 2cb3ae2..0000000
--- a/src/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloCluster.java
+++ /dev/null
@@ -1,364 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.server.mini;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Properties;
-import java.util.TimerTask;
-
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.util.UtilWaitThread;
-import org.apache.accumulo.server.logger.LogService;
-import org.apache.accumulo.server.master.Master;
-import org.apache.accumulo.server.tabletserver.TabletServer;
-import org.apache.accumulo.server.util.Initialize;
-import org.apache.accumulo.server.util.PortUtils;
-import org.apache.accumulo.server.util.time.SimpleTimer;
-import org.apache.accumulo.start.Main;
-import org.apache.zookeeper.server.ZooKeeperServerMain;
-
-/**
- * A utility class that will create Zookeeper and Accumulo processes that write all of their data to a single local directory. This class makes it easy to test
- * code against a real Accumulo instance. Its much more accurate for testing than MockAccumulo, but much slower than MockAccumulo.
- *
- * @since 1.5.0, 1.4.4
- */
-public class MiniAccumuloCluster {
-
- private static final String INSTANCE_SECRET = "DONTTELL";
- private static final String INSTANCE_NAME = "miniInstance";
-
- private static class LogWriter extends Thread {
- private BufferedReader in;
- private BufferedWriter out;
-
- /**
- * @throws java.io.IOException
- */
- public LogWriter(InputStream stream, File logFile) throws IOException {
- this.setDaemon(true);
- this.in = new BufferedReader(new InputStreamReader(stream));
- out = new BufferedWriter(new FileWriter(logFile));
-
- SimpleTimer.getInstance().schedule(new TimerTask() {
- @Override
- public void run() {
- try {
- flush();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- }, 1000, 1000);
- }
-
- public synchronized void flush() throws IOException {
- if (out != null)
- out.flush();
- }
-
- @Override
- public void run() {
- String line;
-
- try {
- while ((line = in.readLine()) != null) {
- out.append(line);
- out.append("\n");
- }
-
- synchronized (this) {
- out.close();
- out = null;
- in.close();
- }
-
- } catch (IOException e) {
- }
- }
- }
-
- private File libDir;
- private File confDir;
- private File zooKeeperDir;
- private File accumuloDir;
- private File zooCfgFile;
- private File logDir;
- private File walogDir;
-
- private Process zooKeeperProcess;
- private Process masterProcess;
- private Process loggerProcess;
-
- private int zooKeeperPort;
-
- private List<LogWriter> logWriters = new ArrayList<LogWriter>();
-
- private MiniAccumuloConfig config;
- private Process[] tabletServerProcesses;
-
- Process exec(Class<? extends Object> clazz, String... args) throws IOException {
- String javaHome = System.getProperty("java.home");
- String javaBin = javaHome + File.separator + "bin" + File.separator + "java";
- String classpath = System.getProperty("java.class.path");
-
- classpath = confDir.getAbsolutePath() + File.pathSeparator + classpath;
-
- String className = clazz.getCanonicalName();
-
- ArrayList<String> argList = new ArrayList<String>();
-
- argList.addAll(Arrays.asList(javaBin, "-cp", classpath, "-Xmx128m", "-XX:+UseConcMarkSweepGC", "-XX:CMSInitiatingOccupancyFraction=75",
- Main.class.getName(), className));
-
- argList.addAll(Arrays.asList(args));
-
- ProcessBuilder builder = new ProcessBuilder(argList);
-
- builder.environment().put("ACCUMULO_HOME", config.getDir().getAbsolutePath());
- builder.environment().put("ACCUMULO_LOG_DIR", logDir.getAbsolutePath());
-
- // if we're running under accumulo.start, we forward these env vars
- String env = System.getenv("HADOOP_PREFIX");
- if (env != null)
- builder.environment().put("HADOOP_PREFIX", env);
- env = System.getenv("ZOOKEEPER_HOME");
- if (env != null)
- builder.environment().put("ZOOKEEPER_HOME", env);
-
- Process process = builder.start();
-
- LogWriter lw;
- lw = new LogWriter(process.getErrorStream(), new File(logDir, clazz.getSimpleName() + "_" + process.hashCode() + ".err"));
- logWriters.add(lw);
- lw.start();
- lw = new LogWriter(process.getInputStream(), new File(logDir, clazz.getSimpleName() + "_" + process.hashCode() + ".out"));
- logWriters.add(lw);
- lw.start();
-
- return process;
- }
-
- private void appendProp(FileWriter fileWriter, Property key, String value, Map<String,String> siteConfig) throws IOException {
- appendProp(fileWriter, key.getKey(), value, siteConfig);
- }
-
- private void appendProp(FileWriter fileWriter, String key, String value, Map<String,String> siteConfig) throws IOException {
- if (!siteConfig.containsKey(key))
- fileWriter.append("<property><name>" + key + "</name><value>" + value + "</value></property>\n");
- }
-
- /**
- * @param dir
- * An empty or nonexistant temp directoy that Accumulo and Zookeeper can store data in. Creating the directory is left to the user. Java 7, Guava,
- * and Junit provide methods for creating temporary directories.
- * @param rootPassword
- * Initial root password for instance.
- * @throws IOException
- */
- public MiniAccumuloCluster(File dir, String rootPassword) throws IOException {
- this(new MiniAccumuloConfig(dir, rootPassword));
- }
-
- /**
- * @param config
- * initial configuration
- * @throws IOException
- */
-
- public MiniAccumuloCluster(MiniAccumuloConfig config) throws IOException {
-
- if (config.getDir().exists() && !config.getDir().isDirectory())
- throw new IllegalArgumentException("Must pass in directory, " + config.getDir() + " is a file");
-
- if (config.getDir().exists() && config.getDir().list().length != 0)
- throw new IllegalArgumentException("Directory " + config.getDir() + " is not empty");
-
- this.config = config;
-
- libDir = new File(config.getDir(), "lib");
- confDir = new File(config.getDir(), "conf");
- accumuloDir = new File(config.getDir(), "accumulo");
- zooKeeperDir = new File(config.getDir(), "zookeeper");
- logDir = new File(config.getDir(), "logs");
- walogDir = new File(config.getDir(), "walogs");
-
- confDir.mkdirs();
- accumuloDir.mkdirs();
- zooKeeperDir.mkdirs();
- logDir.mkdirs();
- walogDir.mkdirs();
- libDir.mkdirs();
-
- zooKeeperPort = PortUtils.getRandomFreePort();
-
- File siteFile = new File(confDir, "accumulo-site.xml");
-
- FileWriter fileWriter = new FileWriter(siteFile);
- fileWriter.append("<configuration>\n");
-
- HashMap<String,String> siteConfig = new HashMap<String,String>(config.getSiteConfig());
-
- appendProp(fileWriter, Property.INSTANCE_DFS_URI, "file:///", siteConfig);
- appendProp(fileWriter, Property.INSTANCE_DFS_DIR, accumuloDir.getAbsolutePath(), siteConfig);
- appendProp(fileWriter, Property.INSTANCE_ZK_HOST, "localhost:" + zooKeeperPort, siteConfig);
- appendProp(fileWriter, Property.INSTANCE_SECRET, INSTANCE_SECRET, siteConfig);
- appendProp(fileWriter, Property.MASTER_CLIENTPORT, "" + PortUtils.getRandomFreePort(), siteConfig);
- appendProp(fileWriter, Property.TSERV_CLIENTPORT, "" + PortUtils.getRandomFreePort(), siteConfig);
- appendProp(fileWriter, Property.TSERV_PORTSEARCH, "true", siteConfig);
- appendProp(fileWriter, Property.LOGGER_DIR, walogDir.getAbsolutePath(), siteConfig);
- appendProp(fileWriter, Property.TSERV_DATACACHE_SIZE, "10M", siteConfig);
- appendProp(fileWriter, Property.TSERV_INDEXCACHE_SIZE, "10M", siteConfig);
- appendProp(fileWriter, Property.TSERV_MAXMEM, "50M", siteConfig);
- appendProp(fileWriter, Property.TSERV_WALOG_MAX_SIZE, "100M", siteConfig);
- appendProp(fileWriter, Property.TSERV_NATIVEMAP_ENABLED, "false", siteConfig);
- appendProp(fileWriter, Property.TRACE_PORT, "" + PortUtils.getRandomFreePort(), siteConfig);
- appendProp(fileWriter, Property.LOGGER_SORT_BUFFER_SIZE, "50M", siteConfig);
- appendProp(fileWriter, Property.LOGGER_PORTSEARCH, "true", siteConfig);
-
- // since there is a small amount of memory, check more frequently for majc... setting may not be needed in 1.5
- appendProp(fileWriter, Property.TSERV_MAJC_DELAY, "3", siteConfig);
- String cp = System.getenv("ACCUMULO_HOME") + "/lib/.*.jar," + "$ZOOKEEPER_HOME/zookeeper[^.].*.jar," + "$HADOOP_HOME/[^.].*.jar,"
- + "$HADOOP_HOME/lib/[^.].*.jar," + "$HADOOP_PREFIX/share/hadoop/common/.*.jar," + "$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,"
- + "$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar," + "$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar";
- appendProp(fileWriter, Property.GENERAL_CLASSPATHS, cp, siteConfig);
- appendProp(fileWriter, Property.GENERAL_DYNAMIC_CLASSPATHS, libDir.getAbsolutePath(), siteConfig);
-
- for (Entry<String,String> entry : siteConfig.entrySet())
- fileWriter.append("<property><name>" + entry.getKey() + "</name><value>" + entry.getValue() + "</value></property>\n");
- fileWriter.append("</configuration>\n");
- fileWriter.close();
-
- zooCfgFile = new File(confDir, "zoo.cfg");
- fileWriter = new FileWriter(zooCfgFile);
-
- // zookeeper uses Properties to read its config, so use that to write in order to properly escape things like Windows paths
- Properties zooCfg = new Properties();
- zooCfg.setProperty("tickTime", "1000");
- zooCfg.setProperty("initLimit", "10");
- zooCfg.setProperty("syncLimit", "5");
- zooCfg.setProperty("clientPort", zooKeeperPort + "");
- zooCfg.setProperty("maxClientCnxns", "100");
- zooCfg.setProperty("dataDir", zooKeeperDir.getAbsolutePath());
- zooCfg.store(fileWriter, null);
-
- fileWriter.close();
- }
-
- /**
- * Starts Accumulo and Zookeeper processes. Can only be called once.
- *
- * @throws IOException
- * @throws InterruptedException
- * @throws IllegalStateException
- * if already started
- */
- public void start() throws IOException, InterruptedException {
- if (zooKeeperProcess != null)
- throw new IllegalStateException("Already started");
-
- Runtime.getRuntime().addShutdownHook(new Thread() {
- @Override
- public void run() {
- try {
- MiniAccumuloCluster.this.stop();
- } catch (IOException e) {
- e.printStackTrace();
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- });
-
- zooKeeperProcess = exec(Main.class, ZooKeeperServerMain.class.getName(), zooCfgFile.getAbsolutePath());
-
- // sleep a little bit to let zookeeper come up before calling init, seems to work better
- UtilWaitThread.sleep(250);
-
- // TODO initialization could probably be done in process
- Process initProcess = exec(Initialize.class);
- initProcess.getOutputStream().write((INSTANCE_NAME + "\n").getBytes());
- initProcess.getOutputStream().write((config.getRootPassword() + "\n").getBytes());
- initProcess.getOutputStream().write((config.getRootPassword() + "\n").getBytes());
- initProcess.getOutputStream().flush();
- int ret = initProcess.waitFor();
- if (ret != 0) {
- throw new RuntimeException("Initialize process returned " + ret);
- }
-
- tabletServerProcesses = new Process[config.getNumTservers()];
- for (int i = 0; i < config.getNumTservers(); i++) {
- tabletServerProcesses[i] = exec(TabletServer.class);
- }
-
- loggerProcess = exec(LogService.class);
- masterProcess = exec(Master.class);
- }
-
- /**
- * @return Accumulo instance name
- */
-
- public String getInstanceName() {
- return INSTANCE_NAME;
- }
-
- /**
- * @return zookeeper connection string
- */
-
- public String getZooKeepers() {
- return "localhost:" + zooKeeperPort;
- }
-
- /**
- * Stops Accumulo and Zookeeper processes. If stop is not called, there is a shutdown hook that is setup to kill the processes. Howerver its probably best to
- * call stop in a finally block as soon as possible.
- *
- * @throws IOException
- * @throws InterruptedException
- */
-
- public void stop() throws IOException, InterruptedException {
- if (zooKeeperProcess != null)
- zooKeeperProcess.destroy();
- if (loggerProcess != null)
- loggerProcess.destroy();
- if (masterProcess != null)
- masterProcess.destroy();
- if (tabletServerProcesses != null) {
- for (Process tserver : tabletServerProcesses) {
- tserver.destroy();
- }
- }
-
- for (LogWriter lw : logWriters)
- lw.flush();
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f1ffc8c/src/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloConfig.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloConfig.java b/src/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloConfig.java
deleted file mode 100644
index a82f277..0000000
--- a/src/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloConfig.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.server.mini;
-
-import java.io.File;
-import java.util.Collections;
-import java.util.Map;
-
-/**
- * Holds configuration for {@link MiniAccumuloCluster}. Required configurations must be passed to constructor and all other configurations are optional.
- *
- * @since 1.5.0, 1.4.4
- */
-
-public class MiniAccumuloConfig {
-
- private File dir = null;
- private String rootPassword = null;
- private Map<String,String> siteConfig = Collections.emptyMap();
- private int numTservers = 2;
-
- /**
- * @param dir
- * An empty or nonexistant temp directoy that Accumulo and Zookeeper can store data in. Creating the directory is left to the user. Java 7, Guava,
- * and Junit provide methods for creating temporary directories.
- * @param rootPassword
- * The initial password for the Accumulo root user
- */
-
- public MiniAccumuloConfig(File dir, String rootPassword) {
- this.dir = dir;
- this.rootPassword = rootPassword;
- }
-
- public File getDir() {
- return dir;
- }
-
- public String getRootPassword() {
- return rootPassword;
- }
-
- public int getNumTservers() {
- return numTservers;
- }
-
- /**
- * Calling this method is optional. If not set, it defaults to two.
- *
- * @param numTservers
- * the number of tablet servers that mini accumulo cluster should start
- */
-
- public MiniAccumuloConfig setNumTservers(int numTservers) {
- if (numTservers < 1)
- throw new IllegalArgumentException("Must have at least one tablet server");
- this.numTservers = numTservers;
- return this;
- }
-
- public Map<String,String> getSiteConfig() {
- return siteConfig;
- }
-
- /**
- * Calling this method is optional. If not set, it defautls to an empty map.
- *
- * @param siteConfig
- * key/values that you normally put in accumulo-site.xml can be put here
- */
- public MiniAccumuloConfig setSiteConfig(Map<String,String> siteConfig) {
- this.siteConfig = siteConfig;
- return this;
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f1ffc8c/src/server/src/test/java/org/apache/accumulo/server/mini/MiniAccumuloClusterTest.java
----------------------------------------------------------------------
diff --git a/src/server/src/test/java/org/apache/accumulo/server/mini/MiniAccumuloClusterTest.java b/src/server/src/test/java/org/apache/accumulo/server/mini/MiniAccumuloClusterTest.java
deleted file mode 100644
index 083d4b8..0000000
--- a/src/server/src/test/java/org/apache/accumulo/server/mini/MiniAccumuloClusterTest.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.server.mini;
-
-import java.util.Collections;
-import java.util.Map.Entry;
-import java.util.UUID;
-
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.IteratorSetting;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.client.ZooKeeperInstance;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.iterators.LongCombiner;
-import org.apache.accumulo.core.iterators.user.SummingCombiner;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.security.ColumnVisibility;
-import org.apache.accumulo.core.security.TablePermission;
-import org.apache.accumulo.core.util.UtilWaitThread;
-import org.apache.accumulo.server.mini.MiniAccumuloCluster;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-public class MiniAccumuloClusterTest {
-
- private static final Logger logger = Logger.getLogger(MiniAccumuloClusterTest.class);
-
- private static TemporaryFolder tmpDir = new TemporaryFolder();
- private static MiniAccumuloCluster accumulo;
-
- @BeforeClass
- public static void setupMiniCluster() throws Exception {
-
- tmpDir.create();
- logger.info("MiniCluster started @ " + tmpDir.getRoot());
- Logger.getLogger("org.apache.zookeeper").setLevel(Level.ERROR);
-
- accumulo = new MiniAccumuloCluster(tmpDir.getRoot(), "superSecret");
- accumulo.start();
- }
-
- @Test(timeout = 30000)
- public void test() throws Exception {
- Connector conn = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers()).getConnector("root", "superSecret".getBytes());
-
- conn.tableOperations().create("table1");
-
- conn.securityOperations().createUser("user1", "pass1".getBytes(), new Authorizations("A", "B"));
- conn.securityOperations().grantTablePermission("user1", "table1", TablePermission.WRITE);
- conn.securityOperations().grantTablePermission("user1", "table1", TablePermission.READ);
-
- IteratorSetting is = new IteratorSetting(10, SummingCombiner.class);
- SummingCombiner.setEncodingType(is, LongCombiner.Type.STRING);
- SummingCombiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("META", "COUNT")));
-
- conn.tableOperations().attachIterator("table1", is);
-
- Connector uconn = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers()).getConnector("user1", "pass1".getBytes());
-
- BatchWriter bw = uconn.createBatchWriter("table1", 10000, 1000000, 2);
-
- UUID uuid = UUID.randomUUID();
-
- Mutation m = new Mutation(uuid.toString());
- m.put("META", "SIZE", new ColumnVisibility("A|B"), "8");
- m.put("META", "CRC", new ColumnVisibility("A|B"), "456");
- m.put("META", "COUNT", new ColumnVisibility("A|B"), "1");
- m.put("DATA", "IMG", new ColumnVisibility("A&B"), "ABCDEFGH");
-
- bw.addMutation(m);
- bw.flush();
-
- m = new Mutation(uuid.toString());
- m.put("META", "COUNT", new ColumnVisibility("A|B"), "1");
- m.put("META", "CRC", new ColumnVisibility("A|B"), "123");
- bw.addMutation(m);
-
- bw.close();
-
- int count = 0;
- Scanner scanner = uconn.createScanner("table1", new Authorizations("A"));
- for (Entry<Key,Value> entry : scanner) {
- if (entry.getKey().getColumnQualifierData().toString().equals("COUNT")) {
- Assert.assertEquals("2", entry.getValue().toString());
- } else if (entry.getKey().getColumnQualifierData().toString().equals("SIZE")) {
- Assert.assertEquals("8", entry.getValue().toString());
- } else if (entry.getKey().getColumnQualifierData().toString().equals("CRC")) {
- Assert.assertEquals("123", entry.getValue().toString());
- } else {
- Assert.assertTrue(false);
- }
- count++;
- }
-
- Assert.assertEquals(3, count);
-
- count = 0;
- scanner = uconn.createScanner("table1", new Authorizations("A", "B"));
- for (Entry<Key,Value> entry : scanner) {
- if (entry.getKey().getColumnQualifierData().toString().equals("IMG")) {
- Assert.assertEquals("ABCDEFGH", entry.getValue().toString());
- }
- count++;
- }
-
- Assert.assertEquals(4, count);
-
- conn.tableOperations().delete("table1");
- }
-
- @Test(timeout = 20000)
- public void testMultipleTabletServersRunning() throws AccumuloException, AccumuloSecurityException {
-
- Connector conn = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers()).getConnector("root", "superSecret".getBytes());
-
- while (conn.instanceOperations().getTabletServers().size() != 2) {
- UtilWaitThread.sleep(500);
- }
- }
-
- @AfterClass
- public static void tearDownMiniCluster() throws Exception {
- accumulo.stop();
- tmpDir.delete();
- }
-}
[22/50] [abbrv] git commit: ACCUMULO-847 merge back to 1.4 branch
Posted by el...@apache.org.
ACCUMULO-847 merge back to 1.4 branch
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.4@1485724 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/730e330e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/730e330e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/730e330e
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 730e330e3d5305a2e831186e4876fe90a79ae72c
Parents: 49201bd
Author: Eric C. Newton <ec...@apache.org>
Authored: Thu May 23 14:19:14 2013 +0000
Committer: Eric C. Newton <ec...@apache.org>
Committed: Thu May 23 14:19:14 2013 +0000
----------------------------------------------------------------------
bin/config.sh | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/730e330e/bin/config.sh
----------------------------------------------------------------------
diff --git a/bin/config.sh b/bin/config.sh
index fc9db14..6a88e61 100755
--- a/bin/config.sh
+++ b/bin/config.sh
@@ -109,3 +109,6 @@ if [ ! -f "$ACCUMULO_HOME/conf/tracers" ]; then
echo "$MASTER1" > "$ACCUMULO_HOME/conf/tracers"
fi
SSH='ssh -qnf -o ConnectTimeout=2'
+
+# See HADOOP-7154 and ACCUMULO-847
+export MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-1}
[09/50] [abbrv] git commit: Applying patch ACCUMULO-1380
Posted by el...@apache.org.
Applying patch ACCUMULO-1380
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.4@1481697 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f04bdfef
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f04bdfef
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f04bdfef
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: f04bdfef070fa7c7febe24a0f07a490bc21e8e4a
Parents: 0e463d4
Author: Corey J. Nolet <cj...@apache.org>
Authored: Mon May 13 04:27:23 2013 +0000
Committer: Corey J. Nolet <cj...@apache.org>
Committed: Mon May 13 04:27:23 2013 +0000
----------------------------------------------------------------------
docs/mockAccumulo.html | 80 --------------------
docs/src/user_manual/accumulo_user_manual.tex | 1 +
.../chapters/development_clients.tex | 1 +
3 files changed, 2 insertions(+), 80 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f04bdfef/docs/mockAccumulo.html
----------------------------------------------------------------------
diff --git a/docs/mockAccumulo.html b/docs/mockAccumulo.html
deleted file mode 100644
index 54c6449..0000000
--- a/docs/mockAccumulo.html
+++ /dev/null
@@ -1,80 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<html>
-<head>
-<title>Accumulo Mock</title>
-<link rel='stylesheet' type='text/css' href='documentation.css' media='screen'/>
-</head>
-<body>
-
-<h1>Apache Accumulo Documentation : Mock Accumulo</h1>
-
-<p>Normally, accumulo consists of lots of moving parts. Even a stand-alone version of accumulo requires hadoop,
-zookeeper, the accumulo master, a tablet server, etc. If you want to write a unit test that uses accumulo, you need a lot of
-infrastructure in place before your test can run. To help developers with their tests, we have created a fake version of
-Accumulo called MockAccumulo. The normal interaction with the client API looks something like this:
-</p>
-
-<pre>
- Instance instance = new ZooKeeperInstance(...);
- Connector conn = instance.getConnector(user, passwd);
- BatchScanner scanner = conn.createBatchScaner(...);
- // configure scanner
- // ...
- for (Entry<Key, Value> entry : scanner) {
- // process entry
- }
-</pre>
-
-<p>MockAccumulo supports the same client API. Just replace the ZooKeeperInstance with MockInstance:</p>
-
-<pre>
- Instance instance = new MockInstance();
- // as above
-</pre>
-
-<p>In fact, you can use the <tt>--fake</tt> option to the accumulo shell and interact with MockAccumulo:</p>
-
-<pre>
-$ ./bin/accumulo shell --fake -u root -p nonsense
-
-Shell - Apache Accumulo Interactive Shell
--
-- version: 1.3.2-SNAPSHOT
-- instance name: mock-instance
-- instance id: mock-instance-id
--
-- type 'help' for a list of available commands
--
-root@mock-instance> createtable test
-root@mock-instance test> insert row1 cf cq value
-root@mock-instance test> insert row2 cf cq value2
-root@mock-instance test> insert row3 cf cq value3
-root@mock-instance test> scan
-row1 cf:cq [] value
-row2 cf:cq [] value2
-row3 cf:cq [] value3
-root@mock-instance test> scan -b row2 -e row2
-row2 cf:cq [] value2
-root@mock-instance test>
-</pre>
-
-<p>MockAccumulo presently does not enforce users, logins, permissions, etc. It <b>does</b> support Iterators and Aggregators.
-Note that MockAccumulo holds all data in memory, and will not retain any data or settings between runs.</p>
-
-</body>
-</html>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f04bdfef/docs/src/user_manual/accumulo_user_manual.tex
----------------------------------------------------------------------
diff --git a/docs/src/user_manual/accumulo_user_manual.tex b/docs/src/user_manual/accumulo_user_manual.tex
index e20ce1d..9c95348 100644
--- a/docs/src/user_manual/accumulo_user_manual.tex
+++ b/docs/src/user_manual/accumulo_user_manual.tex
@@ -43,6 +43,7 @@ Version 1.4}
\include{chapters/design}
\include{chapters/shell}
\include{chapters/clients}
+\include{chapters/development_clients}
\include{chapters/table_configuration}
\include{chapters/table_design}
\include{chapters/high_speed_ingest}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f04bdfef/docs/src/user_manual/chapters/development_clients.tex
----------------------------------------------------------------------
diff --git a/docs/src/user_manual/chapters/development_clients.tex b/docs/src/user_manual/chapters/development_clients.tex
new file mode 100644
index 0000000..3edb88d
--- /dev/null
+++ b/docs/src/user_manual/chapters/development_clients.tex
@@ -0,0 +1 @@
+
% Licensed to the Apache Software Foundation (ASF) under one or more
% contributor license agreements. See the NOTICE file distributed with
% this work for additional information regarding copyright ownership.
% The ASF licenses this file to You under the Apache License, Version 2.0
% (the "License"); you may not use this file except in compliance with
% the License. You may obtain a copy of the License at
%
% http://www.apache.org/licenses/LICENSE-2.0
%
% Unless required by applicable law or agreed to in writing, software
% distributed under the License is distributed on an "AS IS" BASIS,
% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
% See the License for the specific language governing permissions and
% limitations under the License.
\chapter{Development Clients}
Normally, Accumulo consists of lots of moving parts. Even a stand-alone version of
Accumulo requires Hadoop, Zookeeper, the Accumulo master, a tablet server, etc. If
you want to writ
e a unit test that uses Accumulo, you need a lot of infrastructure
in place before your test can run.
\section{Mock Accumulo}
Mock Accumulo supplies mock implementations for much of the client API. It presently
does not enforce users, logins, permissions, etc. It does support Iterators and Combiners.
Note that MockAccumulo holds all data in memory, and will not retain any data or
settings between runs.
While normal interaction with the Accumulo client looks like this:
\small
\begin{verbatim}
Instance instance = new ZooKeeperInstance(...);
Connector conn = instance.getConnector(user, passwd);
\end{verbatim}
\normalsize
To interact with the MockAccumulo, just replace the ZooKeeperInstance with MockInstance:
\small
\begin{verbatim}
Instance instance = new MockInstance();
\end{verbatim}
\normalsize
In fact, you can use the "--fake" option to the Accumulo shell and interact with
MockAccumulo:
\small
\begin{verbatim}
$ ./bin/accumulo shell --fake -u root -p nonsense
Shell - Apach
e Accumulo Interactive Shell
-
- version: 1.4.4
- instance name: mock-instance
- instance id: mock-instance-id
-
- type 'help' for a list of available commands
-
root@mock-instance> createtable test
root@mock-instance test> insert row1 cf cq value
root@mock-instance test> insert row2 cf cq value2
root@mock-instance test> insert row3 cf cq value3
root@mock-instance test> scan
row1 cf:cq [] value
row2 cf:cq [] value2
row3 cf:cq [] value3
root@mock-instance test> scan -b row2 -e row2
row2 cf:cq [] value2
root@mock-instance test>
\end{verbatim}
\normalsize
When testing Map Reduce jobs, you can also set the Mock Accumulo on the AccumuloInputFormat
and AccumuloOutputFormat classes:
\small
\begin{verbatim}
// ... set up job configuration
AccumuloInputFormat.setMockInstance(job, "mockInstance");
AccumuloOutputFormat.setMockInstance(job, "mockInstance");
\end{verbatim}
\normalsize
\section{Mini Accumulo Cluster}
While the Mock Accumulo provides a lightweight implementation of
the client API for unit
testing, it is often necessary to write more realistic end-to-end integration tests that
take advantage of the entire ecosystem. The Mini Accumulo Cluster makes this possible by
configuring and starting Zookeeper, initializing Accumulo, and starting the Master as well
as some Tablet Servers. It runs against the local filesystem instead of having to start
up HDFS.
To start it up, you will need to supply an empty directory and a root password as arguments:
\small
\begin{verbatim}
File tempDirectory = // JUnit and Guava supply mechanisms for creating temp directories
MiniAccumuloCluster accumulo = new MiniAccumuloCluster(tempDirectory, "password");
accumulo.start();
\end{verbatim}
\normalsize
Once we have our mini cluster running, we will want to interact with the Accumulo client API:
\small
\begin{verbatim}
Instance instance = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers());
Connector conn = instance.getConnector("root", "passwo
rd");
\end{verbatim}
\normalsize
Upon completion of our development code, we will want to shutdown our MiniAccumuloCluster:
\small
\begin{verbatim}
accumulo.stop()
// delete your temporary folder
\end{verbatim}
\normalsize
\ No newline at end of file
[26/50] [abbrv] git commit: ACCUMULO-1505 - Applying Ryan Leary's
patch
Posted by el...@apache.org.
ACCUMULO-1505 - Applying Ryan Leary's patch
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.4@1491900 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/225d7286
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/225d7286
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/225d7286
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 225d7286d2d5f382c7caef83ea7e6f22c044291a
Parents: 8fd1c3b
Author: John Vines <vi...@apache.org>
Authored: Tue Jun 11 18:50:30 2013 +0000
Committer: John Vines <vi...@apache.org>
Committed: Tue Jun 11 18:50:30 2013 +0000
----------------------------------------------------------------------
.../core/client/mock/MockBatchWriter.java | 3 ++
.../accumulo/core/client/mock/MockTable.java | 2 +
.../core/client/mock/MockConnectorTest.java | 39 ++++++++++++++++++++
3 files changed, 44 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/225d7286/src/core/src/main/java/org/apache/accumulo/core/client/mock/MockBatchWriter.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/mock/MockBatchWriter.java b/src/core/src/main/java/org/apache/accumulo/core/client/mock/MockBatchWriter.java
index b33ebcb..d89a263 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/client/mock/MockBatchWriter.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/client/mock/MockBatchWriter.java
@@ -19,6 +19,7 @@ package org.apache.accumulo.core.client.mock;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.util.ArgumentChecker;
public class MockBatchWriter implements BatchWriter {
@@ -32,11 +33,13 @@ public class MockBatchWriter implements BatchWriter {
@Override
public void addMutation(Mutation m) throws MutationsRejectedException {
+ ArgumentChecker.notNull(m);
acu.addMutation(tablename, m);
}
@Override
public void addMutations(Iterable<Mutation> iterable) throws MutationsRejectedException {
+ ArgumentChecker.notNull(iterable);
for (Mutation m : iterable) {
acu.addMutation(tablename, m);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/225d7286/src/core/src/main/java/org/apache/accumulo/core/client/mock/MockTable.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/mock/MockTable.java b/src/core/src/main/java/org/apache/accumulo/core/client/mock/MockTable.java
index 0722ba1..999c9b2 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/client/mock/MockTable.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/client/mock/MockTable.java
@@ -97,6 +97,8 @@ public class MockTable {
}
synchronized void addMutation(Mutation m) {
+ if (m.size() == 0)
+ throw new IllegalArgumentException("Can not add empty mutations");
long now = System.currentTimeMillis();
mutationCount++;
for (ColumnUpdate u : m.getUpdates()) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/225d7286/src/core/src/test/java/org/apache/accumulo/core/client/mock/MockConnectorTest.java
----------------------------------------------------------------------
diff --git a/src/core/src/test/java/org/apache/accumulo/core/client/mock/MockConnectorTest.java b/src/core/src/test/java/org/apache/accumulo/core/client/mock/MockConnectorTest.java
index ab1ffb5..6fa9dff 100644
--- a/src/core/src/test/java/org/apache/accumulo/core/client/mock/MockConnectorTest.java
+++ b/src/core/src/test/java/org/apache/accumulo/core/client/mock/MockConnectorTest.java
@@ -20,8 +20,10 @@ import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertFalse;
import static junit.framework.Assert.assertTrue;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
+import java.util.List;
import java.util.Map.Entry;
import java.util.Random;
@@ -92,6 +94,43 @@ public class MockConnectorTest {
}
@Test
+ public void testBadMutations() throws Exception {
+ Connector c = new MockConnector("root", new MockInstance());
+ c.tableOperations().create("test");
+ BatchWriter bw = c.createBatchWriter("test", 10000L, 1000L, 4);
+
+ try {
+ bw.addMutation(null);
+ Assert.fail("addMutation should throw IAE for null mutation");
+ } catch (IllegalArgumentException iae) {}
+ try {
+ bw.addMutations(null);
+ Assert.fail("addMutations should throw IAE for null iterable");
+ } catch (IllegalArgumentException iae) {}
+
+ bw.addMutations(Collections.EMPTY_LIST);
+
+ Mutation bad = new Mutation("bad");
+ try {
+ bw.addMutation(bad);
+ Assert.fail("addMutation should throw IAE for empty mutation");
+ } catch (IllegalArgumentException iae) {}
+
+
+ Mutation good = new Mutation("good");
+ good.put(asText(random.nextInt()), asText(random.nextInt()), new Value("good".getBytes()));
+ List<Mutation> mutations = new ArrayList<Mutation>();
+ mutations.add(good);
+ mutations.add(bad);
+ try {
+ bw.addMutations(mutations);
+ Assert.fail("addMutations should throw IAE if it contains empty mutation");
+ } catch (IllegalArgumentException iae) {}
+
+ bw.close();
+ }
+
+ @Test
public void testAggregation() throws Exception {
MockInstance mockInstance = new MockInstance();
Connector c = mockInstance.getConnector("root", new byte[] {});
[19/50] [abbrv] git commit: Reversing ACCUMULO-1438.
Posted by el...@apache.org.
Reversing ACCUMULO-1438.
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.4@1484854 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e46360cf
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e46360cf
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e46360cf
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: e46360cf272dc1c53ca3a8c009ec899f0134546f
Parents: 7bc5686
Author: Corey J. Nolet <cj...@apache.org>
Authored: Tue May 21 16:25:59 2013 +0000
Committer: Corey J. Nolet <cj...@apache.org>
Committed: Tue May 21 16:25:59 2013 +0000
----------------------------------------------------------------------
docs/config.html | 957 +------------------
pom.xml | 1 -
src/mini/pom.xml | 1 -
.../accumulo/mini/MiniAccumuloCluster.java | 364 -------
.../accumulo/mini/MiniAccumuloConfig.java | 89 --
.../accumulo/mini/MiniAccumuloClusterTest.java | 1 -
src/mini/src/test/resources/log4j.properties | 1 -
src/proxy/pom.xml | 2 +-
.../java/org/apache/accumulo/proxy/Proxy.java | 15 +-
.../org/apache/accumulo/proxy/SimpleTest.java | 8 +-
.../server/mini/MiniAccumuloCluster.java | 364 +++++++
.../server/mini/MiniAccumuloConfig.java | 89 ++
.../server/mini/MiniAccumuloClusterTest.java | 150 +++
13 files changed, 617 insertions(+), 1425 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e46360cf/docs/config.html
----------------------------------------------------------------------
diff --git a/docs/config.html b/docs/config.html
index 70f4855..a36b5ca 100644
--- a/docs/config.html
+++ b/docs/config.html
@@ -1,956 +1 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<html>
- <head>
- <title>Accumulo Configuration</title>
- <link rel='stylesheet' type='text/css' href='documentation.css' media='screen'/>
- </head>
- <body>
- <h1>Apache Accumulo Configuration Management</h1>
- <p>All accumulo properties have a default value in the source code. Properties can also be set
- in accumulo-site.xml and in zookeeper on per-table or system-wide basis. If properties are set in more than one location,
- accumulo will choose the property with the highest precedence. This order of precedence is described
- below (from highest to lowest):</p>
- <table>
- <tr><th>Location</th><th>Description</th></tr>
- <tr class='highlight'><td><b>Zookeeper<br/>table properties</td>
- <td>Table properties are applied to the entire cluster when set in zookeeper using the accumulo API or shell. While table properties take precedent over system properties, both will override properties set in accumulo-site.xml<br/><br/>
- Table properties consist of all properties with the table.* prefix. Table properties are configured on a per-table basis using the following shell commmand:
- <pre>config -t TABLE -s PROPERTY=VALUE</pre></td>
- </tr>
- <tr><td><b>Zookeeper<br/>system properties</td>
- <td>System properties are applied to the entire cluster when set in zookeeper using the accumulo API or shell. System properties consist of all properties with a 'yes' in the 'Zookeeper Mutable' column in the table below. They are set with the following shell command:
- <pre>config -s PROPERTY=VALUE</pre>
- If a table.* property is set using this method, the value will apply to all tables except those configured on per-table basis (which have higher precedence).<br/><br/>
- While most system properties take effect immediately, some require a restart of the process which is indicated in 'Zookeeper Mutable'.</td>
- </tr>
- <tr class='highlight'><td><b>accumulo-site.xml</td>
- <td>Accumulo processes (master, tserver, etc) read their local accumulo-site.xml on start up. Therefore, changes made to accumulo-site.xml must rsynced across the cluster and processes must be restarted to apply changes.<br/><br/>
- Certain properties (indicated by a 'no' in 'Zookeeper Mutable') cannot be set in zookeeper and only set in this file. The accumulo-site.xml also allows you to configure tablet servers with different settings.</td>
- </tr>
- <tr><td><b>Default</td>
- <td>All properties have a default value in the source code. This value has the lowest precedence and is overriden if set in accumulo-site.xml or zookeeper.<br/><br/>While the default value is usually optimal, there are cases where a change can increase query and ingest performance.</td>
- </tr>
- </table>
-
- <p>The 'config' command in the shell allows you to view the current system configuration. You can also use the '-t' option to view a table's configuration as below:
-
- <pre>
- $ ./bin/accumulo shell -u root
- Enter current password for 'root'@'ac14': ******
-
- Shell - Apache Accumulo Interactive Shell
- -
- - version: 1.4.4-SNAPSHOT
- - instance name: ac14
- - instance id: 4f48fa03-f692-43ce-ae03-94c9ea8b7181
- -
- - type 'help' for a list of available commands
- -
- root@ac13> config -t foo
- ---------+---------------------------------------------+------------------------------------------------------
- SCOPE | NAME | VALUE
- ---------+---------------------------------------------+------------------------------------------------------
- default | table.balancer ............................ | org.apache.accumulo.server.master.balancer.DefaultLoadBalancer
- default | table.bloom.enabled ....................... | false
- default | table.bloom.error.rate .................... | 0.5%
- default | table.bloom.hash.type ..................... | murmur
- default | table.bloom.key.functor ................... | org.apache.accumulo.core.file.keyfunctor.RowFunctor
- default | table.bloom.load.threshold ................ | 1
- default | table.bloom.size .......................... | 1048576
- default | table.cache.block.enable .................. | false
- default | table.cache.index.enable .................. | false
- default | table.compaction.major.everything.at ...... | 19700101000000GMT
- default | table.compaction.major.everything.idle .... | 1h
- default | table.compaction.major.ratio .............. | 1.3
- site | @override .............................. | 1.4
- system | @override .............................. | 1.5
- table | @override .............................. | 1.6
- default | table.compaction.minor.idle ............... | 5m
- default | table.compaction.minor.logs.threshold ..... | 3
- default | table.failures.ignore ..................... | false
- </pre>
-
- <h1>Configuration Properties</h1>
-
-
- <p>Jump to:
-<a href='#INSTANCE_PREFIX'>instance.*</a> | <a href='#GENERAL_PREFIX'>general.*</a> | <a href='#MASTER_PREFIX'>master.*</a> | <a href='#TSERV_PREFIX'>tserver.*</a> | <a href='#LOGGER_PREFIX'>logger.*</a> | <a href='#GC_PREFIX'>gc.*</a> | <a href='#MONITOR_PREFIX'>monitor.*</a> | <a href='#TRACE_PREFIX'>trace.*</a> | <a href='#TABLE_PREFIX'>table.*</a> | <a href='#TABLE_CONSTRAINT_PREFIX'>table.constraint.*</a> | <a href='#TABLE_ITERATOR_PREFIX'>table.iterator.*</a> | <a href='#TABLE_LOCALITY_GROUP_PREFIX'>table.group.*</a> </p>
- <table>
- <tr><td colspan='5'><a name='INSTANCE_PREFIX' class='large'>instance.*</a></td></tr>
- <tr><td colspan='5'><i>Properties in this category must be consistent throughout a cloud. This is enforced and servers won't be able to communicate if these differ.</i></td></tr>
- <tr><th>Property</th><th>Type</th><th>Zookeeper Mutable</th><th>Default Value</th><th>Description</th></tr>
- <tr class='highlight'>
- <td>instance.dfs.dir</td>
- <td><b><a href='#ABSOLUTEPATH'>absolute path</a></b></td>
- <td>no</td>
- <td><pre>/accumulo</pre></td>
- <td>HDFS directory in which accumulo instance will run. Do not change after accumulo is initialized.</td>
- </tr>
- <tr >
- <td>instance.dfs.uri</td>
- <td><b><a href='#URI'>uri</a></b></td>
- <td>no</td>
- <td><pre> </pre></td>
- <td>The url accumulo should use to connect to DFS. If this is empty, accumulo will obtain this information from the hadoop configuration.</td>
- </tr>
- <tr class='highlight'>
- <td>instance.secret</td>
- <td><b><a href='#STRING'>string</a></b></td>
- <td>no</td>
- <td><pre>DEFAULT</pre></td>
- <td>A secret unique to a given instance that all servers must know in order to communicate with one another. Change it before initialization. To change it later use ./bin/accumulo accumulo.server.util.ChangeSecret [oldpasswd] [newpasswd], and then update conf/accumulo-site.xml everywhere.</td>
- </tr>
- <tr >
- <td>instance.zookeeper.host</td>
- <td><b><a href='#HOSTLIST'>host list</a></b></td>
- <td>no</td>
- <td><pre>localhost:2181</pre></td>
- <td>Comma separated list of zookeeper servers</td>
- </tr>
- <tr class='highlight'>
- <td>instance.zookeeper.timeout</td>
- <td><b><a href='#TIMEDURATION'>duration</a></b></td>
- <td>no</td>
- <td><pre>30s</pre></td>
- <td>Zookeeper session timeout; max value when represented as milliseconds should be no larger than 2147483647</td>
- </tr>
- <tr><td colspan='5'><a name='GENERAL_PREFIX' class='large'>general.*</a></td></tr>
- <tr><td colspan='5'><i>Properties in this category affect the behavior of accumulo overall, but do not have to be consistent throughout a cloud.</i></td></tr>
- <tr><th>Property</th><th>Type</th><th>Zookeeper Mutable</th><th>Default Value</th><th>Description</th></tr>
- <tr class='highlight'>
- <td>general.classpaths</td>
- <td><b><a href='#STRING'>string</a></b></td>
- <td>no</td>
- <td><pre>$ACCUMULO_HOME/conf,
-$ACCUMULO_HOME/lib/[^.].$ACCUMULO_VERSION.jar,
-$ACCUMULO_HOME/lib/[^.].*.jar,
-$ZOOKEEPER_HOME/zookeeper[^.].*.jar,
-$HADOOP_HOME/[^.].*.jar,
-$HADOOP_HOME/conf,
-$HADOOP_HOME/lib/[^.].*.jar,
-</pre></td>
- <td>A list of all of the places to look for a class. Order does matter, as it will look for the jar starting in the first location to the last. Please note, hadoop conf and hadoop lib directories NEED to be here, along with accumulo lib and zookeeper directory. Supports full regex on filename alone.</td>
- </tr>
- <tr >
- <td>general.dynamic.classpaths</td>
- <td><b><a href='#STRING'>string</a></b></td>
- <td>no</td>
- <td><pre>$ACCUMULO_HOME/lib/ext/[^.].*.jar
-</pre></td>
- <td>A list of all of the places where changes in jars or classes will force a reload of the classloader.</td>
- </tr>
- <tr class='highlight'>
- <td>general.kerberos.keytab</td>
- <td><b><a href='#PATH'>path</a></b></td>
- <td>no</td>
- <td><pre> </pre></td>
- <td>Path to the kerberos keytab to use. Leave blank if not using kerberoized hdfs</td>
- </tr>
- <tr >
- <td>general.kerberos.principal</td>
- <td><b><a href='#STRING'>string</a></b></td>
- <td>no</td>
- <td><pre> </pre></td>
- <td>Name of the kerberos principal to use. _HOST will automatically be replaced by the machines hostname in the hostname portion of the principal. Leave blank if not using kerberoized hdfs</td>
- </tr>
- <tr class='highlight'>
- <td>general.rpc.timeout</td>
- <td><b><a href='#TIMEDURATION'>duration</a></b></td>
- <td>no</td>
- <td><pre>120s</pre></td>
- <td>Time to wait on I/O for simple, short RPC calls</td>
- </tr>
- <tr><td colspan='5'><a name='MASTER_PREFIX' class='large'>master.*</a></td></tr>
- <tr><td colspan='5'><i>Properties in this category affect the behavior of the master server</i></td></tr>
- <tr><th>Property</th><th>Type</th><th>Zookeeper Mutable</th><th>Default Value</th><th>Description</th></tr>
- <tr class='highlight'>
- <td>master.bulk.retries</td>
- <td><b><a href='#COUNT'>count</a></b></td>
- <td>yes</td>
- <td><pre>3</pre></td>
- <td>The number of attempts to bulk-load a file before giving up.</td>
- </tr>
- <tr >
- <td>master.bulk.threadpool.size</td>
- <td><b><a href='#COUNT'>count</a></b></td>
- <td>yes</td>
- <td><pre>5</pre></td>
- <td>The number of threads to use when coordinating a bulk-import.</td>
- </tr>
- <tr class='highlight'>
- <td>master.logger.balancer</td>
- <td><b><a href='#CLASSNAME'>java class</a></b></td>
- <td>yes</td>
- <td><pre>org.apache.accumulo.server.master.balancer.SimpleLoggerBalancer</pre></td>
- <td>The balancer class that accumulo will use to make logger assignment decisions.</td>
- </tr>
- <tr >
- <td>master.port.client</td>
- <td><b><a href='#PORT'>port</a></b></td>
- <td>yes but requires restart of the master</td>
- <td><pre>9999</pre></td>
- <td>The port used for handling client connections on the master</td>
- </tr>
- <tr class='highlight'>
- <td>master.recovery.max.age</td>
- <td><b><a href='#TIMEDURATION'>duration</a></b></td>
- <td>yes</td>
- <td><pre>60m</pre></td>
- <td>Recovery files older than this age will be removed.</td>
- </tr>
- <tr >
- <td>master.recovery.pool</td>
- <td><b><a href='#STRING'>string</a></b></td>
- <td>yes</td>
- <td><pre>recovery</pre></td>
- <td>Priority queue to use for log recovery map/reduce jobs.</td>
- </tr>
- <tr class='highlight'>
- <td>master.recovery.queue</td>
- <td><b><a href='#STRING'>string</a></b></td>
- <td>yes</td>
- <td><pre>default</pre></td>
- <td>Priority queue to use for log recovery map/reduce jobs.</td>
- </tr>
- <tr >
- <td>master.recovery.reducers</td>
- <td><b><a href='#COUNT'>count</a></b></td>
- <td>yes</td>
- <td><pre>10</pre></td>
- <td>Number of reducers to use to sort recovery logs (per log)</td>
- </tr>
- <tr class='highlight'>
- <td>master.recovery.sort.mapreduce</td>
- <td><b><a href='#BOOLEAN'>boolean</a></b></td>
- <td>yes</td>
- <td><pre>false</pre></td>
- <td>If true, use map/reduce to sort write-ahead logs during recovery</td>
- </tr>
- <tr >
- <td>master.recovery.time.max</td>
- <td><b><a href='#TIMEDURATION'>duration</a></b></td>
- <td>yes</td>
- <td><pre>30m</pre></td>
- <td>The maximum time to attempt recovery before giving up</td>
- </tr>
- <tr class='highlight'>
- <td>master.server.threadcheck.time</td>
- <td><b><a href='#TIMEDURATION'>duration</a></b></td>
- <td>yes</td>
- <td><pre>1s</pre></td>
- <td>The time between adjustments of the server thread pool.</td>
- </tr>
- <tr >
- <td>master.server.threads.minimum</td>
- <td><b><a href='#COUNT'>count</a></b></td>
- <td>yes</td>
- <td><pre>2</pre></td>
- <td>The minimum number of threads to use to handle incoming requests.</td>
- </tr>
- <tr class='highlight'>
- <td>master.tablet.balancer</td>
- <td><b><a href='#CLASSNAME'>java class</a></b></td>
- <td>yes</td>
- <td><pre>org.apache.accumulo.server.master.balancer.TableLoadBalancer</pre></td>
- <td>The balancer class that accumulo will use to make tablet assignment and migration decisions.</td>
- </tr>
- <tr><td colspan='5'><a name='TSERV_PREFIX' class='large'>tserver.*</a></td></tr>
- <tr><td colspan='5'><i>Properties in this category affect the behavior of the tablet servers</i></td></tr>
- <tr><th>Property</th><th>Type</th><th>Zookeeper Mutable</th><th>Default Value</th><th>Description</th></tr>
- <tr class='highlight'>
- <td>tserver.bloom.load.concurrent.max</td>
- <td><b><a href='#COUNT'>count</a></b></td>
- <td>yes</td>
- <td><pre>4</pre></td>
- <td>The number of concurrent threads that will load bloom filters in the background. Setting this to zero will make bloom filters load in the foreground.</td>
- </tr>
- <tr >
- <td>tserver.bulk.assign.threads</td>
- <td><b><a href='#COUNT'>count</a></b></td>
- <td>yes</td>
- <td><pre>1</pre></td>
- <td>The master delegates bulk file processing and assignment to tablet servers. After the bulk file has been processed, the tablet server will assign the file to the appropriate tablets on all servers. This property controls the number of threads used to communicate to the other servers.</td>
- </tr>
- <tr class='highlight'>
- <td>tserver.bulk.process.threads</td>
- <td><b><a href='#COUNT'>count</a></b></td>
- <td>yes</td>
- <td><pre>1</pre></td>
- <td>The master will task a tablet server with pre-processing a bulk file prior to assigning it to the appropriate tablet servers. This configuration value controls the number of threads used to process the files.</td>
- </tr>
- <tr >
- <td>tserver.bulk.retry.max</td>
- <td><b><a href='#COUNT'>count</a></b></td>
- <td>yes</td>
- <td><pre>3</pre></td>
- <td>The number of times the tablet server will attempt to assign a file to a tablet as it migrates and splits.</td>
- </tr>
- <tr class='highlight'>
- <td>tserver.cache.data.size</td>
- <td><b><a href='#MEMORY'>memory</a></b></td>
- <td>yes</td>
- <td><pre>100M</pre></td>
- <td>Specifies the size of the cache for file data blocks.</td>
- </tr>
- <tr >
- <td>tserver.cache.index.size</td>
- <td><b><a href='#MEMORY'>memory</a></b></td>
- <td>yes</td>
- <td><pre>512M</pre></td>
- <td>Specifies the size of the cache for file indices.</td>
- </tr>
- <tr class='highlight'>
- <td>tserver.client.timeout</td>
- <td><b><a href='#TIMEDURATION'>duration</a></b></td>
- <td>yes</td>
- <td><pre>3s</pre></td>
- <td>Time to wait for clients to continue scans before closing a session.</td>
- </tr>
- <tr >
- <td>tserver.compaction.major.concurrent.max</td>
- <td><b><a href='#COUNT'>count</a></b></td>
- <td>yes but requires restart of the tserver</td>
- <td><pre>3</pre></td>
- <td>The maximum number of concurrent major compactions for a tablet server</td>
- </tr>
- <tr class='highlight'>
- <td>tserver.compaction.major.delay</td>
- <td><b><a href='#TIMEDURATION'>duration</a></b></td>
- <td>yes</td>
- <td><pre>30s</pre></td>
- <td>Time a tablet server will sleep between checking which tablets need compaction.</td>
- </tr>
- <tr >
- <td>tserver.compaction.major.thread.files.open.max</td>
- <td><b><a href='#COUNT'>count</a></b></td>
- <td>yes but requires restart of the tserver</td>
- <td><pre>10</pre></td>
- <td>Max number of files a major compaction thread can open at once. </td>
- </tr>
- <tr class='highlight'>
- <td>tserver.compaction.minor.concurrent.max</td>
- <td><b><a href='#COUNT'>count</a></b></td>
- <td>yes</td>
- <td><pre>4</pre></td>
- <td>The maximum number of concurrent minor compactions for a tablet server</td>
- </tr>
- <tr >
- <td>tserver.default.blocksize</td>
- <td><b><a href='#MEMORY'>memory</a></b></td>
- <td>yes</td>
- <td><pre>1M</pre></td>
- <td>Specifies a default blocksize for the tserver caches</td>
- </tr>
- <tr class='highlight'>
- <td>tserver.dir.memdump</td>
- <td><b><a href='#PATH'>path</a></b></td>
- <td>yes</td>
- <td><pre>/tmp</pre></td>
- <td>A long running scan could possibly hold memory that has been minor compacted. To prevent this, the in memory map is dumped to a local file and the scan is switched to that local file. We can not switch to the minor compacted file because it may have been modified by iterators. The file dumped to the local dir is an exact copy of what was in memory.</td>
- </tr>
- <tr >
- <td>tserver.files.open.idle</td>
- <td><b><a href='#TIMEDURATION'>duration</a></b></td>
- <td>yes</td>
- <td><pre>1m</pre></td>
- <td>Tablet servers leave previously used map files open for future queries. This setting determines how much time an unused map file should be kept open until it is closed.</td>
- </tr>
- <tr class='highlight'>
- <td>tserver.hold.time.max</td>
- <td><b><a href='#TIMEDURATION'>duration</a></b></td>
- <td>yes</td>
- <td><pre>5m</pre></td>
- <td>The maximum time for a tablet server to be in the "memory full" state. If the tablet server cannot write out memory in this much time, it will assume there is some failure local to its node, and quit. A value of zero is equivalent to forever.</td>
- </tr>
- <tr >
- <td>tserver.logger.count</td>
- <td><b><a href='#COUNT'>count</a></b></td>
- <td>yes but requires restart of the tserver</td>
- <td><pre>2</pre></td>
- <td>The number of loggers that each tablet server should use.</td>
- </tr>
- <tr class='highlight'>
- <td>tserver.logger.strategy</td>
- <td><b><a href='#STRING'>string</a></b></td>
- <td>yes</td>
- <td><pre>org.apache.accumulo.server.tabletserver.log.RoundRobinLoggerStrategy</pre></td>
- <td>The classname used to decide which loggers to use.</td>
- </tr>
- <tr >
- <td>tserver.logger.timeout</td>
- <td><b><a href='#TIMEDURATION'>duration</a></b></td>
- <td>yes</td>
- <td><pre>30s</pre></td>
- <td>The time to wait for a logger to respond to a write-ahead request</td>
- </tr>
- <tr class='highlight'>
- <td>tserver.memory.lock</td>
- <td><b><a href='#BOOLEAN'>boolean</a></b></td>
- <td>yes</td>
- <td><pre>false</pre></td>
- <td>The tablet server must communicate with zookeeper frequently to maintain its locks. If the tablet server's memory is swapped out the java garbage collector can stop all processing for long periods. Change this property to true and the tablet server will attempt to lock all of its memory to RAM, which may reduce delays during java garbage collection. You will have to modify the system limit for "max locked memory". This feature is only available when running on Linux. Alternatively you may also want to set /proc/sys/vm/swappiness to zero (again, this is Linux-specific).</td>
- </tr>
- <tr >
- <td>tserver.memory.manager</td>
- <td><b><a href='#CLASSNAME'>java class</a></b></td>
- <td>yes</td>
- <td><pre>org.apache.accumulo.server.tabletserver.LargestFirstMemoryManager</pre></td>
- <td>An implementation of MemoryManger that accumulo will use.</td>
- </tr>
- <tr class='highlight'>
- <td>tserver.memory.maps.max</td>
- <td><b><a href='#MEMORY'>memory</a></b></td>
- <td>yes</td>
- <td><pre>1G</pre></td>
- <td>Maximum amount of memory that can be used to buffer data written to a tablet server. There are two other properties that can effectively limit memory usage table.compaction.minor.logs.threshold and tserver.walog.max.size. Ensure that table.compaction.minor.logs.threshold * tserver.walog.max.size >= this property.</td>
- </tr>
- <tr >
- <td>tserver.memory.maps.native.enabled</td>
- <td><b><a href='#BOOLEAN'>boolean</a></b></td>
- <td>yes but requires restart of the tserver</td>
- <td><pre>true</pre></td>
- <td>An in-memory data store for accumulo implemented in c++ that increases the amount of data accumulo can hold in memory and avoids Java GC pauses.</td>
- </tr>
- <tr class='highlight'>
- <td>tserver.metadata.readahead.concurrent.max</td>
- <td><b><a href='#COUNT'>count</a></b></td>
- <td>yes</td>
- <td><pre>8</pre></td>
- <td>The maximum number of concurrent metadata read ahead that will execute.</td>
- </tr>
- <tr >
- <td>tserver.migrations.concurrent.max</td>
- <td><b><a href='#COUNT'>count</a></b></td>
- <td>yes</td>
- <td><pre>1</pre></td>
- <td>The maximum number of concurrent tablet migrations for a tablet server</td>
- </tr>
- <tr class='highlight'>
- <td>tserver.monitor.fs</td>
- <td><b><a href='#BOOLEAN'>boolean</a></b></td>
- <td>yes</td>
- <td><pre>true</pre></td>
- <td>When enabled the tserver will monitor file systems and kill itself when one switches from rw to ro. This is usually and indication that Linux has detected a bad disk.</td>
- </tr>
- <tr >
- <td>tserver.mutation.queue.max</td>
- <td><b><a href='#MEMORY'>memory</a></b></td>
- <td>yes</td>
- <td><pre>256K</pre></td>
- <td>The amount of memory to use to store write-ahead-log mutations-per-session before flushing them.</td>
- </tr>
- <tr class='highlight'>
- <td>tserver.port.client</td>
- <td><b><a href='#PORT'>port</a></b></td>
- <td>yes but requires restart of the tserver</td>
- <td><pre>9997</pre></td>
- <td>The port used for handling client connections on the tablet servers</td>
- </tr>
- <tr >
- <td>tserver.port.search</td>
- <td><b><a href='#BOOLEAN'>boolean</a></b></td>
- <td>yes</td>
- <td><pre>false</pre></td>
- <td>if the ports above are in use, search higher ports until one is available</td>
- </tr>
- <tr class='highlight'>
- <td>tserver.readahead.concurrent.max</td>
- <td><b><a href='#COUNT'>count</a></b></td>
- <td>yes</td>
- <td><pre>16</pre></td>
- <td>The maximum number of concurrent read ahead that will execute. This effectively limits the number of long running scans that can run concurrently per tserver.</td>
- </tr>
- <tr >
- <td>tserver.scan.files.open.max</td>
- <td><b><a href='#COUNT'>count</a></b></td>
- <td>yes but requires restart of the tserver</td>
- <td><pre>100</pre></td>
- <td>Maximum total map files that all tablets in a tablet server can open for scans. </td>
- </tr>
- <tr class='highlight'>
- <td>tserver.server.threadcheck.time</td>
- <td><b><a href='#TIMEDURATION'>duration</a></b></td>
- <td>yes</td>
- <td><pre>1s</pre></td>
- <td>The time between adjustments of the server thread pool.</td>
- </tr>
- <tr >
- <td>tserver.server.threads.minimum</td>
- <td><b><a href='#COUNT'>count</a></b></td>
- <td>yes</td>
- <td><pre>2</pre></td>
- <td>The minimum number of threads to use to handle incoming requests.</td>
- </tr>
- <tr class='highlight'>
- <td>tserver.session.idle.max</td>
- <td><b><a href='#TIMEDURATION'>duration</a></b></td>
- <td>yes</td>
- <td><pre>1m</pre></td>
- <td>maximum idle time for a session</td>
- </tr>
- <tr >
- <td>tserver.tablet.split.midpoint.files.max</td>
- <td><b><a href='#COUNT'>count</a></b></td>
- <td>yes</td>
- <td><pre>30</pre></td>
- <td>To find a tablets split points, all index files are opened. This setting determines how many index files can be opened at once. When there are more index files than this setting multiple passes must be made, which is slower. However opening too many files at once can cause problems.</td>
- </tr>
- <tr class='highlight'>
- <td>tserver.walog.max.size</td>
- <td><b><a href='#MEMORY'>memory</a></b></td>
- <td>yes</td>
- <td><pre>1G</pre></td>
- <td>The maximum size for each write-ahead log. See comment for property tserver.memory.maps.max</td>
- </tr>
- <tr><td colspan='5'><a name='LOGGER_PREFIX' class='large'>logger.*</a></td></tr>
- <tr><td colspan='5'><i>Properties in this category affect the behavior of the write-ahead logger servers</i></td></tr>
- <tr><th>Property</th><th>Type</th><th>Zookeeper Mutable</th><th>Default Value</th><th>Description</th></tr>
- <tr class='highlight'>
- <td>logger.archive</td>
- <td><b><a href='#BOOLEAN'>boolean</a></b></td>
- <td>yes</td>
- <td><pre>false</pre></td>
- <td>determines if logs are archived in hdfs</td>
- </tr>
- <tr >
- <td>logger.archive.replication</td>
- <td><b><a href='#COUNT'>count</a></b></td>
- <td>yes</td>
- <td><pre>0</pre></td>
- <td>determines the replication factor for walogs archived in hdfs, set to zero to use default</td>
- </tr>
- <tr class='highlight'>
- <td>logger.copy.threadpool.size</td>
- <td><b><a href='#COUNT'>count</a></b></td>
- <td>yes</td>
- <td><pre>2</pre></td>
- <td>size of the thread pool used to copy files from the local log area to HDFS</td>
- </tr>
- <tr >
- <td>logger.dir.walog</td>
- <td><b><a href='#PATH'>path</a></b></td>
- <td>yes</td>
- <td><pre>walogs</pre></td>
- <td>The directory used to store write-ahead logs on the local filesystem. It is possible to specify a comma-separated list of directories.</td>
- </tr>
- <tr class='highlight'>
- <td>logger.monitor.fs</td>
- <td><b><a href='#BOOLEAN'>boolean</a></b></td>
- <td>yes</td>
- <td><pre>true</pre></td>
- <td>When enabled the logger will monitor file systems and kill itself when one switches from rw to ro. This is usually and indication that Linux has detected a bad disk.</td>
- </tr>
- <tr >
- <td>logger.port.client</td>
- <td><b><a href='#PORT'>port</a></b></td>
- <td>yes but requires restart of the logger</td>
- <td><pre>11224</pre></td>
- <td>The port used for write-ahead logger services</td>
- </tr>
- <tr class='highlight'>
- <td>logger.port.search</td>
- <td><b><a href='#BOOLEAN'>boolean</a></b></td>
- <td>yes</td>
- <td><pre>false</pre></td>
- <td>if the port above is in use, search higher ports until one is available</td>
- </tr>
- <tr >
- <td>logger.recovery.file.replication</td>
- <td><b><a href='#COUNT'>count</a></b></td>
- <td>yes</td>
- <td><pre>2</pre></td>
- <td>When a logger puts a WALOG into HDFS, it will use this as the replication factor.</td>
- </tr>
- <tr class='highlight'>
- <td>logger.server.threadcheck.time</td>
- <td><b><a href='#TIMEDURATION'>duration</a></b></td>
- <td>yes</td>
- <td><pre>1s</pre></td>
- <td>The time between adjustments of the server thread pool.</td>
- </tr>
- <tr >
- <td>logger.server.threads.minimum</td>
- <td><b><a href='#COUNT'>count</a></b></td>
- <td>yes</td>
- <td><pre>2</pre></td>
- <td>The miniumum number of threads to use to handle incoming requests.</td>
- </tr>
- <tr class='highlight'>
- <td>logger.sort.buffer.size</td>
- <td><b><a href='#MEMORY'>memory</a></b></td>
- <td>yes</td>
- <td><pre>200M</pre></td>
- <td>The amount of memory to use when sorting logs during recovery. Only used when *not* sorting logs with map/reduce.</td>
- </tr>
- <tr><td colspan='5'><a name='GC_PREFIX' class='large'>gc.*</a></td></tr>
- <tr><td colspan='5'><i>Properties in this category affect the behavior of the accumulo garbage collector.</i></td></tr>
- <tr><th>Property</th><th>Type</th><th>Zookeeper Mutable</th><th>Default Value</th><th>Description</th></tr>
- <tr class='highlight'>
- <td>gc.cycle.delay</td>
- <td><b><a href='#TIMEDURATION'>duration</a></b></td>
- <td>yes</td>
- <td><pre>5m</pre></td>
- <td>Time between garbage collection cycles. In each cycle, old files no longer in use are removed from the filesystem.</td>
- </tr>
- <tr >
- <td>gc.cycle.start</td>
- <td><b><a href='#TIMEDURATION'>duration</a></b></td>
- <td>yes</td>
- <td><pre>30s</pre></td>
- <td>Time to wait before attempting to garbage collect any old files.</td>
- </tr>
- <tr class='highlight'>
- <td>gc.port.client</td>
- <td><b><a href='#PORT'>port</a></b></td>
- <td>yes but requires restart of the gc</td>
- <td><pre>50091</pre></td>
- <td>The listening port for the garbage collector's monitor service</td>
- </tr>
- <tr >
- <td>gc.threads.delete</td>
- <td><b><a href='#COUNT'>count</a></b></td>
- <td>yes</td>
- <td><pre>16</pre></td>
- <td>The number of threads used to delete files</td>
- </tr>
- <tr><td colspan='5'><a name='MONITOR_PREFIX' class='large'>monitor.*</a></td></tr>
- <tr><td colspan='5'><i>Properties in this category affect the behavior of the monitor web server.</i></td></tr>
- <tr><th>Property</th><th>Type</th><th>Zookeeper Mutable</th><th>Default Value</th><th>Description</th></tr>
- <tr class='highlight'>
- <td>monitor.banner.background</td>
- <td><b><a href='#STRING'>string</a></b></td>
- <td>yes</td>
- <td><pre>#304065</pre></td>
- <td>The background color of the banner text displayed on the monitor page.</td>
- </tr>
- <tr >
- <td>monitor.banner.color</td>
- <td><b><a href='#STRING'>string</a></b></td>
- <td>yes</td>
- <td><pre>#c4c4c4</pre></td>
- <td>The color of the banner text displayed on the monitor page.</td>
- </tr>
- <tr class='highlight'>
- <td>monitor.banner.text</td>
- <td><b><a href='#STRING'>string</a></b></td>
- <td>yes</td>
- <td><pre> </pre></td>
- <td>The banner text displayed on the monitor page.</td>
- </tr>
- <tr >
- <td>monitor.port.client</td>
- <td><b><a href='#PORT'>port</a></b></td>
- <td>no</td>
- <td><pre>50095</pre></td>
- <td>The listening port for the monitor's http service</td>
- </tr>
- <tr class='highlight'>
- <td>monitor.port.log4j</td>
- <td><b><a href='#PORT'>port</a></b></td>
- <td>no</td>
- <td><pre>4560</pre></td>
- <td>The listening port for the monitor's log4j logging collection.</td>
- </tr>
- <tr><td colspan='5'><a name='TRACE_PREFIX' class='large'>trace.*</a></td></tr>
- <tr><td colspan='5'><i>Properties in this category affect the behavior of distributed tracing.</i></td></tr>
- <tr><th>Property</th><th>Type</th><th>Zookeeper Mutable</th><th>Default Value</th><th>Description</th></tr>
- <tr class='highlight'>
- <td>trace.password</td>
- <td><b><a href='#STRING'>string</a></b></td>
- <td>no</td>
- <td><pre>secret</pre></td>
- <td>The password for the user used to store distributed traces</td>
- </tr>
- <tr >
- <td>trace.port.client</td>
- <td><b><a href='#PORT'>port</a></b></td>
- <td>no</td>
- <td><pre>12234</pre></td>
- <td>The listening port for the trace server</td>
- </tr>
- <tr class='highlight'>
- <td>trace.table</td>
- <td><b><a href='#STRING'>string</a></b></td>
- <td>no</td>
- <td><pre>trace</pre></td>
- <td>The name of the table to store distributed traces</td>
- </tr>
- <tr >
- <td>trace.user</td>
- <td><b><a href='#STRING'>string</a></b></td>
- <td>no</td>
- <td><pre>root</pre></td>
- <td>The name of the user to store distributed traces</td>
- </tr>
- <tr><td colspan='5'><a name='TABLE_PREFIX' class='large'>table.*</a></td></tr>
- <tr><td colspan='5'><i>Properties in this category affect tablet server treatment of tablets, but can be configured on a per-table basis. Setting these properties in the site file will override the default globally for all tables and not any specific table. However, both the default and the global setting can be overridden per table using the table operations API or in the shell, which sets the overridden value in zookeeper. Restarting accumulo tablet servers after setting these properties in the site file will cause the global setting to take effect. However, you must use the API or the shell to change properties in zookeeper that are set on a table.</i></td></tr>
- <tr><th>Property</th><th>Type</th><th>Zookeeper Mutable</th><th>Default Value</th><th>Description</th></tr>
- <tr class='highlight'>
- <td>table.balancer</td>
- <td><b><a href='#STRING'>string</a></b></td>
- <td>yes</td>
- <td><pre>org.apache.accumulo.server.master.balancer.DefaultLoadBalancer</pre></td>
- <td>This property can be set to allow the LoadBalanceByTable load balancer to change the called Load Balancer for this table</td>
- </tr>
- <tr >
- <td>table.bloom.enabled</td>
- <td><b><a href='#BOOLEAN'>boolean</a></b></td>
- <td>yes</td>
- <td><pre>false</pre></td>
- <td>Use bloom filters on this table.</td>
- </tr>
- <tr class='highlight'>
- <td>table.bloom.error.rate</td>
- <td><b><a href='#FRACTION'>fraction/percentage</a></b></td>
- <td>yes</td>
- <td><pre>0.5%</pre></td>
- <td>Bloom filter error rate.</td>
- </tr>
- <tr >
- <td>table.bloom.hash.type</td>
- <td><b><a href='#STRING'>string</a></b></td>
- <td>yes</td>
- <td><pre>murmur</pre></td>
- <td>The bloom filter hash type</td>
- </tr>
- <tr class='highlight'>
- <td>table.bloom.key.functor</td>
- <td><b><a href='#CLASSNAME'>java class</a></b></td>
- <td>yes</td>
- <td><pre>org.apache.accumulo.core.file.keyfunctor.RowFunctor</pre></td>
- <td>A function that can transform the key prior to insertion and check of bloom filter. org.apache.accumulo.core.file.keyfunctor.RowFunctor,,org.apache.accumulo.core.file.keyfunctor.ColumnFamilyFunctor, and org.apache.accumulo.core.file.keyfunctor.ColumnQualifierFunctor are allowable values. One can extend any of the above mentioned classes to perform specialized parsing of the key. </td>
- </tr>
- <tr >
- <td>table.bloom.load.threshold</td>
- <td><b><a href='#COUNT'>count</a></b></td>
- <td>yes</td>
- <td><pre>1</pre></td>
- <td>This number of seeks that would actually use a bloom filter must occur before a map files bloom filter is loaded. Set this to zero to initiate loading of bloom filters when a map file opened.</td>
- </tr>
- <tr class='highlight'>
- <td>table.bloom.size</td>
- <td><b><a href='#COUNT'>count</a></b></td>
- <td>yes</td>
- <td><pre>1048576</pre></td>
- <td>Bloom filter size, as number of keys.</td>
- </tr>
- <tr >
- <td>table.cache.block.enable</td>
- <td><b><a href='#BOOLEAN'>boolean</a></b></td>
- <td>yes</td>
- <td><pre>false</pre></td>
- <td>Determines whether file block cache is enabled.</td>
- </tr>
- <tr class='highlight'>
- <td>table.cache.index.enable</td>
- <td><b><a href='#BOOLEAN'>boolean</a></b></td>
- <td>yes</td>
- <td><pre>true</pre></td>
- <td>Determines whether index cache is enabled.</td>
- </tr>
- <tr >
- <td>table.compaction.major.everything.idle</td>
- <td><b><a href='#TIMEDURATION'>duration</a></b></td>
- <td>yes</td>
- <td><pre>1h</pre></td>
- <td>After a tablet has been idle (no mutations) for this time period it may have all of its map file compacted into one. There is no guarantee an idle tablet will be compacted. Compactions of idle tablets are only started when regular compactions are not running. Idle compactions only take place for tablets that have one or more map files.</td>
- </tr>
- <tr class='highlight'>
- <td>table.compaction.major.ratio</td>
- <td><b><a href='#FRACTION'>fraction/percentage</a></b></td>
- <td>yes</td>
- <td><pre>3</pre></td>
- <td>minimum ratio of total input size to maximum input file size for running a major compaction. When adjusting this property you may want to also adjust table.file.max. Want to avoid the situation where only merging minor compactions occur.</td>
- </tr>
- <tr >
- <td>table.compaction.minor.idle</td>
- <td><b><a href='#TIMEDURATION'>duration</a></b></td>
- <td>yes</td>
- <td><pre>5m</pre></td>
- <td>After a tablet has been idle (no mutations) for this time period it may have its in-memory map flushed to disk in a minor compaction. There is no guarantee an idle tablet will be compacted.</td>
- </tr>
- <tr class='highlight'>
- <td>table.compaction.minor.logs.threshold</td>
- <td><b><a href='#COUNT'>count</a></b></td>
- <td>yes</td>
- <td><pre>3</pre></td>
- <td>When there are more than this many write-ahead logs against a tablet, it will be minor compacted. See comment for property tserver.memory.maps.max</td>
- </tr>
- <tr >
- <td>table.failures.ignore</td>
- <td><b><a href='#BOOLEAN'>boolean</a></b></td>
- <td>yes</td>
- <td><pre>false</pre></td>
- <td>If you want queries for your table to hang or fail when data is missing from the system, then set this to false. When this set to true missing data will be reported but queries will still run possibly returning a subset of the data.</td>
- </tr>
- <tr class='highlight'>
- <td>table.file.blocksize</td>
- <td><b><a href='#MEMORY'>memory</a></b></td>
- <td>yes</td>
- <td><pre>0B</pre></td>
- <td>Overrides the hadoop dfs.block.size setting so that map files have better query performance. The maximum value for this is 2147483647</td>
- </tr>
- <tr >
- <td>table.file.compress.blocksize</td>
- <td><b><a href='#MEMORY'>memory</a></b></td>
- <td>yes</td>
- <td><pre>100K</pre></td>
- <td>Overrides the hadoop io.seqfile.compress.blocksize setting so that map files have better query performance. The maximum value for this is 2147483647</td>
- </tr>
- <tr class='highlight'>
- <td>table.file.compress.blocksize.index</td>
- <td><b><a href='#MEMORY'>memory</a></b></td>
- <td>yes</td>
- <td><pre>128K</pre></td>
- <td>Determines how large index blocks can be in files that support multilevel indexes. The maximum value for this is 2147483647</td>
- </tr>
- <tr >
- <td>table.file.compress.type</td>
- <td><b><a href='#STRING'>string</a></b></td>
- <td>yes</td>
- <td><pre>gz</pre></td>
- <td>One of gz,lzo,none</td>
- </tr>
- <tr class='highlight'>
- <td>table.file.max</td>
- <td><b><a href='#COUNT'>count</a></b></td>
- <td>yes</td>
- <td><pre>15</pre></td>
- <td>Determines the max # of files each tablet in a table can have. When adjusting this property you may want to consider adjusting table.compaction.major.ratio also. Setting this property to 0 will make it default to tserver.scan.files.open.max-1, this will prevent a tablet from having more files than can be opened. Setting this property low may throttle ingest and increase query performance.</td>
- </tr>
- <tr >
- <td>table.file.replication</td>
- <td><b><a href='#COUNT'>count</a></b></td>
- <td>yes</td>
- <td><pre>0</pre></td>
- <td>Determines how many replicas to keep of a tables map files in HDFS. When this value is LTE 0, HDFS defaults are used.</td>
- </tr>
- <tr class='highlight'>
- <td>table.file.type</td>
- <td><b><a href='#STRING'>string</a></b></td>
- <td>yes</td>
- <td><pre>rf</pre></td>
- <td>Change the type of file a table writes</td>
- </tr>
- <tr >
- <td>table.formatter</td>
- <td><b><a href='#STRING'>string</a></b></td>
- <td>yes</td>
- <td><pre>org.apache.accumulo.core.util.format.DefaultFormatter</pre></td>
- <td>The Formatter class to apply on results in the shell</td>
- </tr>
- <tr class='highlight'>
- <td>table.groups.enabled</td>
- <td><b><a href='#STRING'>string</a></b></td>
- <td>yes</td>
- <td><pre> </pre></td>
- <td>A comma separated list of locality group names to enable for this table.</td>
- </tr>
- <tr >
- <td>table.scan.max.memory</td>
- <td><b><a href='#MEMORY'>memory</a></b></td>
- <td>yes</td>
- <td><pre>1M</pre></td>
- <td>The maximum amount of memory that will be used to cache results of a client query/scan. Once this limit is reached, the buffered data is sent to the client.</td>
- </tr>
- <tr class='highlight'>
- <td>table.security.scan.visibility.default</td>
- <td><b><a href='#STRING'>string</a></b></td>
- <td>yes</td>
- <td><pre> </pre></td>
- <td>The security label that will be assumed at scan time if an entry does not have a visibility set.<br />Note: An empty security label is displayed as []. The scan results will show an empty visibility even if the visibility from this setting is applied to the entry.<br />CAUTION: If a particular key has an empty security label AND its table's default visibility is also empty, access will ALWAYS be granted for users with permission to that table. Additionally, if this field is changed, all existing data with an empty visibility label will be interpreted with the new label on the next scan.</td>
- </tr>
- <tr >
- <td>table.split.threshold</td>
- <td><b><a href='#MEMORY'>memory</a></b></td>
- <td>yes</td>
- <td><pre>1G</pre></td>
- <td>When combined size of files exceeds this amount a tablet is split.</td>
- </tr>
- <tr class='highlight'>
- <td>table.walog.enabled</td>
- <td><b><a href='#BOOLEAN'>boolean</a></b></td>
- <td>yes</td>
- <td><pre>true</pre></td>
- <td>Use the write-ahead log to prevent the loss of data.</td>
- </tr>
- <tr><td colspan='5'><a name='TABLE_CONSTRAINT_PREFIX' class='large'>table.constraint.*</a></td></tr>
- <tr><td colspan='5'><i>Properties in this category are per-table properties that add constraints to a table. These properties start with the category prefix, followed by a number, and their values correspond to a fully qualified Java class that implements the Constraint interface.<br />For example, table.constraint.1 = org.apache.accumulo.core.constraints.MyCustomConstraint and table.constraint.2 = my.package.constraints.MySecondConstraint</i></td></tr>
- <tr><td colspan='5'><a name='TABLE_ITERATOR_PREFIX' class='large'>table.iterator.*</a></td></tr>
- <tr><td colspan='5'><i>Properties in this category specify iterators that are applied at various stages (scopes) of interaction with a table. These properties start with the category prefix, followed by a scope (minc, majc, scan, etc.), followed by a period, followed by a name, as in table.iterator.scan.vers, or table.iterator.scan.custom. The values for these properties are a number indicating the ordering in which it is applied, and a class name such as table.iterator.scan.vers = 10,org.apache.accumulo.core.iterators.VersioningIterator<br /> These iterators can take options if additional properties are set that look like this property, but are suffixed with a period, followed by 'opt' followed by another period, and a property name.<br />For example, table.iterator.minc.vers.opt.maxVersions = 3</i></td></tr>
- <tr><td colspan='5'><a name='TABLE_LOCALITY_GROUP_PREFIX' class='large'>table.group.*</a></td></tr>
- <tr><td colspan='5'><i>Properties in this category are per-table properties that define locality groups in a table. These properties start with the category prefix, followed by a name, followed by a period, and followed by a property for that group.<br />For example table.group.group1=x,y,z sets the column families for a group called group1. Once configured, group1 can be enabled by adding it to the list of groups in the table.groups.enabled property.<br />Additional group options may be specified for a named group by setting table.group.<name>.opt.<key>=<value>.</i></td></tr>
- </table>
- <h1>Property Type Descriptions</h1>
- <table>
- <tr><th>Property Type</th><th>Description</th></tr>
- <tr class='highlight'>
- <td><h3><a name='TIMEDURATION'>duration</a></td>
- <td>A non-negative integer optionally followed by a unit of time (whitespace disallowed), as in 30s.<br />If no unit of time is specified, seconds are assumed. Valid units are 'ms', 's', 'm', 'h' for milliseconds, seconds, minutes, and hours.<br />Examples of valid durations are '600', '30s', '45m', '30000ms', '3d', and '1h'.<br />Examples of invalid durations are '1w', '1h30m', '1s 200ms', 'ms', '', and 'a'.<br />Unless otherwise stated, the max value for the duration represented in milliseconds is 9223372036854775807</td>
- </tr>
- <tr >
- <td><h3><a name='DATETIME'>date/time</a></td>
- <td>A date/time string in the format: YYYYMMDDhhmmssTTT where TTT is the 3 character time zone</td>
- </tr>
- <tr class='highlight'>
- <td><h3><a name='MEMORY'>memory</a></td>
- <td>A positive integer optionally followed by a unit of memory (whitespace disallowed), as in 2G.<br />If no unit is specified, bytes are assumed. Valid units are 'B', 'K', 'M', 'G', for bytes, kilobytes, megabytes, and gigabytes.<br />Examples of valid memories are '1024', '20B', '100K', '1500M', '2G'.<br />Examples of invalid memories are '1M500K', '1M 2K', '1MB', '1.5G', '1,024K', '', and 'a'.<br .>Unless otherwise stated, the max value for the memory represented in bytes is 9223372036854775807</td>
- </tr>
- <tr >
- <td><h3><a name='HOSTLIST'>host list</a></td>
- <td>A comma-separated list of hostnames or ip addresses, with optional port numbers.<br />Examples of valid host lists are 'localhost:2000,www.example.com,10.10.1.1:500' and 'localhost'.<br />Examples of invalid host lists are '', ':1000', and 'localhost:80000'</td>
- </tr>
- <tr class='highlight'>
- <td><h3><a name='PORT'>port</a></td>
- <td>An positive integer in the range 1024-65535, not already in use or specified elsewhere in the configuration</td>
- </tr>
- <tr >
- <td><h3><a name='COUNT'>count</a></td>
- <td>A non-negative integer in the range of 0-2147483647</td>
- </tr>
- <tr class='highlight'>
- <td><h3><a name='FRACTION'>fraction/percentage</a></td>
- <td>A floating point number that represents either a fraction or, if suffixed with the '%' character, a percentage.<br />Examples of valid fractions/percentages are '10', '1000%', '0.05', '5%', '0.2%', '0.0005'.<br />Examples of invalid fractions/percentages are '', '10 percent', 'Hulk Hogan'</td>
- </tr>
- <tr >
- <td><h3><a name='PATH'>path</a></td>
- <td>A string that represents a filesystem path, which can be either relative or absolute to some directory. The filesystem depends on the property.</td>
- </tr>
- <tr class='highlight'>
- <td><h3><a name='ABSOLUTEPATH'>absolute path</a></td>
- <td>An absolute filesystem path. The filesystem depends on the property. This is the same as path, but enforces that its root is explicitly specified.</td>
- </tr>
- <tr >
- <td><h3><a name='CLASSNAME'>java class</a></td>
- <td>A fully qualified java class name representing a class on the classpath.<br />An example is 'java.lang.String', rather than 'String'</td>
- </tr>
- <tr class='highlight'>
- <td><h3><a name='STRING'>string</a></td>
- <td>An arbitrary string of characters whose format is unspecified and interpreted based on the context of the property to which it applies.</td>
- </tr>
- <tr >
- <td><h3><a name='BOOLEAN'>boolean</a></td>
- <td>Has a value of either 'true' or 'false'</td>
- </tr>
- <tr class='highlight'>
- <td><h3><a name='URI'>uri</a></td>
- <td>A valid URI</td>
- </tr>
- </table>
- </body>
-</html>
+Classname org.apache.accumulo.core.conf.DefaultConfiguration not found. Please make sure you use the wholly qualified package name.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e46360cf/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index c6ab32a..5dcc0c2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -37,7 +37,6 @@
<module>src/start</module>
<module>src/examples</module>
<module>src/proxy</module>
- <module>src/mini</module>
</modules>
<build>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e46360cf/src/mini/pom.xml
----------------------------------------------------------------------
diff --git a/src/mini/pom.xml b/src/mini/pom.xml
deleted file mode 100644
index a6177fa..0000000
--- a/src/mini/pom.xml
+++ /dev/null
@@ -1 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 ht
tp://maven.apache.org/maven-v4_0_0.xsd">
<parent>
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo</artifactId>
<version>1.4.4-SNAPSHOT</version>
<relativePath>../../</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>accumulo-mini</artifactId>
<name>accumulo-mini</name>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<configuration>
<archive>
<manifestSections>
<manifestSection>
<name>accumulo/mini/</name>
<manifestEntries>
<Sealed>true</Sealed>
</manifestEntries>
</manifestSection>
</manifestSections>
</archive>
</configuration>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>zookeeper</artifactId>
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-core</artifactId>
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-start</artifactId>
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-server</artifactId>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</dependency>
<dependency>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
<dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
</dependency>
</dependencies>
</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e46360cf/src/mini/src/main/java/org/apache/accumulo/mini/MiniAccumuloCluster.java
----------------------------------------------------------------------
diff --git a/src/mini/src/main/java/org/apache/accumulo/mini/MiniAccumuloCluster.java b/src/mini/src/main/java/org/apache/accumulo/mini/MiniAccumuloCluster.java
deleted file mode 100644
index ed50755..0000000
--- a/src/mini/src/main/java/org/apache/accumulo/mini/MiniAccumuloCluster.java
+++ /dev/null
@@ -1,364 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.mini;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Properties;
-import java.util.TimerTask;
-
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.util.UtilWaitThread;
-import org.apache.accumulo.server.logger.LogService;
-import org.apache.accumulo.server.master.Master;
-import org.apache.accumulo.server.tabletserver.TabletServer;
-import org.apache.accumulo.server.util.Initialize;
-import org.apache.accumulo.server.util.PortUtils;
-import org.apache.accumulo.server.util.time.SimpleTimer;
-import org.apache.accumulo.start.Main;
-import org.apache.zookeeper.server.ZooKeeperServerMain;
-
-/**
- * A utility class that will create Zookeeper and Accumulo processes that write all of their data to a single local directory. This class makes it easy to test
- * code against a real Accumulo instance. Its much more accurate for testing than MockAccumulo, but much slower than MockAccumulo.
- *
- * @since 1.5.0, 1.4.4
- */
-public class MiniAccumuloCluster {
-
- private static final String INSTANCE_SECRET = "DONTTELL";
- private static final String INSTANCE_NAME = "miniInstance";
-
- private static class LogWriter extends Thread {
- private BufferedReader in;
- private BufferedWriter out;
-
- /**
- * @throws java.io.IOException
- */
- public LogWriter(InputStream stream, File logFile) throws IOException {
- this.setDaemon(true);
- this.in = new BufferedReader(new InputStreamReader(stream));
- out = new BufferedWriter(new FileWriter(logFile));
-
- SimpleTimer.getInstance().schedule(new TimerTask() {
- @Override
- public void run() {
- try {
- flush();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- }, 1000, 1000);
- }
-
- public synchronized void flush() throws IOException {
- if (out != null)
- out.flush();
- }
-
- @Override
- public void run() {
- String line;
-
- try {
- while ((line = in.readLine()) != null) {
- out.append(line);
- out.append("\n");
- }
-
- synchronized (this) {
- out.close();
- out = null;
- in.close();
- }
-
- } catch (IOException e) {
- }
- }
- }
-
- private File libDir;
- private File confDir;
- private File zooKeeperDir;
- private File accumuloDir;
- private File zooCfgFile;
- private File logDir;
- private File walogDir;
-
- private Process zooKeeperProcess;
- private Process masterProcess;
- private Process loggerProcess;
-
- private int zooKeeperPort;
-
- private List<LogWriter> logWriters = new ArrayList<LogWriter>();
-
- private MiniAccumuloConfig config;
- private Process[] tabletServerProcesses;
-
- Process exec(Class<? extends Object> clazz, String... args) throws IOException {
- String javaHome = System.getProperty("java.home");
- String javaBin = javaHome + File.separator + "bin" + File.separator + "java";
- String classpath = System.getProperty("java.class.path");
-
- classpath = confDir.getAbsolutePath() + File.pathSeparator + classpath;
-
- String className = clazz.getCanonicalName();
-
- ArrayList<String> argList = new ArrayList<String>();
-
- argList.addAll(Arrays.asList(javaBin, "-cp", classpath, "-Xmx128m", "-XX:+UseConcMarkSweepGC", "-XX:CMSInitiatingOccupancyFraction=75",
- Main.class.getName(), className));
-
- argList.addAll(Arrays.asList(args));
-
- ProcessBuilder builder = new ProcessBuilder(argList);
-
- builder.environment().put("ACCUMULO_HOME", config.getDir().getAbsolutePath());
- builder.environment().put("ACCUMULO_LOG_DIR", logDir.getAbsolutePath());
-
- // if we're running under accumulo.start, we forward these env vars
- String env = System.getenv("HADOOP_PREFIX");
- if (env != null)
- builder.environment().put("HADOOP_PREFIX", env);
- env = System.getenv("ZOOKEEPER_HOME");
- if (env != null)
- builder.environment().put("ZOOKEEPER_HOME", env);
-
- Process process = builder.start();
-
- LogWriter lw;
- lw = new LogWriter(process.getErrorStream(), new File(logDir, clazz.getSimpleName() + "_" + process.hashCode() + ".err"));
- logWriters.add(lw);
- lw.start();
- lw = new LogWriter(process.getInputStream(), new File(logDir, clazz.getSimpleName() + "_" + process.hashCode() + ".out"));
- logWriters.add(lw);
- lw.start();
-
- return process;
- }
-
- private void appendProp(FileWriter fileWriter, Property key, String value, Map<String,String> siteConfig) throws IOException {
- appendProp(fileWriter, key.getKey(), value, siteConfig);
- }
-
- private void appendProp(FileWriter fileWriter, String key, String value, Map<String,String> siteConfig) throws IOException {
- if (!siteConfig.containsKey(key))
- fileWriter.append("<property><name>" + key + "</name><value>" + value + "</value></property>\n");
- }
-
- /**
- * @param dir
- * An empty or nonexistant temp directoy that Accumulo and Zookeeper can store data in. Creating the directory is left to the user. Java 7, Guava,
- * and Junit provide methods for creating temporary directories.
- * @param rootPassword
- * Initial root password for instance.
- * @throws IOException
- */
- public MiniAccumuloCluster(File dir, String rootPassword) throws IOException {
- this(new MiniAccumuloConfig(dir, rootPassword));
- }
-
- /**
- * @param config
- * initial configuration
- * @throws IOException
- */
-
- public MiniAccumuloCluster(MiniAccumuloConfig config) throws IOException {
-
- if (config.getDir().exists() && !config.getDir().isDirectory())
- throw new IllegalArgumentException("Must pass in directory, " + config.getDir() + " is a file");
-
- if (config.getDir().exists() && config.getDir().list().length != 0)
- throw new IllegalArgumentException("Directory " + config.getDir() + " is not empty");
-
- this.config = config;
-
- libDir = new File(config.getDir(), "lib");
- confDir = new File(config.getDir(), "conf");
- accumuloDir = new File(config.getDir(), "accumulo");
- zooKeeperDir = new File(config.getDir(), "zookeeper");
- logDir = new File(config.getDir(), "logs");
- walogDir = new File(config.getDir(), "walogs");
-
- confDir.mkdirs();
- accumuloDir.mkdirs();
- zooKeeperDir.mkdirs();
- logDir.mkdirs();
- walogDir.mkdirs();
- libDir.mkdirs();
-
- zooKeeperPort = PortUtils.getRandomFreePort();
-
- File siteFile = new File(confDir, "accumulo-site.xml");
-
- FileWriter fileWriter = new FileWriter(siteFile);
- fileWriter.append("<configuration>\n");
-
- HashMap<String,String> siteConfig = new HashMap<String,String>(config.getSiteConfig());
-
- appendProp(fileWriter, Property.INSTANCE_DFS_URI, "file:///", siteConfig);
- appendProp(fileWriter, Property.INSTANCE_DFS_DIR, accumuloDir.getAbsolutePath(), siteConfig);
- appendProp(fileWriter, Property.INSTANCE_ZK_HOST, "localhost:" + zooKeeperPort, siteConfig);
- appendProp(fileWriter, Property.INSTANCE_SECRET, INSTANCE_SECRET, siteConfig);
- appendProp(fileWriter, Property.MASTER_CLIENTPORT, "" + PortUtils.getRandomFreePort(), siteConfig);
- appendProp(fileWriter, Property.TSERV_CLIENTPORT, "" + PortUtils.getRandomFreePort(), siteConfig);
- appendProp(fileWriter, Property.TSERV_PORTSEARCH, "true", siteConfig);
- appendProp(fileWriter, Property.LOGGER_DIR, walogDir.getAbsolutePath(), siteConfig);
- appendProp(fileWriter, Property.TSERV_DATACACHE_SIZE, "10M", siteConfig);
- appendProp(fileWriter, Property.TSERV_INDEXCACHE_SIZE, "10M", siteConfig);
- appendProp(fileWriter, Property.TSERV_MAXMEM, "50M", siteConfig);
- appendProp(fileWriter, Property.TSERV_WALOG_MAX_SIZE, "100M", siteConfig);
- appendProp(fileWriter, Property.TSERV_NATIVEMAP_ENABLED, "false", siteConfig);
- appendProp(fileWriter, Property.TRACE_PORT, "" + PortUtils.getRandomFreePort(), siteConfig);
- appendProp(fileWriter, Property.LOGGER_SORT_BUFFER_SIZE, "50M", siteConfig);
- appendProp(fileWriter, Property.LOGGER_PORTSEARCH, "true", siteConfig);
-
- // since there is a small amount of memory, check more frequently for majc... setting may not be needed in 1.5
- appendProp(fileWriter, Property.TSERV_MAJC_DELAY, "3", siteConfig);
- String cp = System.getenv("ACCUMULO_HOME") + "/lib/.*.jar," + "$ZOOKEEPER_HOME/zookeeper[^.].*.jar," + "$HADOOP_HOME/[^.].*.jar,"
- + "$HADOOP_HOME/lib/[^.].*.jar," + "$HADOOP_PREFIX/share/hadoop/common/.*.jar," + "$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,"
- + "$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar," + "$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar";
- appendProp(fileWriter, Property.GENERAL_CLASSPATHS, cp, siteConfig);
- appendProp(fileWriter, Property.GENERAL_DYNAMIC_CLASSPATHS, libDir.getAbsolutePath(), siteConfig);
-
- for (Entry<String,String> entry : siteConfig.entrySet())
- fileWriter.append("<property><name>" + entry.getKey() + "</name><value>" + entry.getValue() + "</value></property>\n");
- fileWriter.append("</configuration>\n");
- fileWriter.close();
-
- zooCfgFile = new File(confDir, "zoo.cfg");
- fileWriter = new FileWriter(zooCfgFile);
-
- // zookeeper uses Properties to read its config, so use that to write in order to properly escape things like Windows paths
- Properties zooCfg = new Properties();
- zooCfg.setProperty("tickTime", "1000");
- zooCfg.setProperty("initLimit", "10");
- zooCfg.setProperty("syncLimit", "5");
- zooCfg.setProperty("clientPort", zooKeeperPort + "");
- zooCfg.setProperty("maxClientCnxns", "100");
- zooCfg.setProperty("dataDir", zooKeeperDir.getAbsolutePath());
- zooCfg.store(fileWriter, null);
-
- fileWriter.close();
- }
-
- /**
- * Starts Accumulo and Zookeeper processes. Can only be called once.
- *
- * @throws IOException
- * @throws InterruptedException
- * @throws IllegalStateException
- * if already started
- */
- public void start() throws IOException, InterruptedException {
- if (zooKeeperProcess != null)
- throw new IllegalStateException("Already started");
-
- Runtime.getRuntime().addShutdownHook(new Thread() {
- @Override
- public void run() {
- try {
- MiniAccumuloCluster.this.stop();
- } catch (IOException e) {
- e.printStackTrace();
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- });
-
- zooKeeperProcess = exec(Main.class, ZooKeeperServerMain.class.getName(), zooCfgFile.getAbsolutePath());
-
- // sleep a little bit to let zookeeper come up before calling init, seems to work better
- UtilWaitThread.sleep(250);
-
- // TODO initialization could probably be done in process
- Process initProcess = exec(Initialize.class);
- initProcess.getOutputStream().write((INSTANCE_NAME + "\n").getBytes());
- initProcess.getOutputStream().write((config.getRootPassword() + "\n").getBytes());
- initProcess.getOutputStream().write((config.getRootPassword() + "\n").getBytes());
- initProcess.getOutputStream().flush();
- int ret = initProcess.waitFor();
- if (ret != 0) {
- throw new RuntimeException("Initialize process returned " + ret);
- }
-
- tabletServerProcesses = new Process[config.getNumTservers()];
- for (int i = 0; i < config.getNumTservers(); i++) {
- tabletServerProcesses[i] = exec(TabletServer.class);
- }
-
- loggerProcess = exec(LogService.class);
- masterProcess = exec(Master.class);
- }
-
- /**
- * @return Accumulo instance name
- */
-
- public String getInstanceName() {
- return INSTANCE_NAME;
- }
-
- /**
- * @return zookeeper connection string
- */
-
- public String getZooKeepers() {
- return "localhost:" + zooKeeperPort;
- }
-
- /**
- * Stops Accumulo and Zookeeper processes. If stop is not called, there is a shutdown hook that is setup to kill the processes. Howerver its probably best to
- * call stop in a finally block as soon as possible.
- *
- * @throws IOException
- * @throws InterruptedException
- */
-
- public void stop() throws IOException, InterruptedException {
- if (zooKeeperProcess != null)
- zooKeeperProcess.destroy();
- if (loggerProcess != null)
- loggerProcess.destroy();
- if (masterProcess != null)
- masterProcess.destroy();
- if (tabletServerProcesses != null) {
- for (Process tserver : tabletServerProcesses) {
- tserver.destroy();
- }
- }
-
- for (LogWriter lw : logWriters)
- lw.flush();
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e46360cf/src/mini/src/main/java/org/apache/accumulo/mini/MiniAccumuloConfig.java
----------------------------------------------------------------------
diff --git a/src/mini/src/main/java/org/apache/accumulo/mini/MiniAccumuloConfig.java b/src/mini/src/main/java/org/apache/accumulo/mini/MiniAccumuloConfig.java
deleted file mode 100644
index 7472b13..0000000
--- a/src/mini/src/main/java/org/apache/accumulo/mini/MiniAccumuloConfig.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.mini;
-
-import java.io.File;
-import java.util.Collections;
-import java.util.Map;
-
-/**
- * Holds configuration for {@link MiniAccumuloCluster}. Required configurations must be passed to constructor and all other configurations are optional.
- *
- * @since 1.5.0, 1.4.4
- */
-
-public class MiniAccumuloConfig {
-
- private File dir = null;
- private String rootPassword = null;
- private Map<String,String> siteConfig = Collections.emptyMap();
- private int numTservers = 2;
-
- /**
- * @param dir
- * An empty or nonexistant temp directoy that Accumulo and Zookeeper can store data in. Creating the directory is left to the user. Java 7, Guava,
- * and Junit provide methods for creating temporary directories.
- * @param rootPassword
- * The initial password for the Accumulo root user
- */
-
- public MiniAccumuloConfig(File dir, String rootPassword) {
- this.dir = dir;
- this.rootPassword = rootPassword;
- }
-
- public File getDir() {
- return dir;
- }
-
- public String getRootPassword() {
- return rootPassword;
- }
-
- public int getNumTservers() {
- return numTservers;
- }
-
- /**
- * Calling this method is optional. If not set, it defaults to two.
- *
- * @param numTservers
- * the number of tablet servers that mini accumulo cluster should start
- */
-
- public MiniAccumuloConfig setNumTservers(int numTservers) {
- if (numTservers < 1)
- throw new IllegalArgumentException("Must have at least one tablet server");
- this.numTservers = numTservers;
- return this;
- }
-
- public Map<String,String> getSiteConfig() {
- return siteConfig;
- }
-
- /**
- * Calling this method is optional. If not set, it defautls to an empty map.
- *
- * @param siteConfig
- * key/values that you normally put in accumulo-site.xml can be put here
- */
- public MiniAccumuloConfig setSiteConfig(Map<String,String> siteConfig) {
- this.siteConfig = siteConfig;
- return this;
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e46360cf/src/mini/src/test/java/org/apache/accumulo/mini/MiniAccumuloClusterTest.java
----------------------------------------------------------------------
diff --git a/src/mini/src/test/java/org/apache/accumulo/mini/MiniAccumuloClusterTest.java b/src/mini/src/test/java/org/apache/accumulo/mini/MiniAccumuloClusterTest.java
deleted file mode 100644
index 041eb2a..0000000
--- a/src/mini/src/test/java/org/apache/accumulo/mini/MiniAccumuloClusterTest.java
+++ /dev/null
@@ -1 +0,0 @@
-/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.accumulo.mini;
import java.util.Collections;
import java.util.Map.Entry;
import java.util.UUID;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.ac
cumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.ZooKeeperInstance;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.LongCombiner;
import org.apache.accumulo.core.iterators.user.SummingCombiner;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.accumulo.core.security.TablePermission;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.rules.TemporaryFolde
r;
public class MiniAccumuloClusterTest {
private static final Logger logger = Logger.getLogger(MiniAccumuloClusterTest.class);
private static TemporaryFolder tmpDir = new TemporaryFolder();
private static MiniAccumuloCluster accumulo;
@BeforeClass
public static void setupMiniCluster() throws Exception {
tmpDir.create();
logger.info("MiniCluster started @ " + tmpDir.getRoot());
Logger.getLogger("org.apache.zookeeper").setLevel(Level.ERROR);
accumulo = new MiniAccumuloCluster(tmpDir.getRoot(), "superSecret");
accumulo.start();
}
@Test(timeout = 30000)
public void test() throws Exception {
Connector conn = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers()).getConnector("root", "superSecret".getBytes());
conn.tableOperations().create("table1");
conn.securityOperations().createUser("user1", "pass1".getBytes(), new Authorizations("A", "B"));
conn.securityOperations().grantTablePermission("user1", "table
1", TablePermission.WRITE);
conn.securityOperations().grantTablePermission("user1", "table1", TablePermission.READ);
IteratorSetting is = new IteratorSetting(10, SummingCombiner.class);
SummingCombiner.setEncodingType(is, LongCombiner.Type.STRING);
SummingCombiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("META", "COUNT")));
conn.tableOperations().attachIterator("table1", is);
Connector uconn = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers()).getConnector("user1", "pass1".getBytes());
BatchWriter bw = uconn.createBatchWriter("table1", 10000, 1000000, 2);
UUID uuid = UUID.randomUUID();
Mutation m = new Mutation(uuid.toString());
m.put("META", "SIZE", new ColumnVisibility("A|B"), "8");
m.put("META", "CRC", new ColumnVisibility("A|B"), "456");
m.put("META", "COUNT", new ColumnVisibility("A|B"), "1");
m.put("DATA", "IMG", new ColumnVisibility("A&B"), "ABCDEFGH");
bw.addMut
ation(m);
bw.flush();
m = new Mutation(uuid.toString());
m.put("META", "COUNT", new ColumnVisibility("A|B"), "1");
m.put("META", "CRC", new ColumnVisibility("A|B"), "123");
bw.addMutation(m);
bw.close();
int count = 0;
Scanner scanner = uconn.createScanner("table1", new Authorizations("A"));
for (Entry<Key,Value> entry : scanner) {
if (entry.getKey().getColumnQualifierData().toString().equals("COUNT")) {
Assert.assertEquals("2", entry.getValue().toString());
} else if (entry.getKey().getColumnQualifierData().toString().equals("SIZE")) {
Assert.assertEquals("8", entry.getValue().toString());
} else if (entry.getKey().getColumnQualifierData().toString().equals("CRC")) {
Assert.assertEquals("123", entry.getValue().toString());
} else {
Assert.assertTrue(false);
}
count++;
}
Assert.assertEquals(3, count);
count = 0;
scanner = uconn.createScanner("table1", new Auth
orizations("A", "B"));
for (Entry<Key,Value> entry : scanner) {
if (entry.getKey().getColumnQualifierData().toString().equals("IMG")) {
Assert.assertEquals("ABCDEFGH", entry.getValue().toString());
}
count++;
}
Assert.assertEquals(4, count);
conn.tableOperations().delete("table1");
}
@Test(timeout = 20000)
public void testMultipleTabletServersRunning() throws AccumuloException, AccumuloSecurityException {
Connector conn = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers()).getConnector("root", "superSecret".getBytes());
while (conn.instanceOperations().getTabletServers().size() != 2) {
UtilWaitThread.sleep(500);
}
}
@AfterClass
public static void tearDownMiniCluster() throws Exception {
accumulo.stop();
tmpDir.delete();
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e46360cf/src/mini/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/src/mini/src/test/resources/log4j.properties b/src/mini/src/test/resources/log4j.properties
deleted file mode 100644
index c833d57..0000000
--- a/src/mini/src/test/resources/log4j.properties
+++ /dev/null
@@ -1 +0,0 @@
-log4j.rootLogger=INFO, CA
log4j.appender.CA=org.apache.log4j.ConsoleAppender
log4j.appender.CA.layout=org.apache.log4j.PatternLayout
log4j.appender.CA.layout.ConversionPattern=[%t] %-5p %c %x - %m%n
log4j.logger.org.apache.zookeeper=ERROR,CA
log4j.logger.org.apache.accumulo.core.client.impl.ServerClient=ERROR
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e46360cf/src/proxy/pom.xml
----------------------------------------------------------------------
diff --git a/src/proxy/pom.xml b/src/proxy/pom.xml
index c03d022..5cfde7b 100644
--- a/src/proxy/pom.xml
+++ b/src/proxy/pom.xml
@@ -115,7 +115,7 @@
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-mini</artifactId>
+ <artifactId>accumulo-server</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e46360cf/src/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java
----------------------------------------------------------------------
diff --git a/src/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java b/src/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java
index ae788af..7b106dd 100644
--- a/src/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java
+++ b/src/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java
@@ -16,9 +16,14 @@
*/
package org.apache.accumulo.proxy;
-import com.google.common.io.Files;
-import org.apache.accumulo.mini.MiniAccumuloCluster;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.util.Properties;
+
import org.apache.accumulo.proxy.thrift.AccumuloProxy;
+import org.apache.accumulo.server.mini.MiniAccumuloCluster;
import org.apache.log4j.Logger;
import org.apache.thrift.TProcessor;
import org.apache.thrift.protocol.TCompactProtocol;
@@ -28,11 +33,7 @@ import org.apache.thrift.server.TServer;
import org.apache.thrift.transport.TFramedTransport;
import org.apache.thrift.transport.TNonblockingServerSocket;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.lang.reflect.Constructor;
-import java.util.Properties;
+import com.google.common.io.Files;
public class Proxy {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e46360cf/src/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
----------------------------------------------------------------------
diff --git a/src/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java b/src/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
index ce49bfb..8a898f4 100644
--- a/src/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
+++ b/src/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
@@ -49,7 +49,6 @@ import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.iterators.user.SummingCombiner;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.examples.simple.constraints.NumericValueConstraint;
-import org.apache.accumulo.mini.MiniAccumuloCluster;
import org.apache.accumulo.proxy.thrift.AccumuloException;
import org.apache.accumulo.proxy.thrift.AccumuloProxy.Client;
import org.apache.accumulo.proxy.thrift.AccumuloSecurityException;
@@ -75,6 +74,7 @@ import org.apache.accumulo.proxy.thrift.TimeType;
import org.apache.accumulo.proxy.thrift.UnknownScanner;
import org.apache.accumulo.proxy.thrift.UnknownWriter;
import org.apache.accumulo.proxy.thrift.WriterOptions;
+import org.apache.accumulo.server.mini.MiniAccumuloCluster;
import org.apache.accumulo.server.test.functional.SlowIterator;
import org.apache.accumulo.server.util.PortUtils;
import org.apache.hadoop.conf.Configuration;
@@ -133,7 +133,7 @@ public class SimpleTest {
Properties props = new Properties();
props.put("instance", accumulo.getInstanceName());
props.put("zookeepers", accumulo.getZooKeepers());
-
+
protocolClass = getRandomProtocol();
System.out.println(protocolClass.getName());
@@ -325,7 +325,7 @@ public class SimpleTest {
client.hasNext(scanner);
fail("exception not thrown");
} catch (UnknownScanner us) {}
-
+
try {
client.closeScanner(scanner);
fail("exception not thrown");
@@ -365,7 +365,7 @@ public class SimpleTest {
// this is a oneway call, so it does not throw exceptions
client.update(writer, mutation("row2", "cf", "cq", "value2"));
-
+
try {
client.flush(writer);
fail("exception not thrown");
[44/50] [abbrv] git commit: ACCUMULO-1603 Merge commit '333062d' into
1.5.1-SNAPSHOT. Merging 1.4 into 1.5,
using 1.5 changes in the face of conflicts as 1.4 and 1.5 were in a correct
merged state before the transition from svn to git.
Posted by el...@apache.org.
ACCUMULO-1603 Merge commit '333062d' into 1.5.1-SNAPSHOT. Merging 1.4 into 1.5, using 1.5 changes in the face of conflicts as 1.4 and 1.5 were in a correct merged state before the transition from svn to git.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/b03b8649
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/b03b8649
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/b03b8649
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: b03b86491a74a079d0f81bb22e0811c87b8a1d40
Parents: 35c7591 333062d
Author: Josh Elser <el...@apache.org>
Authored: Tue Jul 23 20:21:58 2013 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Tue Jul 23 20:21:58 2013 -0400
----------------------------------------------------------------------
----------------------------------------------------------------------
[43/50] [abbrv] git commit: ACCUMULO-1603 Pass the exception on to
the logger instead of logging the toString of the exception so we actually
get a stacktrace.
Posted by el...@apache.org.
ACCUMULO-1603 Pass the exception on to the logger instead of logging the
toString of the exception so we actually get a stacktrace.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/108c492c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/108c492c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/108c492c
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 108c492cfac0edcad0dd1d09ff9d90e1de1e2ff1
Parents: 513364f
Author: Josh Elser <el...@apache.org>
Authored: Tue Jul 23 19:58:25 2013 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Tue Jul 23 19:58:25 2013 -0400
----------------------------------------------------------------------
.../main/java/org/apache/accumulo/server/trace/TraceServer.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/108c492c/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java b/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
index 614da56..8e61a89 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
@@ -157,7 +157,7 @@ public class TraceServer implements Watcher {
connector.tableOperations().setProperty(table, Property.TABLE_FORMATTER_CLASS.getKey(), TraceFormatter.class.getName());
break;
} catch (Exception ex) {
- log.info("waiting to checking/create the trace table: " + ex);
+ log.info("Waiting to checking/create the trace table.", ex);
UtilWaitThread.sleep(1000);
}
}
[20/50] [abbrv] git commit: Fixing docs/config.html from previous
commit
Posted by el...@apache.org.
Fixing docs/config.html from previous commit
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.4@1484858 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7d63bdc1
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7d63bdc1
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7d63bdc1
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 7d63bdc15c6be19073857fd7dcec92fdde004830
Parents: e46360c
Author: Corey J. Nolet <cj...@apache.org>
Authored: Tue May 21 16:29:15 2013 +0000
Committer: Corey J. Nolet <cj...@apache.org>
Committed: Tue May 21 16:29:15 2013 +0000
----------------------------------------------------------------------
docs/config.html | 957 +++++++++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 956 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7d63bdc1/docs/config.html
----------------------------------------------------------------------
diff --git a/docs/config.html b/docs/config.html
index a36b5ca..70f4855 100644
--- a/docs/config.html
+++ b/docs/config.html
@@ -1 +1,956 @@
-Classname org.apache.accumulo.core.conf.DefaultConfiguration not found. Please make sure you use the wholly qualified package name.
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+ <head>
+ <title>Accumulo Configuration</title>
+ <link rel='stylesheet' type='text/css' href='documentation.css' media='screen'/>
+ </head>
+ <body>
+ <h1>Apache Accumulo Configuration Management</h1>
+ <p>All accumulo properties have a default value in the source code. Properties can also be set
+ in accumulo-site.xml and in zookeeper on per-table or system-wide basis. If properties are set in more than one location,
+ accumulo will choose the property with the highest precedence. This order of precedence is described
+ below (from highest to lowest):</p>
+ <table>
+ <tr><th>Location</th><th>Description</th></tr>
+ <tr class='highlight'><td><b>Zookeeper<br/>table properties</td>
+ <td>Table properties are applied to the entire cluster when set in zookeeper using the accumulo API or shell. While table properties take precedent over system properties, both will override properties set in accumulo-site.xml<br/><br/>
+ Table properties consist of all properties with the table.* prefix. Table properties are configured on a per-table basis using the following shell commmand:
+ <pre>config -t TABLE -s PROPERTY=VALUE</pre></td>
+ </tr>
+ <tr><td><b>Zookeeper<br/>system properties</td>
+ <td>System properties are applied to the entire cluster when set in zookeeper using the accumulo API or shell. System properties consist of all properties with a 'yes' in the 'Zookeeper Mutable' column in the table below. They are set with the following shell command:
+ <pre>config -s PROPERTY=VALUE</pre>
+ If a table.* property is set using this method, the value will apply to all tables except those configured on per-table basis (which have higher precedence).<br/><br/>
+ While most system properties take effect immediately, some require a restart of the process which is indicated in 'Zookeeper Mutable'.</td>
+ </tr>
+ <tr class='highlight'><td><b>accumulo-site.xml</td>
+ <td>Accumulo processes (master, tserver, etc) read their local accumulo-site.xml on start up. Therefore, changes made to accumulo-site.xml must rsynced across the cluster and processes must be restarted to apply changes.<br/><br/>
+ Certain properties (indicated by a 'no' in 'Zookeeper Mutable') cannot be set in zookeeper and only set in this file. The accumulo-site.xml also allows you to configure tablet servers with different settings.</td>
+ </tr>
+ <tr><td><b>Default</td>
+ <td>All properties have a default value in the source code. This value has the lowest precedence and is overriden if set in accumulo-site.xml or zookeeper.<br/><br/>While the default value is usually optimal, there are cases where a change can increase query and ingest performance.</td>
+ </tr>
+ </table>
+
+ <p>The 'config' command in the shell allows you to view the current system configuration. You can also use the '-t' option to view a table's configuration as below:
+
+ <pre>
+ $ ./bin/accumulo shell -u root
+ Enter current password for 'root'@'ac14': ******
+
+ Shell - Apache Accumulo Interactive Shell
+ -
+ - version: 1.4.4-SNAPSHOT
+ - instance name: ac14
+ - instance id: 4f48fa03-f692-43ce-ae03-94c9ea8b7181
+ -
+ - type 'help' for a list of available commands
+ -
+ root@ac13> config -t foo
+ ---------+---------------------------------------------+------------------------------------------------------
+ SCOPE | NAME | VALUE
+ ---------+---------------------------------------------+------------------------------------------------------
+ default | table.balancer ............................ | org.apache.accumulo.server.master.balancer.DefaultLoadBalancer
+ default | table.bloom.enabled ....................... | false
+ default | table.bloom.error.rate .................... | 0.5%
+ default | table.bloom.hash.type ..................... | murmur
+ default | table.bloom.key.functor ................... | org.apache.accumulo.core.file.keyfunctor.RowFunctor
+ default | table.bloom.load.threshold ................ | 1
+ default | table.bloom.size .......................... | 1048576
+ default | table.cache.block.enable .................. | false
+ default | table.cache.index.enable .................. | false
+ default | table.compaction.major.everything.at ...... | 19700101000000GMT
+ default | table.compaction.major.everything.idle .... | 1h
+ default | table.compaction.major.ratio .............. | 1.3
+ site | @override .............................. | 1.4
+ system | @override .............................. | 1.5
+ table | @override .............................. | 1.6
+ default | table.compaction.minor.idle ............... | 5m
+ default | table.compaction.minor.logs.threshold ..... | 3
+ default | table.failures.ignore ..................... | false
+ </pre>
+
+ <h1>Configuration Properties</h1>
+
+
+ <p>Jump to:
+<a href='#INSTANCE_PREFIX'>instance.*</a> | <a href='#GENERAL_PREFIX'>general.*</a> | <a href='#MASTER_PREFIX'>master.*</a> | <a href='#TSERV_PREFIX'>tserver.*</a> | <a href='#LOGGER_PREFIX'>logger.*</a> | <a href='#GC_PREFIX'>gc.*</a> | <a href='#MONITOR_PREFIX'>monitor.*</a> | <a href='#TRACE_PREFIX'>trace.*</a> | <a href='#TABLE_PREFIX'>table.*</a> | <a href='#TABLE_CONSTRAINT_PREFIX'>table.constraint.*</a> | <a href='#TABLE_ITERATOR_PREFIX'>table.iterator.*</a> | <a href='#TABLE_LOCALITY_GROUP_PREFIX'>table.group.*</a> </p>
+ <table>
+ <tr><td colspan='5'><a name='INSTANCE_PREFIX' class='large'>instance.*</a></td></tr>
+ <tr><td colspan='5'><i>Properties in this category must be consistent throughout a cloud. This is enforced and servers won't be able to communicate if these differ.</i></td></tr>
+ <tr><th>Property</th><th>Type</th><th>Zookeeper Mutable</th><th>Default Value</th><th>Description</th></tr>
+ <tr class='highlight'>
+ <td>instance.dfs.dir</td>
+ <td><b><a href='#ABSOLUTEPATH'>absolute path</a></b></td>
+ <td>no</td>
+ <td><pre>/accumulo</pre></td>
+ <td>HDFS directory in which accumulo instance will run. Do not change after accumulo is initialized.</td>
+ </tr>
+ <tr >
+ <td>instance.dfs.uri</td>
+ <td><b><a href='#URI'>uri</a></b></td>
+ <td>no</td>
+ <td><pre> </pre></td>
+ <td>The url accumulo should use to connect to DFS. If this is empty, accumulo will obtain this information from the hadoop configuration.</td>
+ </tr>
+ <tr class='highlight'>
+ <td>instance.secret</td>
+ <td><b><a href='#STRING'>string</a></b></td>
+ <td>no</td>
+ <td><pre>DEFAULT</pre></td>
+ <td>A secret unique to a given instance that all servers must know in order to communicate with one another. Change it before initialization. To change it later use ./bin/accumulo accumulo.server.util.ChangeSecret [oldpasswd] [newpasswd], and then update conf/accumulo-site.xml everywhere.</td>
+ </tr>
+ <tr >
+ <td>instance.zookeeper.host</td>
+ <td><b><a href='#HOSTLIST'>host list</a></b></td>
+ <td>no</td>
+ <td><pre>localhost:2181</pre></td>
+ <td>Comma separated list of zookeeper servers</td>
+ </tr>
+ <tr class='highlight'>
+ <td>instance.zookeeper.timeout</td>
+ <td><b><a href='#TIMEDURATION'>duration</a></b></td>
+ <td>no</td>
+ <td><pre>30s</pre></td>
+ <td>Zookeeper session timeout; max value when represented as milliseconds should be no larger than 2147483647</td>
+ </tr>
+ <tr><td colspan='5'><a name='GENERAL_PREFIX' class='large'>general.*</a></td></tr>
+ <tr><td colspan='5'><i>Properties in this category affect the behavior of accumulo overall, but do not have to be consistent throughout a cloud.</i></td></tr>
+ <tr><th>Property</th><th>Type</th><th>Zookeeper Mutable</th><th>Default Value</th><th>Description</th></tr>
+ <tr class='highlight'>
+ <td>general.classpaths</td>
+ <td><b><a href='#STRING'>string</a></b></td>
+ <td>no</td>
+ <td><pre>$ACCUMULO_HOME/conf,
+$ACCUMULO_HOME/lib/[^.].$ACCUMULO_VERSION.jar,
+$ACCUMULO_HOME/lib/[^.].*.jar,
+$ZOOKEEPER_HOME/zookeeper[^.].*.jar,
+$HADOOP_HOME/[^.].*.jar,
+$HADOOP_HOME/conf,
+$HADOOP_HOME/lib/[^.].*.jar,
+</pre></td>
+ <td>A list of all of the places to look for a class. Order does matter, as it will look for the jar starting in the first location to the last. Please note, hadoop conf and hadoop lib directories NEED to be here, along with accumulo lib and zookeeper directory. Supports full regex on filename alone.</td>
+ </tr>
+ <tr >
+ <td>general.dynamic.classpaths</td>
+ <td><b><a href='#STRING'>string</a></b></td>
+ <td>no</td>
+ <td><pre>$ACCUMULO_HOME/lib/ext/[^.].*.jar
+</pre></td>
+ <td>A list of all of the places where changes in jars or classes will force a reload of the classloader.</td>
+ </tr>
+ <tr class='highlight'>
+ <td>general.kerberos.keytab</td>
+ <td><b><a href='#PATH'>path</a></b></td>
+ <td>no</td>
+ <td><pre> </pre></td>
+ <td>Path to the kerberos keytab to use. Leave blank if not using kerberoized hdfs</td>
+ </tr>
+ <tr >
+ <td>general.kerberos.principal</td>
+ <td><b><a href='#STRING'>string</a></b></td>
+ <td>no</td>
+ <td><pre> </pre></td>
+ <td>Name of the kerberos principal to use. _HOST will automatically be replaced by the machines hostname in the hostname portion of the principal. Leave blank if not using kerberoized hdfs</td>
+ </tr>
+ <tr class='highlight'>
+ <td>general.rpc.timeout</td>
+ <td><b><a href='#TIMEDURATION'>duration</a></b></td>
+ <td>no</td>
+ <td><pre>120s</pre></td>
+ <td>Time to wait on I/O for simple, short RPC calls</td>
+ </tr>
+ <tr><td colspan='5'><a name='MASTER_PREFIX' class='large'>master.*</a></td></tr>
+ <tr><td colspan='5'><i>Properties in this category affect the behavior of the master server</i></td></tr>
+ <tr><th>Property</th><th>Type</th><th>Zookeeper Mutable</th><th>Default Value</th><th>Description</th></tr>
+ <tr class='highlight'>
+ <td>master.bulk.retries</td>
+ <td><b><a href='#COUNT'>count</a></b></td>
+ <td>yes</td>
+ <td><pre>3</pre></td>
+ <td>The number of attempts to bulk-load a file before giving up.</td>
+ </tr>
+ <tr >
+ <td>master.bulk.threadpool.size</td>
+ <td><b><a href='#COUNT'>count</a></b></td>
+ <td>yes</td>
+ <td><pre>5</pre></td>
+ <td>The number of threads to use when coordinating a bulk-import.</td>
+ </tr>
+ <tr class='highlight'>
+ <td>master.logger.balancer</td>
+ <td><b><a href='#CLASSNAME'>java class</a></b></td>
+ <td>yes</td>
+ <td><pre>org.apache.accumulo.server.master.balancer.SimpleLoggerBalancer</pre></td>
+ <td>The balancer class that accumulo will use to make logger assignment decisions.</td>
+ </tr>
+ <tr >
+ <td>master.port.client</td>
+ <td><b><a href='#PORT'>port</a></b></td>
+ <td>yes but requires restart of the master</td>
+ <td><pre>9999</pre></td>
+ <td>The port used for handling client connections on the master</td>
+ </tr>
+ <tr class='highlight'>
+ <td>master.recovery.max.age</td>
+ <td><b><a href='#TIMEDURATION'>duration</a></b></td>
+ <td>yes</td>
+ <td><pre>60m</pre></td>
+ <td>Recovery files older than this age will be removed.</td>
+ </tr>
+ <tr >
+ <td>master.recovery.pool</td>
+ <td><b><a href='#STRING'>string</a></b></td>
+ <td>yes</td>
+ <td><pre>recovery</pre></td>
+ <td>Priority queue to use for log recovery map/reduce jobs.</td>
+ </tr>
+ <tr class='highlight'>
+ <td>master.recovery.queue</td>
+ <td><b><a href='#STRING'>string</a></b></td>
+ <td>yes</td>
+ <td><pre>default</pre></td>
+ <td>Priority queue to use for log recovery map/reduce jobs.</td>
+ </tr>
+ <tr >
+ <td>master.recovery.reducers</td>
+ <td><b><a href='#COUNT'>count</a></b></td>
+ <td>yes</td>
+ <td><pre>10</pre></td>
+ <td>Number of reducers to use to sort recovery logs (per log)</td>
+ </tr>
+ <tr class='highlight'>
+ <td>master.recovery.sort.mapreduce</td>
+ <td><b><a href='#BOOLEAN'>boolean</a></b></td>
+ <td>yes</td>
+ <td><pre>false</pre></td>
+ <td>If true, use map/reduce to sort write-ahead logs during recovery</td>
+ </tr>
+ <tr >
+ <td>master.recovery.time.max</td>
+ <td><b><a href='#TIMEDURATION'>duration</a></b></td>
+ <td>yes</td>
+ <td><pre>30m</pre></td>
+ <td>The maximum time to attempt recovery before giving up</td>
+ </tr>
+ <tr class='highlight'>
+ <td>master.server.threadcheck.time</td>
+ <td><b><a href='#TIMEDURATION'>duration</a></b></td>
+ <td>yes</td>
+ <td><pre>1s</pre></td>
+ <td>The time between adjustments of the server thread pool.</td>
+ </tr>
+ <tr >
+ <td>master.server.threads.minimum</td>
+ <td><b><a href='#COUNT'>count</a></b></td>
+ <td>yes</td>
+ <td><pre>2</pre></td>
+ <td>The minimum number of threads to use to handle incoming requests.</td>
+ </tr>
+ <tr class='highlight'>
+ <td>master.tablet.balancer</td>
+ <td><b><a href='#CLASSNAME'>java class</a></b></td>
+ <td>yes</td>
+ <td><pre>org.apache.accumulo.server.master.balancer.TableLoadBalancer</pre></td>
+ <td>The balancer class that accumulo will use to make tablet assignment and migration decisions.</td>
+ </tr>
+ <tr><td colspan='5'><a name='TSERV_PREFIX' class='large'>tserver.*</a></td></tr>
+ <tr><td colspan='5'><i>Properties in this category affect the behavior of the tablet servers</i></td></tr>
+ <tr><th>Property</th><th>Type</th><th>Zookeeper Mutable</th><th>Default Value</th><th>Description</th></tr>
+ <tr class='highlight'>
+ <td>tserver.bloom.load.concurrent.max</td>
+ <td><b><a href='#COUNT'>count</a></b></td>
+ <td>yes</td>
+ <td><pre>4</pre></td>
+ <td>The number of concurrent threads that will load bloom filters in the background. Setting this to zero will make bloom filters load in the foreground.</td>
+ </tr>
+ <tr >
+ <td>tserver.bulk.assign.threads</td>
+ <td><b><a href='#COUNT'>count</a></b></td>
+ <td>yes</td>
+ <td><pre>1</pre></td>
+ <td>The master delegates bulk file processing and assignment to tablet servers. After the bulk file has been processed, the tablet server will assign the file to the appropriate tablets on all servers. This property controls the number of threads used to communicate to the other servers.</td>
+ </tr>
+ <tr class='highlight'>
+ <td>tserver.bulk.process.threads</td>
+ <td><b><a href='#COUNT'>count</a></b></td>
+ <td>yes</td>
+ <td><pre>1</pre></td>
+ <td>The master will task a tablet server with pre-processing a bulk file prior to assigning it to the appropriate tablet servers. This configuration value controls the number of threads used to process the files.</td>
+ </tr>
+ <tr >
+ <td>tserver.bulk.retry.max</td>
+ <td><b><a href='#COUNT'>count</a></b></td>
+ <td>yes</td>
+ <td><pre>3</pre></td>
+ <td>The number of times the tablet server will attempt to assign a file to a tablet as it migrates and splits.</td>
+ </tr>
+ <tr class='highlight'>
+ <td>tserver.cache.data.size</td>
+ <td><b><a href='#MEMORY'>memory</a></b></td>
+ <td>yes</td>
+ <td><pre>100M</pre></td>
+ <td>Specifies the size of the cache for file data blocks.</td>
+ </tr>
+ <tr >
+ <td>tserver.cache.index.size</td>
+ <td><b><a href='#MEMORY'>memory</a></b></td>
+ <td>yes</td>
+ <td><pre>512M</pre></td>
+ <td>Specifies the size of the cache for file indices.</td>
+ </tr>
+ <tr class='highlight'>
+ <td>tserver.client.timeout</td>
+ <td><b><a href='#TIMEDURATION'>duration</a></b></td>
+ <td>yes</td>
+ <td><pre>3s</pre></td>
+ <td>Time to wait for clients to continue scans before closing a session.</td>
+ </tr>
+ <tr >
+ <td>tserver.compaction.major.concurrent.max</td>
+ <td><b><a href='#COUNT'>count</a></b></td>
+ <td>yes but requires restart of the tserver</td>
+ <td><pre>3</pre></td>
+ <td>The maximum number of concurrent major compactions for a tablet server</td>
+ </tr>
+ <tr class='highlight'>
+ <td>tserver.compaction.major.delay</td>
+ <td><b><a href='#TIMEDURATION'>duration</a></b></td>
+ <td>yes</td>
+ <td><pre>30s</pre></td>
+ <td>Time a tablet server will sleep between checking which tablets need compaction.</td>
+ </tr>
+ <tr >
+ <td>tserver.compaction.major.thread.files.open.max</td>
+ <td><b><a href='#COUNT'>count</a></b></td>
+ <td>yes but requires restart of the tserver</td>
+ <td><pre>10</pre></td>
+ <td>Max number of files a major compaction thread can open at once. </td>
+ </tr>
+ <tr class='highlight'>
+ <td>tserver.compaction.minor.concurrent.max</td>
+ <td><b><a href='#COUNT'>count</a></b></td>
+ <td>yes</td>
+ <td><pre>4</pre></td>
+ <td>The maximum number of concurrent minor compactions for a tablet server</td>
+ </tr>
+ <tr >
+ <td>tserver.default.blocksize</td>
+ <td><b><a href='#MEMORY'>memory</a></b></td>
+ <td>yes</td>
+ <td><pre>1M</pre></td>
+ <td>Specifies a default blocksize for the tserver caches</td>
+ </tr>
+ <tr class='highlight'>
+ <td>tserver.dir.memdump</td>
+ <td><b><a href='#PATH'>path</a></b></td>
+ <td>yes</td>
+ <td><pre>/tmp</pre></td>
+ <td>A long running scan could possibly hold memory that has been minor compacted. To prevent this, the in memory map is dumped to a local file and the scan is switched to that local file. We can not switch to the minor compacted file because it may have been modified by iterators. The file dumped to the local dir is an exact copy of what was in memory.</td>
+ </tr>
+ <tr >
+ <td>tserver.files.open.idle</td>
+ <td><b><a href='#TIMEDURATION'>duration</a></b></td>
+ <td>yes</td>
+ <td><pre>1m</pre></td>
+ <td>Tablet servers leave previously used map files open for future queries. This setting determines how much time an unused map file should be kept open until it is closed.</td>
+ </tr>
+ <tr class='highlight'>
+ <td>tserver.hold.time.max</td>
+ <td><b><a href='#TIMEDURATION'>duration</a></b></td>
+ <td>yes</td>
+ <td><pre>5m</pre></td>
+ <td>The maximum time for a tablet server to be in the "memory full" state. If the tablet server cannot write out memory in this much time, it will assume there is some failure local to its node, and quit. A value of zero is equivalent to forever.</td>
+ </tr>
+ <tr >
+ <td>tserver.logger.count</td>
+ <td><b><a href='#COUNT'>count</a></b></td>
+ <td>yes but requires restart of the tserver</td>
+ <td><pre>2</pre></td>
+ <td>The number of loggers that each tablet server should use.</td>
+ </tr>
+ <tr class='highlight'>
+ <td>tserver.logger.strategy</td>
+ <td><b><a href='#STRING'>string</a></b></td>
+ <td>yes</td>
+ <td><pre>org.apache.accumulo.server.tabletserver.log.RoundRobinLoggerStrategy</pre></td>
+ <td>The classname used to decide which loggers to use.</td>
+ </tr>
+ <tr >
+ <td>tserver.logger.timeout</td>
+ <td><b><a href='#TIMEDURATION'>duration</a></b></td>
+ <td>yes</td>
+ <td><pre>30s</pre></td>
+ <td>The time to wait for a logger to respond to a write-ahead request</td>
+ </tr>
+ <tr class='highlight'>
+ <td>tserver.memory.lock</td>
+ <td><b><a href='#BOOLEAN'>boolean</a></b></td>
+ <td>yes</td>
+ <td><pre>false</pre></td>
+ <td>The tablet server must communicate with zookeeper frequently to maintain its locks. If the tablet server's memory is swapped out the java garbage collector can stop all processing for long periods. Change this property to true and the tablet server will attempt to lock all of its memory to RAM, which may reduce delays during java garbage collection. You will have to modify the system limit for "max locked memory". This feature is only available when running on Linux. Alternatively you may also want to set /proc/sys/vm/swappiness to zero (again, this is Linux-specific).</td>
+ </tr>
+ <tr >
+ <td>tserver.memory.manager</td>
+ <td><b><a href='#CLASSNAME'>java class</a></b></td>
+ <td>yes</td>
+ <td><pre>org.apache.accumulo.server.tabletserver.LargestFirstMemoryManager</pre></td>
+ <td>An implementation of MemoryManger that accumulo will use.</td>
+ </tr>
+ <tr class='highlight'>
+ <td>tserver.memory.maps.max</td>
+ <td><b><a href='#MEMORY'>memory</a></b></td>
+ <td>yes</td>
+ <td><pre>1G</pre></td>
+ <td>Maximum amount of memory that can be used to buffer data written to a tablet server. There are two other properties that can effectively limit memory usage table.compaction.minor.logs.threshold and tserver.walog.max.size. Ensure that table.compaction.minor.logs.threshold * tserver.walog.max.size >= this property.</td>
+ </tr>
+ <tr >
+ <td>tserver.memory.maps.native.enabled</td>
+ <td><b><a href='#BOOLEAN'>boolean</a></b></td>
+ <td>yes but requires restart of the tserver</td>
+ <td><pre>true</pre></td>
+ <td>An in-memory data store for accumulo implemented in c++ that increases the amount of data accumulo can hold in memory and avoids Java GC pauses.</td>
+ </tr>
+ <tr class='highlight'>
+ <td>tserver.metadata.readahead.concurrent.max</td>
+ <td><b><a href='#COUNT'>count</a></b></td>
+ <td>yes</td>
+ <td><pre>8</pre></td>
+ <td>The maximum number of concurrent metadata read ahead that will execute.</td>
+ </tr>
+ <tr >
+ <td>tserver.migrations.concurrent.max</td>
+ <td><b><a href='#COUNT'>count</a></b></td>
+ <td>yes</td>
+ <td><pre>1</pre></td>
+ <td>The maximum number of concurrent tablet migrations for a tablet server</td>
+ </tr>
+ <tr class='highlight'>
+ <td>tserver.monitor.fs</td>
+ <td><b><a href='#BOOLEAN'>boolean</a></b></td>
+ <td>yes</td>
+ <td><pre>true</pre></td>
+ <td>When enabled the tserver will monitor file systems and kill itself when one switches from rw to ro. This is usually and indication that Linux has detected a bad disk.</td>
+ </tr>
+ <tr >
+ <td>tserver.mutation.queue.max</td>
+ <td><b><a href='#MEMORY'>memory</a></b></td>
+ <td>yes</td>
+ <td><pre>256K</pre></td>
+ <td>The amount of memory to use to store write-ahead-log mutations-per-session before flushing them.</td>
+ </tr>
+ <tr class='highlight'>
+ <td>tserver.port.client</td>
+ <td><b><a href='#PORT'>port</a></b></td>
+ <td>yes but requires restart of the tserver</td>
+ <td><pre>9997</pre></td>
+ <td>The port used for handling client connections on the tablet servers</td>
+ </tr>
+ <tr >
+ <td>tserver.port.search</td>
+ <td><b><a href='#BOOLEAN'>boolean</a></b></td>
+ <td>yes</td>
+ <td><pre>false</pre></td>
+ <td>if the ports above are in use, search higher ports until one is available</td>
+ </tr>
+ <tr class='highlight'>
+ <td>tserver.readahead.concurrent.max</td>
+ <td><b><a href='#COUNT'>count</a></b></td>
+ <td>yes</td>
+ <td><pre>16</pre></td>
+ <td>The maximum number of concurrent read ahead that will execute. This effectively limits the number of long running scans that can run concurrently per tserver.</td>
+ </tr>
+ <tr >
+ <td>tserver.scan.files.open.max</td>
+ <td><b><a href='#COUNT'>count</a></b></td>
+ <td>yes but requires restart of the tserver</td>
+ <td><pre>100</pre></td>
+ <td>Maximum total map files that all tablets in a tablet server can open for scans. </td>
+ </tr>
+ <tr class='highlight'>
+ <td>tserver.server.threadcheck.time</td>
+ <td><b><a href='#TIMEDURATION'>duration</a></b></td>
+ <td>yes</td>
+ <td><pre>1s</pre></td>
+ <td>The time between adjustments of the server thread pool.</td>
+ </tr>
+ <tr >
+ <td>tserver.server.threads.minimum</td>
+ <td><b><a href='#COUNT'>count</a></b></td>
+ <td>yes</td>
+ <td><pre>2</pre></td>
+ <td>The minimum number of threads to use to handle incoming requests.</td>
+ </tr>
+ <tr class='highlight'>
+ <td>tserver.session.idle.max</td>
+ <td><b><a href='#TIMEDURATION'>duration</a></b></td>
+ <td>yes</td>
+ <td><pre>1m</pre></td>
+ <td>maximum idle time for a session</td>
+ </tr>
+ <tr >
+ <td>tserver.tablet.split.midpoint.files.max</td>
+ <td><b><a href='#COUNT'>count</a></b></td>
+ <td>yes</td>
+ <td><pre>30</pre></td>
+ <td>To find a tablets split points, all index files are opened. This setting determines how many index files can be opened at once. When there are more index files than this setting multiple passes must be made, which is slower. However opening too many files at once can cause problems.</td>
+ </tr>
+ <tr class='highlight'>
+ <td>tserver.walog.max.size</td>
+ <td><b><a href='#MEMORY'>memory</a></b></td>
+ <td>yes</td>
+ <td><pre>1G</pre></td>
+ <td>The maximum size for each write-ahead log. See comment for property tserver.memory.maps.max</td>
+ </tr>
+ <tr><td colspan='5'><a name='LOGGER_PREFIX' class='large'>logger.*</a></td></tr>
+ <tr><td colspan='5'><i>Properties in this category affect the behavior of the write-ahead logger servers</i></td></tr>
+ <tr><th>Property</th><th>Type</th><th>Zookeeper Mutable</th><th>Default Value</th><th>Description</th></tr>
+ <tr class='highlight'>
+ <td>logger.archive</td>
+ <td><b><a href='#BOOLEAN'>boolean</a></b></td>
+ <td>yes</td>
+ <td><pre>false</pre></td>
+ <td>determines if logs are archived in hdfs</td>
+ </tr>
+ <tr >
+ <td>logger.archive.replication</td>
+ <td><b><a href='#COUNT'>count</a></b></td>
+ <td>yes</td>
+ <td><pre>0</pre></td>
+ <td>determines the replication factor for walogs archived in hdfs, set to zero to use default</td>
+ </tr>
+ <tr class='highlight'>
+ <td>logger.copy.threadpool.size</td>
+ <td><b><a href='#COUNT'>count</a></b></td>
+ <td>yes</td>
+ <td><pre>2</pre></td>
+ <td>size of the thread pool used to copy files from the local log area to HDFS</td>
+ </tr>
+ <tr >
+ <td>logger.dir.walog</td>
+ <td><b><a href='#PATH'>path</a></b></td>
+ <td>yes</td>
+ <td><pre>walogs</pre></td>
+ <td>The directory used to store write-ahead logs on the local filesystem. It is possible to specify a comma-separated list of directories.</td>
+ </tr>
+ <tr class='highlight'>
+ <td>logger.monitor.fs</td>
+ <td><b><a href='#BOOLEAN'>boolean</a></b></td>
+ <td>yes</td>
+ <td><pre>true</pre></td>
+ <td>When enabled the logger will monitor file systems and kill itself when one switches from rw to ro. This is usually and indication that Linux has detected a bad disk.</td>
+ </tr>
+ <tr >
+ <td>logger.port.client</td>
+ <td><b><a href='#PORT'>port</a></b></td>
+ <td>yes but requires restart of the logger</td>
+ <td><pre>11224</pre></td>
+ <td>The port used for write-ahead logger services</td>
+ </tr>
+ <tr class='highlight'>
+ <td>logger.port.search</td>
+ <td><b><a href='#BOOLEAN'>boolean</a></b></td>
+ <td>yes</td>
+ <td><pre>false</pre></td>
+ <td>if the port above is in use, search higher ports until one is available</td>
+ </tr>
+ <tr >
+ <td>logger.recovery.file.replication</td>
+ <td><b><a href='#COUNT'>count</a></b></td>
+ <td>yes</td>
+ <td><pre>2</pre></td>
+ <td>When a logger puts a WALOG into HDFS, it will use this as the replication factor.</td>
+ </tr>
+ <tr class='highlight'>
+ <td>logger.server.threadcheck.time</td>
+ <td><b><a href='#TIMEDURATION'>duration</a></b></td>
+ <td>yes</td>
+ <td><pre>1s</pre></td>
+ <td>The time between adjustments of the server thread pool.</td>
+ </tr>
+ <tr >
+ <td>logger.server.threads.minimum</td>
+ <td><b><a href='#COUNT'>count</a></b></td>
+ <td>yes</td>
+ <td><pre>2</pre></td>
+ <td>The miniumum number of threads to use to handle incoming requests.</td>
+ </tr>
+ <tr class='highlight'>
+ <td>logger.sort.buffer.size</td>
+ <td><b><a href='#MEMORY'>memory</a></b></td>
+ <td>yes</td>
+ <td><pre>200M</pre></td>
+ <td>The amount of memory to use when sorting logs during recovery. Only used when *not* sorting logs with map/reduce.</td>
+ </tr>
+ <tr><td colspan='5'><a name='GC_PREFIX' class='large'>gc.*</a></td></tr>
+ <tr><td colspan='5'><i>Properties in this category affect the behavior of the accumulo garbage collector.</i></td></tr>
+ <tr><th>Property</th><th>Type</th><th>Zookeeper Mutable</th><th>Default Value</th><th>Description</th></tr>
+ <tr class='highlight'>
+ <td>gc.cycle.delay</td>
+ <td><b><a href='#TIMEDURATION'>duration</a></b></td>
+ <td>yes</td>
+ <td><pre>5m</pre></td>
+ <td>Time between garbage collection cycles. In each cycle, old files no longer in use are removed from the filesystem.</td>
+ </tr>
+ <tr >
+ <td>gc.cycle.start</td>
+ <td><b><a href='#TIMEDURATION'>duration</a></b></td>
+ <td>yes</td>
+ <td><pre>30s</pre></td>
+ <td>Time to wait before attempting to garbage collect any old files.</td>
+ </tr>
+ <tr class='highlight'>
+ <td>gc.port.client</td>
+ <td><b><a href='#PORT'>port</a></b></td>
+ <td>yes but requires restart of the gc</td>
+ <td><pre>50091</pre></td>
+ <td>The listening port for the garbage collector's monitor service</td>
+ </tr>
+ <tr >
+ <td>gc.threads.delete</td>
+ <td><b><a href='#COUNT'>count</a></b></td>
+ <td>yes</td>
+ <td><pre>16</pre></td>
+ <td>The number of threads used to delete files</td>
+ </tr>
+ <tr><td colspan='5'><a name='MONITOR_PREFIX' class='large'>monitor.*</a></td></tr>
+ <tr><td colspan='5'><i>Properties in this category affect the behavior of the monitor web server.</i></td></tr>
+ <tr><th>Property</th><th>Type</th><th>Zookeeper Mutable</th><th>Default Value</th><th>Description</th></tr>
+ <tr class='highlight'>
+ <td>monitor.banner.background</td>
+ <td><b><a href='#STRING'>string</a></b></td>
+ <td>yes</td>
+ <td><pre>#304065</pre></td>
+ <td>The background color of the banner text displayed on the monitor page.</td>
+ </tr>
+ <tr >
+ <td>monitor.banner.color</td>
+ <td><b><a href='#STRING'>string</a></b></td>
+ <td>yes</td>
+ <td><pre>#c4c4c4</pre></td>
+ <td>The color of the banner text displayed on the monitor page.</td>
+ </tr>
+ <tr class='highlight'>
+ <td>monitor.banner.text</td>
+ <td><b><a href='#STRING'>string</a></b></td>
+ <td>yes</td>
+ <td><pre> </pre></td>
+ <td>The banner text displayed on the monitor page.</td>
+ </tr>
+ <tr >
+ <td>monitor.port.client</td>
+ <td><b><a href='#PORT'>port</a></b></td>
+ <td>no</td>
+ <td><pre>50095</pre></td>
+ <td>The listening port for the monitor's http service</td>
+ </tr>
+ <tr class='highlight'>
+ <td>monitor.port.log4j</td>
+ <td><b><a href='#PORT'>port</a></b></td>
+ <td>no</td>
+ <td><pre>4560</pre></td>
+ <td>The listening port for the monitor's log4j logging collection.</td>
+ </tr>
+ <tr><td colspan='5'><a name='TRACE_PREFIX' class='large'>trace.*</a></td></tr>
+ <tr><td colspan='5'><i>Properties in this category affect the behavior of distributed tracing.</i></td></tr>
+ <tr><th>Property</th><th>Type</th><th>Zookeeper Mutable</th><th>Default Value</th><th>Description</th></tr>
+ <tr class='highlight'>
+ <td>trace.password</td>
+ <td><b><a href='#STRING'>string</a></b></td>
+ <td>no</td>
+ <td><pre>secret</pre></td>
+ <td>The password for the user used to store distributed traces</td>
+ </tr>
+ <tr >
+ <td>trace.port.client</td>
+ <td><b><a href='#PORT'>port</a></b></td>
+ <td>no</td>
+ <td><pre>12234</pre></td>
+ <td>The listening port for the trace server</td>
+ </tr>
+ <tr class='highlight'>
+ <td>trace.table</td>
+ <td><b><a href='#STRING'>string</a></b></td>
+ <td>no</td>
+ <td><pre>trace</pre></td>
+ <td>The name of the table to store distributed traces</td>
+ </tr>
+ <tr >
+ <td>trace.user</td>
+ <td><b><a href='#STRING'>string</a></b></td>
+ <td>no</td>
+ <td><pre>root</pre></td>
+ <td>The name of the user to store distributed traces</td>
+ </tr>
+ <tr><td colspan='5'><a name='TABLE_PREFIX' class='large'>table.*</a></td></tr>
+ <tr><td colspan='5'><i>Properties in this category affect tablet server treatment of tablets, but can be configured on a per-table basis. Setting these properties in the site file will override the default globally for all tables and not any specific table. However, both the default and the global setting can be overridden per table using the table operations API or in the shell, which sets the overridden value in zookeeper. Restarting accumulo tablet servers after setting these properties in the site file will cause the global setting to take effect. However, you must use the API or the shell to change properties in zookeeper that are set on a table.</i></td></tr>
+ <tr><th>Property</th><th>Type</th><th>Zookeeper Mutable</th><th>Default Value</th><th>Description</th></tr>
+ <tr class='highlight'>
+ <td>table.balancer</td>
+ <td><b><a href='#STRING'>string</a></b></td>
+ <td>yes</td>
+ <td><pre>org.apache.accumulo.server.master.balancer.DefaultLoadBalancer</pre></td>
+ <td>This property can be set to allow the LoadBalanceByTable load balancer to change the called Load Balancer for this table</td>
+ </tr>
+ <tr >
+ <td>table.bloom.enabled</td>
+ <td><b><a href='#BOOLEAN'>boolean</a></b></td>
+ <td>yes</td>
+ <td><pre>false</pre></td>
+ <td>Use bloom filters on this table.</td>
+ </tr>
+ <tr class='highlight'>
+ <td>table.bloom.error.rate</td>
+ <td><b><a href='#FRACTION'>fraction/percentage</a></b></td>
+ <td>yes</td>
+ <td><pre>0.5%</pre></td>
+ <td>Bloom filter error rate.</td>
+ </tr>
+ <tr >
+ <td>table.bloom.hash.type</td>
+ <td><b><a href='#STRING'>string</a></b></td>
+ <td>yes</td>
+ <td><pre>murmur</pre></td>
+ <td>The bloom filter hash type</td>
+ </tr>
+ <tr class='highlight'>
+ <td>table.bloom.key.functor</td>
+ <td><b><a href='#CLASSNAME'>java class</a></b></td>
+ <td>yes</td>
+ <td><pre>org.apache.accumulo.core.file.keyfunctor.RowFunctor</pre></td>
+ <td>A function that can transform the key prior to insertion and check of bloom filter. org.apache.accumulo.core.file.keyfunctor.RowFunctor,,org.apache.accumulo.core.file.keyfunctor.ColumnFamilyFunctor, and org.apache.accumulo.core.file.keyfunctor.ColumnQualifierFunctor are allowable values. One can extend any of the above mentioned classes to perform specialized parsing of the key. </td>
+ </tr>
+ <tr >
+ <td>table.bloom.load.threshold</td>
+ <td><b><a href='#COUNT'>count</a></b></td>
+ <td>yes</td>
+ <td><pre>1</pre></td>
+ <td>This number of seeks that would actually use a bloom filter must occur before a map files bloom filter is loaded. Set this to zero to initiate loading of bloom filters when a map file opened.</td>
+ </tr>
+ <tr class='highlight'>
+ <td>table.bloom.size</td>
+ <td><b><a href='#COUNT'>count</a></b></td>
+ <td>yes</td>
+ <td><pre>1048576</pre></td>
+ <td>Bloom filter size, as number of keys.</td>
+ </tr>
+ <tr >
+ <td>table.cache.block.enable</td>
+ <td><b><a href='#BOOLEAN'>boolean</a></b></td>
+ <td>yes</td>
+ <td><pre>false</pre></td>
+ <td>Determines whether file block cache is enabled.</td>
+ </tr>
+ <tr class='highlight'>
+ <td>table.cache.index.enable</td>
+ <td><b><a href='#BOOLEAN'>boolean</a></b></td>
+ <td>yes</td>
+ <td><pre>true</pre></td>
+ <td>Determines whether index cache is enabled.</td>
+ </tr>
+ <tr >
+ <td>table.compaction.major.everything.idle</td>
+ <td><b><a href='#TIMEDURATION'>duration</a></b></td>
+ <td>yes</td>
+ <td><pre>1h</pre></td>
+ <td>After a tablet has been idle (no mutations) for this time period it may have all of its map file compacted into one. There is no guarantee an idle tablet will be compacted. Compactions of idle tablets are only started when regular compactions are not running. Idle compactions only take place for tablets that have one or more map files.</td>
+ </tr>
+ <tr class='highlight'>
+ <td>table.compaction.major.ratio</td>
+ <td><b><a href='#FRACTION'>fraction/percentage</a></b></td>
+ <td>yes</td>
+ <td><pre>3</pre></td>
+ <td>minimum ratio of total input size to maximum input file size for running a major compaction. When adjusting this property you may want to also adjust table.file.max. Want to avoid the situation where only merging minor compactions occur.</td>
+ </tr>
+ <tr >
+ <td>table.compaction.minor.idle</td>
+ <td><b><a href='#TIMEDURATION'>duration</a></b></td>
+ <td>yes</td>
+ <td><pre>5m</pre></td>
+ <td>After a tablet has been idle (no mutations) for this time period it may have its in-memory map flushed to disk in a minor compaction. There is no guarantee an idle tablet will be compacted.</td>
+ </tr>
+ <tr class='highlight'>
+ <td>table.compaction.minor.logs.threshold</td>
+ <td><b><a href='#COUNT'>count</a></b></td>
+ <td>yes</td>
+ <td><pre>3</pre></td>
+ <td>When there are more than this many write-ahead logs against a tablet, it will be minor compacted. See comment for property tserver.memory.maps.max</td>
+ </tr>
+ <tr >
+ <td>table.failures.ignore</td>
+ <td><b><a href='#BOOLEAN'>boolean</a></b></td>
+ <td>yes</td>
+ <td><pre>false</pre></td>
+ <td>If you want queries for your table to hang or fail when data is missing from the system, then set this to false. When this set to true missing data will be reported but queries will still run possibly returning a subset of the data.</td>
+ </tr>
+ <tr class='highlight'>
+ <td>table.file.blocksize</td>
+ <td><b><a href='#MEMORY'>memory</a></b></td>
+ <td>yes</td>
+ <td><pre>0B</pre></td>
+ <td>Overrides the hadoop dfs.block.size setting so that map files have better query performance. The maximum value for this is 2147483647</td>
+ </tr>
+ <tr >
+ <td>table.file.compress.blocksize</td>
+ <td><b><a href='#MEMORY'>memory</a></b></td>
+ <td>yes</td>
+ <td><pre>100K</pre></td>
+ <td>Overrides the hadoop io.seqfile.compress.blocksize setting so that map files have better query performance. The maximum value for this is 2147483647</td>
+ </tr>
+ <tr class='highlight'>
+ <td>table.file.compress.blocksize.index</td>
+ <td><b><a href='#MEMORY'>memory</a></b></td>
+ <td>yes</td>
+ <td><pre>128K</pre></td>
+ <td>Determines how large index blocks can be in files that support multilevel indexes. The maximum value for this is 2147483647</td>
+ </tr>
+ <tr >
+ <td>table.file.compress.type</td>
+ <td><b><a href='#STRING'>string</a></b></td>
+ <td>yes</td>
+ <td><pre>gz</pre></td>
+ <td>One of gz,lzo,none</td>
+ </tr>
+ <tr class='highlight'>
+ <td>table.file.max</td>
+ <td><b><a href='#COUNT'>count</a></b></td>
+ <td>yes</td>
+ <td><pre>15</pre></td>
+ <td>Determines the max # of files each tablet in a table can have. When adjusting this property you may want to consider adjusting table.compaction.major.ratio also. Setting this property to 0 will make it default to tserver.scan.files.open.max-1, this will prevent a tablet from having more files than can be opened. Setting this property low may throttle ingest and increase query performance.</td>
+ </tr>
+ <tr >
+ <td>table.file.replication</td>
+ <td><b><a href='#COUNT'>count</a></b></td>
+ <td>yes</td>
+ <td><pre>0</pre></td>
+ <td>Determines how many replicas to keep of a tables map files in HDFS. When this value is LTE 0, HDFS defaults are used.</td>
+ </tr>
+ <tr class='highlight'>
+ <td>table.file.type</td>
+ <td><b><a href='#STRING'>string</a></b></td>
+ <td>yes</td>
+ <td><pre>rf</pre></td>
+ <td>Change the type of file a table writes</td>
+ </tr>
+ <tr >
+ <td>table.formatter</td>
+ <td><b><a href='#STRING'>string</a></b></td>
+ <td>yes</td>
+ <td><pre>org.apache.accumulo.core.util.format.DefaultFormatter</pre></td>
+ <td>The Formatter class to apply on results in the shell</td>
+ </tr>
+ <tr class='highlight'>
+ <td>table.groups.enabled</td>
+ <td><b><a href='#STRING'>string</a></b></td>
+ <td>yes</td>
+ <td><pre> </pre></td>
+ <td>A comma separated list of locality group names to enable for this table.</td>
+ </tr>
+ <tr >
+ <td>table.scan.max.memory</td>
+ <td><b><a href='#MEMORY'>memory</a></b></td>
+ <td>yes</td>
+ <td><pre>1M</pre></td>
+ <td>The maximum amount of memory that will be used to cache results of a client query/scan. Once this limit is reached, the buffered data is sent to the client.</td>
+ </tr>
+ <tr class='highlight'>
+ <td>table.security.scan.visibility.default</td>
+ <td><b><a href='#STRING'>string</a></b></td>
+ <td>yes</td>
+ <td><pre> </pre></td>
+ <td>The security label that will be assumed at scan time if an entry does not have a visibility set.<br />Note: An empty security label is displayed as []. The scan results will show an empty visibility even if the visibility from this setting is applied to the entry.<br />CAUTION: If a particular key has an empty security label AND its table's default visibility is also empty, access will ALWAYS be granted for users with permission to that table. Additionally, if this field is changed, all existing data with an empty visibility label will be interpreted with the new label on the next scan.</td>
+ </tr>
+ <tr >
+ <td>table.split.threshold</td>
+ <td><b><a href='#MEMORY'>memory</a></b></td>
+ <td>yes</td>
+ <td><pre>1G</pre></td>
+ <td>When combined size of files exceeds this amount a tablet is split.</td>
+ </tr>
+ <tr class='highlight'>
+ <td>table.walog.enabled</td>
+ <td><b><a href='#BOOLEAN'>boolean</a></b></td>
+ <td>yes</td>
+ <td><pre>true</pre></td>
+ <td>Use the write-ahead log to prevent the loss of data.</td>
+ </tr>
+ <tr><td colspan='5'><a name='TABLE_CONSTRAINT_PREFIX' class='large'>table.constraint.*</a></td></tr>
+ <tr><td colspan='5'><i>Properties in this category are per-table properties that add constraints to a table. These properties start with the category prefix, followed by a number, and their values correspond to a fully qualified Java class that implements the Constraint interface.<br />For example, table.constraint.1 = org.apache.accumulo.core.constraints.MyCustomConstraint and table.constraint.2 = my.package.constraints.MySecondConstraint</i></td></tr>
+ <tr><td colspan='5'><a name='TABLE_ITERATOR_PREFIX' class='large'>table.iterator.*</a></td></tr>
+ <tr><td colspan='5'><i>Properties in this category specify iterators that are applied at various stages (scopes) of interaction with a table. These properties start with the category prefix, followed by a scope (minc, majc, scan, etc.), followed by a period, followed by a name, as in table.iterator.scan.vers, or table.iterator.scan.custom. The values for these properties are a number indicating the ordering in which it is applied, and a class name such as table.iterator.scan.vers = 10,org.apache.accumulo.core.iterators.VersioningIterator<br /> These iterators can take options if additional properties are set that look like this property, but are suffixed with a period, followed by 'opt' followed by another period, and a property name.<br />For example, table.iterator.minc.vers.opt.maxVersions = 3</i></td></tr>
+ <tr><td colspan='5'><a name='TABLE_LOCALITY_GROUP_PREFIX' class='large'>table.group.*</a></td></tr>
+ <tr><td colspan='5'><i>Properties in this category are per-table properties that define locality groups in a table. These properties start with the category prefix, followed by a name, followed by a period, and followed by a property for that group.<br />For example table.group.group1=x,y,z sets the column families for a group called group1. Once configured, group1 can be enabled by adding it to the list of groups in the table.groups.enabled property.<br />Additional group options may be specified for a named group by setting table.group.<name>.opt.<key>=<value>.</i></td></tr>
+ </table>
+ <h1>Property Type Descriptions</h1>
+ <table>
+ <tr><th>Property Type</th><th>Description</th></tr>
+ <tr class='highlight'>
+ <td><h3><a name='TIMEDURATION'>duration</a></td>
+ <td>A non-negative integer optionally followed by a unit of time (whitespace disallowed), as in 30s.<br />If no unit of time is specified, seconds are assumed. Valid units are 'ms', 's', 'm', 'h' for milliseconds, seconds, minutes, and hours.<br />Examples of valid durations are '600', '30s', '45m', '30000ms', '3d', and '1h'.<br />Examples of invalid durations are '1w', '1h30m', '1s 200ms', 'ms', '', and 'a'.<br />Unless otherwise stated, the max value for the duration represented in milliseconds is 9223372036854775807</td>
+ </tr>
+ <tr >
+ <td><h3><a name='DATETIME'>date/time</a></td>
+ <td>A date/time string in the format: YYYYMMDDhhmmssTTT where TTT is the 3 character time zone</td>
+ </tr>
+ <tr class='highlight'>
+ <td><h3><a name='MEMORY'>memory</a></td>
+ <td>A positive integer optionally followed by a unit of memory (whitespace disallowed), as in 2G.<br />If no unit is specified, bytes are assumed. Valid units are 'B', 'K', 'M', 'G', for bytes, kilobytes, megabytes, and gigabytes.<br />Examples of valid memories are '1024', '20B', '100K', '1500M', '2G'.<br />Examples of invalid memories are '1M500K', '1M 2K', '1MB', '1.5G', '1,024K', '', and 'a'.<br .>Unless otherwise stated, the max value for the memory represented in bytes is 9223372036854775807</td>
+ </tr>
+ <tr >
+ <td><h3><a name='HOSTLIST'>host list</a></td>
+ <td>A comma-separated list of hostnames or ip addresses, with optional port numbers.<br />Examples of valid host lists are 'localhost:2000,www.example.com,10.10.1.1:500' and 'localhost'.<br />Examples of invalid host lists are '', ':1000', and 'localhost:80000'</td>
+ </tr>
+ <tr class='highlight'>
+ <td><h3><a name='PORT'>port</a></td>
+ <td>An positive integer in the range 1024-65535, not already in use or specified elsewhere in the configuration</td>
+ </tr>
+ <tr >
+ <td><h3><a name='COUNT'>count</a></td>
+ <td>A non-negative integer in the range of 0-2147483647</td>
+ </tr>
+ <tr class='highlight'>
+ <td><h3><a name='FRACTION'>fraction/percentage</a></td>
+ <td>A floating point number that represents either a fraction or, if suffixed with the '%' character, a percentage.<br />Examples of valid fractions/percentages are '10', '1000%', '0.05', '5%', '0.2%', '0.0005'.<br />Examples of invalid fractions/percentages are '', '10 percent', 'Hulk Hogan'</td>
+ </tr>
+ <tr >
+ <td><h3><a name='PATH'>path</a></td>
+ <td>A string that represents a filesystem path, which can be either relative or absolute to some directory. The filesystem depends on the property.</td>
+ </tr>
+ <tr class='highlight'>
+ <td><h3><a name='ABSOLUTEPATH'>absolute path</a></td>
+ <td>An absolute filesystem path. The filesystem depends on the property. This is the same as path, but enforces that its root is explicitly specified.</td>
+ </tr>
+ <tr >
+ <td><h3><a name='CLASSNAME'>java class</a></td>
+ <td>A fully qualified java class name representing a class on the classpath.<br />An example is 'java.lang.String', rather than 'String'</td>
+ </tr>
+ <tr class='highlight'>
+ <td><h3><a name='STRING'>string</a></td>
+ <td>An arbitrary string of characters whose format is unspecified and interpreted based on the context of the property to which it applies.</td>
+ </tr>
+ <tr >
+ <td><h3><a name='BOOLEAN'>boolean</a></td>
+ <td>Has a value of either 'true' or 'false'</td>
+ </tr>
+ <tr class='highlight'>
+ <td><h3><a name='URI'>uri</a></td>
+ <td>A valid URI</td>
+ </tr>
+ </table>
+ </body>
+</html>
[24/50] [abbrv] git commit: ACCUMULO-1477 made master create new
connection when getting tserver status
Posted by el...@apache.org.
ACCUMULO-1477 made master create new connection when getting tserver status
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.4@1488368 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/07b18f25
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/07b18f25
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/07b18f25
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 07b18f2559c3d759b2d56f70c3291a17245251a2
Parents: e5dad25
Author: Keith Turner <kt...@apache.org>
Authored: Fri May 31 19:45:38 2013 +0000
Committer: Keith Turner <kt...@apache.org>
Committed: Fri May 31 19:45:38 2013 +0000
----------------------------------------------------------------------
.../apache/accumulo/core/util/ThriftUtil.java | 33 ++++++++++++++++++++
.../accumulo/server/master/LiveTServerSet.java | 14 +++++++--
.../apache/accumulo/server/master/Master.java | 2 +-
.../master/tserverOps/ShutdownTServer.java | 2 +-
4 files changed, 46 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/07b18f25/src/core/src/main/java/org/apache/accumulo/core/util/ThriftUtil.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/util/ThriftUtil.java b/src/core/src/main/java/org/apache/accumulo/core/util/ThriftUtil.java
index 22f735c..1b1cdd7 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/util/ThriftUtil.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/util/ThriftUtil.java
@@ -16,6 +16,7 @@
*/
package org.apache.accumulo.core.util;
+import java.io.IOException;
import java.net.InetSocketAddress;
import org.apache.accumulo.cloudtrace.instrument.thrift.TraceWrap;
@@ -29,6 +30,7 @@ import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.security.thrift.ThriftSecurityException;
import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
import org.apache.log4j.Logger;
+import org.apache.thrift.TException;
import org.apache.thrift.TServiceClient;
import org.apache.thrift.TServiceClientFactory;
import org.apache.thrift.protocol.TCompactProtocol;
@@ -125,6 +127,37 @@ public class ThriftUtil {
}
}
+ /**
+ * create a transport that is not pooled
+ */
+ public static TTransport createTransport(String address, int port, AccumuloConfiguration conf) throws TException {
+ TTransport transport = null;
+
+ try {
+ transport = TTimeoutTransport.create(org.apache.accumulo.core.util.AddressUtil.parseAddress(address, port),
+ conf.getTimeInMillis(Property.GENERAL_RPC_TIMEOUT));
+ transport = ThriftUtil.transportFactory().getTransport(transport);
+ transport.open();
+ TTransport tmp = transport;
+ transport = null;
+ return tmp;
+ } catch (IOException ex) {
+ throw new TTransportException(ex);
+ } finally {
+ if (transport != null)
+ transport.close();
+ }
+
+
+ }
+
+ /**
+ * create a transport that is not pooled
+ */
+ public static TTransport createTransport(InetSocketAddress address, AccumuloConfiguration conf) throws TException {
+ return createTransport(address.getAddress().getHostAddress(), address.getPort(), conf);
+ }
+
public static TTransportFactory transportFactory() {
return transportFactory;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/07b18f25/src/server/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java b/src/server/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
index b298004..95243a9 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
@@ -49,6 +49,7 @@ import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Logger;
import org.apache.thrift.TException;
+import org.apache.thrift.transport.TTransport;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.KeeperException.NoNodeException;
import org.apache.zookeeper.KeeperException.NotEmptyException;
@@ -97,12 +98,19 @@ public class LiveTServerSet implements Watcher {
}
}
- public TabletServerStatus getTableMap() throws TException, ThriftSecurityException {
- TabletClientService.Iface client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, ServerConfiguration.getSystemConfiguration());
+ public TabletServerStatus getTableMap(boolean usePooledConnection) throws TException, ThriftSecurityException {
+
+ if (usePooledConnection == true)
+ throw new UnsupportedOperationException();
+
+ TTransport transport = ThriftUtil.createTransport(address, ServerConfiguration.getSystemConfiguration());
+
try {
+ TabletClientService.Iface client = ThriftUtil.createClient(new TabletClientService.Client.Factory(), transport);
return client.getTabletServerStatus(null, SecurityConstants.getSystemCredentials());
} finally {
- ThriftUtil.returnClient(client);
+ if (transport != null)
+ transport.close();
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/07b18f25/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/master/Master.java b/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
index b486343..9970ee8 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
@@ -2037,7 +2037,7 @@ public class Master implements LiveTServerSet.Listener, LoggerWatcher, TableObse
Set<TServerInstance> currentServers = tserverSet.getCurrentServers();
for (TServerInstance server : currentServers) {
try {
- TabletServerStatus status = tserverSet.getConnection(server).getTableMap();
+ TabletServerStatus status = tserverSet.getConnection(server).getTableMap(false);
result.put(server, status);
// TODO maybe remove from bad servers
} catch (Exception ex) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/07b18f25/src/server/src/main/java/org/apache/accumulo/server/master/tserverOps/ShutdownTServer.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/master/tserverOps/ShutdownTServer.java b/src/server/src/main/java/org/apache/accumulo/server/master/tserverOps/ShutdownTServer.java
index 28c025b..2af04e1 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/master/tserverOps/ShutdownTServer.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/master/tserverOps/ShutdownTServer.java
@@ -70,7 +70,7 @@ public class ShutdownTServer extends MasterRepo {
TServerConnection connection = m.getConnection(server);
if (connection != null) {
try {
- TabletServerStatus status = connection.getTableMap();
+ TabletServerStatus status = connection.getTableMap(false);
if (status.tableMap != null && status.tableMap.isEmpty()) {
log.info("tablet server hosts no tablets " + server);
connection.halt(m.getMasterLock());
[07/50] [abbrv] git commit: ACCUMULO-1381 applying Drew Farris's patch
Posted by el...@apache.org.
ACCUMULO-1381 applying Drew Farris's patch
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.4@1480033 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/82511dc0
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/82511dc0
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/82511dc0
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 82511dc0435af68fd6cd21eaf87480552839b3ba
Parents: 87cd623
Author: Eric C. Newton <ec...@apache.org>
Authored: Tue May 7 19:22:48 2013 +0000
Committer: Eric C. Newton <ec...@apache.org>
Committed: Tue May 7 19:22:48 2013 +0000
----------------------------------------------------------------------
.../main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java | 4 +++-
.../org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java | 4 +++-
2 files changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/82511dc0/src/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java b/src/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
index 61ddb2d..045291d 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
@@ -38,7 +38,8 @@ public class PrintInfo {
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
@SuppressWarnings("deprecation")
- FileSystem fs = FileUtil.getFileSystem(conf, AccumuloConfiguration.getSiteConfiguration());
+ FileSystem hadoopFs = FileUtil.getFileSystem(conf, AccumuloConfiguration.getSiteConfiguration());
+ FileSystem localFs = FileSystem.getLocal(conf);
Options opts = new Options();
Option dumpKeys = new Option("d", "dump", false, "dump the key/value pairs");
@@ -57,6 +58,7 @@ public class PrintInfo {
for (String arg : commandLine.getArgs()) {
Path path = new Path(arg);
+ FileSystem fs = hadoopFs.exists(path) ? hadoopFs : localFs; // fall back to local
CachableBlockFile.Reader _rdr = new CachableBlockFile.Reader(fs, path, conf, null, null);
Reader iter = new RFile.Reader(_rdr);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/82511dc0/src/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java b/src/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java
index 4a295f8..88e32d1 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java
@@ -48,8 +48,10 @@ public class PrintInfo {
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
@SuppressWarnings("deprecation")
- FileSystem fs = FileUtil.getFileSystem(conf, AccumuloConfiguration.getSiteConfiguration());
+ FileSystem hadoopFs = FileUtil.getFileSystem(conf, AccumuloConfiguration.getSiteConfiguration());
+ FileSystem localFs = FileSystem.getLocal(conf);
Path path = new Path(args[0]);
+ FileSystem fs = hadoopFs.exists(path) ? hadoopFs : localFs; // fall back to local
printMetaBlockInfo(conf, fs, path);
}
}
[39/50] [abbrv] git commit: ACCUMULO-1563 committing Jonathan Hsieh's
patch to prevent the test from writing to /
Posted by el...@apache.org.
ACCUMULO-1563 committing Jonathan Hsieh's patch to prevent the test from writing to /
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.4@1502582 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/8113ca67
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/8113ca67
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/8113ca67
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 8113ca679bd3dab38cb224cab609e1980f5d0614
Parents: e6fe91a
Author: Eric C. Newton <ec...@apache.org>
Authored: Fri Jul 12 15:07:31 2013 +0000
Committer: Eric C. Newton <ec...@apache.org>
Committed: Fri Jul 12 15:07:31 2013 +0000
----------------------------------------------------------------------
test/system/auto/simple/bulk.py | 10 +++++-----
test/system/auto/simple/compaction.py | 4 ++--
2 files changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8113ca67/test/system/auto/simple/bulk.py
----------------------------------------------------------------------
diff --git a/test/system/auto/simple/bulk.py b/test/system/auto/simple/bulk.py
index 7c53f46..079eafc 100755
--- a/test/system/auto/simple/bulk.py
+++ b/test/system/auto/simple/bulk.py
@@ -42,13 +42,13 @@ class SimpleBulkTest(TestUtilsMixin, unittest.TestCase):
self.masterHost(),
'org.apache.accumulo.server.test.BulkImportDirectory',
[ROOT, ROOT_PASSWORD,
- 'test_ingest', '/testmf', '/testmfFail'])
+ 'test_ingest', 'testmf', 'testmfFail'])
self.wait(handle)
self.assert_(handle.returncode == 0)
def createMapFiles(self):
- args = '-mapFile /testmf/mf%02d -timestamp 1 -size 50 -random 56 %1d %ld 1'
+ args = '-mapFile testmf/mf%02d -timestamp 1 -size 50 -random 56 %1d %ld 1'
log.info('creating map files')
handles = []
for i in range(COUNT):
@@ -73,9 +73,9 @@ class SimpleBulkTest(TestUtilsMixin, unittest.TestCase):
# initialize the database
self.createTable('test_ingest')
- self.execute(self.masterHost(), 'hadoop dfs -rmr /testmf'.split())
- self.execute(self.masterHost(), 'hadoop dfs -rmr /testmfFail'.split())
- self.execute(self.masterHost(), 'hadoop dfs -mkdir /testmfFail'.split())
+ self.execute(self.masterHost(), 'hadoop dfs -rmr testmf'.split())
+ self.execute(self.masterHost(), 'hadoop dfs -rmr testmfFail'.split())
+ self.execute(self.masterHost(), 'hadoop dfs -mkdir testmfFail'.split())
# insert some data
self.createMapFiles()
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8113ca67/test/system/auto/simple/compaction.py
----------------------------------------------------------------------
diff --git a/test/system/auto/simple/compaction.py b/test/system/auto/simple/compaction.py
index ad16f1c..51eb84c 100755
--- a/test/system/auto/simple/compaction.py
+++ b/test/system/auto/simple/compaction.py
@@ -52,8 +52,8 @@ class CompactionTest(SimpleBulkTest):
# initialize the database
self.createTable('test_ingest')
- self.execute(self.masterHost(), 'hadoop dfs -rmr /testmf'.split())
- self.execute(self.masterHost(), 'hadoop dfs -rmr /testmfFail'.split())
+ self.execute(self.masterHost(), 'hadoop dfs -rmr testmf'.split())
+ self.execute(self.masterHost(), 'hadoop dfs -rmr testmfFail'.split())
# insert some data
self.createMapFiles(self.masterHost())
[48/50] [abbrv] git commit: ACCUMULO-1605 avoid recursion in
Span#traceId
Posted by el...@apache.org.
ACCUMULO-1605 avoid recursion in Span#traceId
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/3e971675
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/3e971675
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/3e971675
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 3e971675cb20248f2d4747219e97d7e28d185e08
Parents: 108c492
Author: Eric Newton <ec...@apache.org>
Authored: Wed Jul 24 12:01:36 2013 -0400
Committer: Eric Newton <ec...@apache.org>
Committed: Wed Jul 24 12:01:36 2013 -0400
----------------------------------------------------------------------
.../accumulo/cloudtrace/instrument/impl/MilliSpan.java | 8 +++++---
.../accumulo/cloudtrace/instrument/impl/RootMilliSpan.java | 2 +-
2 files changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3e971675/src/trace/src/main/java/org/apache/accumulo/cloudtrace/instrument/impl/MilliSpan.java
----------------------------------------------------------------------
diff --git a/src/trace/src/main/java/org/apache/accumulo/cloudtrace/instrument/impl/MilliSpan.java b/src/trace/src/main/java/org/apache/accumulo/cloudtrace/instrument/impl/MilliSpan.java
index 4af31d2..34a83ff 100644
--- a/src/trace/src/main/java/org/apache/accumulo/cloudtrace/instrument/impl/MilliSpan.java
+++ b/src/trace/src/main/java/org/apache/accumulo/cloudtrace/instrument/impl/MilliSpan.java
@@ -37,15 +37,17 @@ public class MilliSpan implements Span {
final private Span parent;
final private String description;
final private long spanId;
+ final private long traceId;
private Map<String,String> traceInfo = null;
public Span child(String description) {
- return new MilliSpan(description, next.nextLong(), this);
+ return new MilliSpan(description, next.nextLong(), traceId, this);
}
- public MilliSpan(String description, long id, Span parent) {
+ public MilliSpan(String description, long id, long traceId, Span parent) {
this.description = description;
this.spanId = id;
+ this.traceId = traceId;
this.parent = parent;
this.start = 0;
this.stop = 0;
@@ -110,7 +112,7 @@ public class MilliSpan implements Span {
@Override
public long traceId() {
- return parent.traceId();
+ return traceId;
}
@Override
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3e971675/src/trace/src/main/java/org/apache/accumulo/cloudtrace/instrument/impl/RootMilliSpan.java
----------------------------------------------------------------------
diff --git a/src/trace/src/main/java/org/apache/accumulo/cloudtrace/instrument/impl/RootMilliSpan.java b/src/trace/src/main/java/org/apache/accumulo/cloudtrace/instrument/impl/RootMilliSpan.java
index dc0e0b6..d531e6d 100644
--- a/src/trace/src/main/java/org/apache/accumulo/cloudtrace/instrument/impl/RootMilliSpan.java
+++ b/src/trace/src/main/java/org/apache/accumulo/cloudtrace/instrument/impl/RootMilliSpan.java
@@ -31,7 +31,7 @@ public class RootMilliSpan extends MilliSpan {
}
public RootMilliSpan(String description, long traceId, long spanId, long parentId) {
- super(description, spanId, null);
+ super(description, spanId, traceId, null);
this.traceId = traceId;
this.parentId = parentId;
}
[21/50] [abbrv] git commit: ACCUMULO-1438,
ACCUMULO-1414 Move minicluster into its own package and add proxy to
user manual
Posted by el...@apache.org.
ACCUMULO-1438, ACCUMULO-1414 Move minicluster into its own package and add proxy to user manual
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.4@1484935 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/49201bd2
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/49201bd2
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/49201bd2
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 49201bd2aecbc7fda0adb6a2e5c3f29b62366c44
Parents: 7d63bdc
Author: Corey J. Nolet <cj...@apache.org>
Authored: Tue May 21 20:17:35 2013 +0000
Committer: Corey J. Nolet <cj...@apache.org>
Committed: Tue May 21 20:17:35 2013 +0000
----------------------------------------------------------------------
docs/src/user_manual/chapters/clients.tex | 130 +++++++
pom.xml | 6 +
src/minicluster/pom.xml | 109 ++++++
.../minicluster/MiniAccumuloCluster.java | 364 +++++++++++++++++++
.../minicluster/MiniAccumuloConfig.java | 89 +++++
.../minicluster/MiniAccumuloClusterTest.java | 149 ++++++++
src/proxy/pom.xml | 5 +
.../java/org/apache/accumulo/proxy/Proxy.java | 2 +-
.../org/apache/accumulo/proxy/SimpleTest.java | 8 +-
.../server/mini/MiniAccumuloCluster.java | 364 -------------------
.../server/mini/MiniAccumuloConfig.java | 89 -----
.../server/mini/MiniAccumuloClusterTest.java | 150 --------
12 files changed, 857 insertions(+), 608 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/49201bd2/docs/src/user_manual/chapters/clients.tex
----------------------------------------------------------------------
diff --git a/docs/src/user_manual/chapters/clients.tex b/docs/src/user_manual/chapters/clients.tex
index 86336d6..12e8b69 100644
--- a/docs/src/user_manual/chapters/clients.tex
+++ b/docs/src/user_manual/chapters/clients.tex
@@ -202,3 +202,133 @@ for(Entry<Key,Value> entry : scan)
An example of the BatchScanner can be found at\\
accumulo/docs/examples/README.batch
+
+\section{Proxy}
+
+The proxy API allows the interaction with Accumulo with languages other than Java.
+A proxy server is provided in the codebase and a client can further be generated.
+
+\subsection{Prequisites}
+
+The proxy server can live on any node in which the basic client API would work. That
+means it must be able to communicate with the Master, ZooKeepers, NameNode, and the
+Data nodes. A proxy client only needs the ability to communicate with the proxy server.
+
+
+\subsection{Configuration}
+
+The configuration options for the proxy server live inside of a properties file. At
+the very least, you need to supply the following properties:
+
+\small
+\begin{verbatim}
+protocolFactory=org.apache.thrift.protocol.TCompactProtocol$Factory
+tokenClass=org.apache.accumulo.core.client.security.tokens.PasswordToken
+port=42424
+instance=test
+zookeepers=localhost:2181
+\end{verbatim}
+\normalsize
+
+You can find a sample configuration file in your distribution:
+
+\small
+\begin{verbatim}
+$ACCUMULO_HOME/proxy/proxy.properties.
+\end{verbatim}
+\normalsize
+
+This sample configuration file further demonstrates an abilty to back the proxy server
+by MockAccumulo or the MiniAccumuloCluster.
+
+\subsection{Running the Proxy Server}
+
+After the properties file holding the configuration is created, the proxy server
+can be started using the following command in the Accumulo distribution (assuming
+you your properties file is named config.properties):
+
+\small
+\begin{verbatim}
+$ACCUMULO_HOME/bin/accumulo proxy -p config.properties
+\end{verbatim}
+\normalsize
+
+\subsection{Creating a Proxy Client}
+
+Aside from installing the Thrift compiler, you will also need the language-specific library
+for Thrift installed to generate client code in that language. Typically, your operating
+system's package manager will be able to automatically install these for you in an expected
+location such as /usr/lib/python/site-packages/thrift.
+
+You can find the thrift file for generating the client:
+
+\small
+\begin{verbatim}
+$ACCUMULO_HOME/proxy/proxy.thrift.
+\end{verbatim}
+\normalsize
+
+After a client is generated, the port specified in the configuration properties above will be
+used to connect to the server.
+
+\subsection{Using a Proxy Client}
+
+The following examples have been written in Java and the method signatures may be
+slightly different depending on the language specified when generating client with
+the Thrift compiler. After initiating a connection to the Proxy (see Apache Thrift's
+documentation for examples of connecting to a Thrift service), the methods on the
+proxy client will be available. The first thing to do is log in:
+
+\small
+\begin{verbatim}
+Map password = new HashMap<String,String>();
+password.put("password", "secret");
+ByteBuffer token = client.login("root", password);
+\end{verbatim}
+\normalsize
+
+Once logged in, the token returned will be used for most subsequent calls to the client.
+Let's create a table, add some data, scan the table, and delete it.
+
+
+First, create a table.
+
+\small
+\begin{verbatim}
+client.createTable(token, "myTable", true, TimeType.MILLIS);
+\end{verbatim}
+\normalsize
+
+
+Next, add some data:
+
+\small
+\begin{verbatim}
+// first, create a writer on the server
+String writer = client.createWriter(token, "myTable", new WriterOptions());
+
+// build column updates
+Map<ByteBuffer, List<ColumnUpdate> cells> cellsToUpdate = //...
+
+// send updates to the server
+client.updateAndFlush(writer, "myTable", cellsToUpdate);
+
+client.closeWriter(writer);
+\end{verbatim}
+\normalsize
+
+
+Scan for the data and batch the return of the results on the server:
+
+\small
+\begin{verbatim}
+String scanner = client.createScanner(token, "myTable", new ScanOptions());
+ScanResult results = client.nextK(scanner, 100);
+
+for(KeyValue keyValue : results.getResultsIterator()) {
+ // do something with results
+}
+
+client.closeScanner(scanner);
+\end{verbatim}
+\normalsize
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/accumulo/blob/49201bd2/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 5dcc0c2..d1877b1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -37,6 +37,7 @@
<module>src/start</module>
<module>src/examples</module>
<module>src/proxy</module>
+ <module>src/minicluster</module>
</modules>
<build>
@@ -698,6 +699,11 @@
<artifactId>accumulo-server</artifactId>
<version>1.4.4-SNAPSHOT</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-minicluster</artifactId>
+ <version>1.4.4-SNAPSHOT</version>
+ </dependency>
<!-- additional dependencies we need to provide -->
<dependency>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/49201bd2/src/minicluster/pom.xml
----------------------------------------------------------------------
diff --git a/src/minicluster/pom.xml b/src/minicluster/pom.xml
new file mode 100644
index 0000000..198d1f7
--- /dev/null
+++ b/src/minicluster/pom.xml
@@ -0,0 +1,109 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <parent>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo</artifactId>
+ <version>1.4.4-SNAPSHOT</version>
+ <relativePath>../../</relativePath>
+ </parent>
+
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>accumulo-minicluster</artifactId>
+ <name>accumulo-minicluster</name>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <configuration>
+ <archive>
+ <manifestSections>
+ <manifestSection>
+ <name>accumulo/minicluster/</name>
+ <manifestEntries>
+ <Sealed>true</Sealed>
+ </manifestEntries>
+ </manifestSection>
+ </manifestSections>
+ </archive>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>zookeeper</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-server</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-start</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>cloudtrace</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>jline</groupId>
+ <artifactId>jline</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-configuration</groupId>
+ <artifactId>commons-configuration</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/49201bd2/src/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
----------------------------------------------------------------------
diff --git a/src/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java b/src/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
new file mode 100644
index 0000000..2335560
--- /dev/null
+++ b/src/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
@@ -0,0 +1,364 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.minicluster;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.TimerTask;
+
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.server.logger.LogService;
+import org.apache.accumulo.server.master.Master;
+import org.apache.accumulo.server.tabletserver.TabletServer;
+import org.apache.accumulo.server.util.Initialize;
+import org.apache.accumulo.server.util.PortUtils;
+import org.apache.accumulo.server.util.time.SimpleTimer;
+import org.apache.accumulo.start.Main;
+import org.apache.zookeeper.server.ZooKeeperServerMain;
+
+/**
+ * A utility class that will create Zookeeper and Accumulo processes that write all of their data to a single local directory. This class makes it easy to test
+ * code against a real Accumulo instance. Its much more accurate for testing than MockAccumulo, but much slower than MockAccumulo.
+ *
+ * @since 1.5.0, 1.4.4
+ */
+public class MiniAccumuloCluster {
+
+ private static final String INSTANCE_SECRET = "DONTTELL";
+ private static final String INSTANCE_NAME = "miniInstance";
+
+ private static class LogWriter extends Thread {
+ private BufferedReader in;
+ private BufferedWriter out;
+
+ /**
+ * @throws java.io.IOException
+ */
+ public LogWriter(InputStream stream, File logFile) throws IOException {
+ this.setDaemon(true);
+ this.in = new BufferedReader(new InputStreamReader(stream));
+ out = new BufferedWriter(new FileWriter(logFile));
+
+ SimpleTimer.getInstance().schedule(new TimerTask() {
+ @Override
+ public void run() {
+ try {
+ flush();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }, 1000, 1000);
+ }
+
+ public synchronized void flush() throws IOException {
+ if (out != null)
+ out.flush();
+ }
+
+ @Override
+ public void run() {
+ String line;
+
+ try {
+ while ((line = in.readLine()) != null) {
+ out.append(line);
+ out.append("\n");
+ }
+
+ synchronized (this) {
+ out.close();
+ out = null;
+ in.close();
+ }
+
+ } catch (IOException e) {
+ }
+ }
+ }
+
+ private File libDir;
+ private File confDir;
+ private File zooKeeperDir;
+ private File accumuloDir;
+ private File zooCfgFile;
+ private File logDir;
+ private File walogDir;
+
+ private Process zooKeeperProcess;
+ private Process masterProcess;
+ private Process loggerProcess;
+
+ private int zooKeeperPort;
+
+ private List<LogWriter> logWriters = new ArrayList<LogWriter>();
+
+ private MiniAccumuloConfig config;
+ private Process[] tabletServerProcesses;
+
+ Process exec(Class<? extends Object> clazz, String... args) throws IOException {
+ String javaHome = System.getProperty("java.home");
+ String javaBin = javaHome + File.separator + "bin" + File.separator + "java";
+ String classpath = System.getProperty("java.class.path");
+
+ classpath = confDir.getAbsolutePath() + File.pathSeparator + classpath;
+
+ String className = clazz.getCanonicalName();
+
+ ArrayList<String> argList = new ArrayList<String>();
+
+ argList.addAll(Arrays.asList(javaBin, "-cp", classpath, "-Xmx128m", "-XX:+UseConcMarkSweepGC", "-XX:CMSInitiatingOccupancyFraction=75",
+ Main.class.getName(), className));
+
+ argList.addAll(Arrays.asList(args));
+
+ ProcessBuilder builder = new ProcessBuilder(argList);
+
+ builder.environment().put("ACCUMULO_HOME", config.getDir().getAbsolutePath());
+ builder.environment().put("ACCUMULO_LOG_DIR", logDir.getAbsolutePath());
+
+ // if we're running under accumulo.start, we forward these env vars
+ String env = System.getenv("HADOOP_PREFIX");
+ if (env != null)
+ builder.environment().put("HADOOP_PREFIX", env);
+ env = System.getenv("ZOOKEEPER_HOME");
+ if (env != null)
+ builder.environment().put("ZOOKEEPER_HOME", env);
+
+ Process process = builder.start();
+
+ LogWriter lw;
+ lw = new LogWriter(process.getErrorStream(), new File(logDir, clazz.getSimpleName() + "_" + process.hashCode() + ".err"));
+ logWriters.add(lw);
+ lw.start();
+ lw = new LogWriter(process.getInputStream(), new File(logDir, clazz.getSimpleName() + "_" + process.hashCode() + ".out"));
+ logWriters.add(lw);
+ lw.start();
+
+ return process;
+ }
+
+ private void appendProp(FileWriter fileWriter, Property key, String value, Map<String,String> siteConfig) throws IOException {
+ appendProp(fileWriter, key.getKey(), value, siteConfig);
+ }
+
+ private void appendProp(FileWriter fileWriter, String key, String value, Map<String,String> siteConfig) throws IOException {
+ if (!siteConfig.containsKey(key))
+ fileWriter.append("<property><name>" + key + "</name><value>" + value + "</value></property>\n");
+ }
+
+ /**
+ * @param dir
+ * An empty or nonexistant temp directoy that Accumulo and Zookeeper can store data in. Creating the directory is left to the user. Java 7, Guava,
+ * and Junit provide methods for creating temporary directories.
+ * @param rootPassword
+ * Initial root password for instance.
+ * @throws IOException
+ */
+ public MiniAccumuloCluster(File dir, String rootPassword) throws IOException {
+ this(new MiniAccumuloConfig(dir, rootPassword));
+ }
+
+ /**
+ * @param config
+ * initial configuration
+ * @throws IOException
+ */
+
+ public MiniAccumuloCluster(MiniAccumuloConfig config) throws IOException {
+
+ if (config.getDir().exists() && !config.getDir().isDirectory())
+ throw new IllegalArgumentException("Must pass in directory, " + config.getDir() + " is a file");
+
+ if (config.getDir().exists() && config.getDir().list().length != 0)
+ throw new IllegalArgumentException("Directory " + config.getDir() + " is not empty");
+
+ this.config = config;
+
+ libDir = new File(config.getDir(), "lib");
+ confDir = new File(config.getDir(), "conf");
+ accumuloDir = new File(config.getDir(), "accumulo");
+ zooKeeperDir = new File(config.getDir(), "zookeeper");
+ logDir = new File(config.getDir(), "logs");
+ walogDir = new File(config.getDir(), "walogs");
+
+ confDir.mkdirs();
+ accumuloDir.mkdirs();
+ zooKeeperDir.mkdirs();
+ logDir.mkdirs();
+ walogDir.mkdirs();
+ libDir.mkdirs();
+
+ zooKeeperPort = PortUtils.getRandomFreePort();
+
+ File siteFile = new File(confDir, "accumulo-site.xml");
+
+ FileWriter fileWriter = new FileWriter(siteFile);
+ fileWriter.append("<configuration>\n");
+
+ HashMap<String,String> siteConfig = new HashMap<String,String>(config.getSiteConfig());
+
+ appendProp(fileWriter, Property.INSTANCE_DFS_URI, "file:///", siteConfig);
+ appendProp(fileWriter, Property.INSTANCE_DFS_DIR, accumuloDir.getAbsolutePath(), siteConfig);
+ appendProp(fileWriter, Property.INSTANCE_ZK_HOST, "localhost:" + zooKeeperPort, siteConfig);
+ appendProp(fileWriter, Property.INSTANCE_SECRET, INSTANCE_SECRET, siteConfig);
+ appendProp(fileWriter, Property.MASTER_CLIENTPORT, "" + PortUtils.getRandomFreePort(), siteConfig);
+ appendProp(fileWriter, Property.TSERV_CLIENTPORT, "" + PortUtils.getRandomFreePort(), siteConfig);
+ appendProp(fileWriter, Property.TSERV_PORTSEARCH, "true", siteConfig);
+ appendProp(fileWriter, Property.LOGGER_DIR, walogDir.getAbsolutePath(), siteConfig);
+ appendProp(fileWriter, Property.TSERV_DATACACHE_SIZE, "10M", siteConfig);
+ appendProp(fileWriter, Property.TSERV_INDEXCACHE_SIZE, "10M", siteConfig);
+ appendProp(fileWriter, Property.TSERV_MAXMEM, "50M", siteConfig);
+ appendProp(fileWriter, Property.TSERV_WALOG_MAX_SIZE, "100M", siteConfig);
+ appendProp(fileWriter, Property.TSERV_NATIVEMAP_ENABLED, "false", siteConfig);
+ appendProp(fileWriter, Property.TRACE_PORT, "" + PortUtils.getRandomFreePort(), siteConfig);
+ appendProp(fileWriter, Property.LOGGER_SORT_BUFFER_SIZE, "50M", siteConfig);
+ appendProp(fileWriter, Property.LOGGER_PORTSEARCH, "true", siteConfig);
+
+ // since there is a small amount of memory, check more frequently for majc... setting may not be needed in 1.5
+ appendProp(fileWriter, Property.TSERV_MAJC_DELAY, "3", siteConfig);
+ String cp = System.getenv("ACCUMULO_HOME") + "/lib/.*.jar," + "$ZOOKEEPER_HOME/zookeeper[^.].*.jar," + "$HADOOP_HOME/[^.].*.jar,"
+ + "$HADOOP_HOME/lib/[^.].*.jar," + "$HADOOP_PREFIX/share/hadoop/common/.*.jar," + "$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,"
+ + "$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar," + "$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar";
+ appendProp(fileWriter, Property.GENERAL_CLASSPATHS, cp, siteConfig);
+ appendProp(fileWriter, Property.GENERAL_DYNAMIC_CLASSPATHS, libDir.getAbsolutePath(), siteConfig);
+
+ for (Entry<String,String> entry : siteConfig.entrySet())
+ fileWriter.append("<property><name>" + entry.getKey() + "</name><value>" + entry.getValue() + "</value></property>\n");
+ fileWriter.append("</configuration>\n");
+ fileWriter.close();
+
+ zooCfgFile = new File(confDir, "zoo.cfg");
+ fileWriter = new FileWriter(zooCfgFile);
+
+ // zookeeper uses Properties to read its config, so use that to write in order to properly escape things like Windows paths
+ Properties zooCfg = new Properties();
+ zooCfg.setProperty("tickTime", "1000");
+ zooCfg.setProperty("initLimit", "10");
+ zooCfg.setProperty("syncLimit", "5");
+ zooCfg.setProperty("clientPort", zooKeeperPort + "");
+ zooCfg.setProperty("maxClientCnxns", "100");
+ zooCfg.setProperty("dataDir", zooKeeperDir.getAbsolutePath());
+ zooCfg.store(fileWriter, null);
+
+ fileWriter.close();
+ }
+
+ /**
+ * Starts Accumulo and Zookeeper processes. Can only be called once.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws IllegalStateException
+ * if already started
+ */
+ public void start() throws IOException, InterruptedException {
+ if (zooKeeperProcess != null)
+ throw new IllegalStateException("Already started");
+
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ try {
+ MiniAccumuloCluster.this.stop();
+ } catch (IOException e) {
+ e.printStackTrace();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ });
+
+ zooKeeperProcess = exec(Main.class, ZooKeeperServerMain.class.getName(), zooCfgFile.getAbsolutePath());
+
+ // sleep a little bit to let zookeeper come up before calling init, seems to work better
+ UtilWaitThread.sleep(250);
+
+ // TODO initialization could probably be done in process
+ Process initProcess = exec(Initialize.class);
+ initProcess.getOutputStream().write((INSTANCE_NAME + "\n").getBytes());
+ initProcess.getOutputStream().write((config.getRootPassword() + "\n").getBytes());
+ initProcess.getOutputStream().write((config.getRootPassword() + "\n").getBytes());
+ initProcess.getOutputStream().flush();
+ int ret = initProcess.waitFor();
+ if (ret != 0) {
+ throw new RuntimeException("Initialize process returned " + ret);
+ }
+
+ tabletServerProcesses = new Process[config.getNumTservers()];
+ for (int i = 0; i < config.getNumTservers(); i++) {
+ tabletServerProcesses[i] = exec(TabletServer.class);
+ }
+
+ loggerProcess = exec(LogService.class);
+ masterProcess = exec(Master.class);
+ }
+
+ /**
+ * @return Accumulo instance name
+ */
+
+ public String getInstanceName() {
+ return INSTANCE_NAME;
+ }
+
+ /**
+ * @return zookeeper connection string
+ */
+
+ public String getZooKeepers() {
+ return "localhost:" + zooKeeperPort;
+ }
+
+ /**
+ * Stops Accumulo and Zookeeper processes. If stop is not called, there is a shutdown hook that is setup to kill the processes. Howerver its probably best to
+ * call stop in a finally block as soon as possible.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ */
+
+ public void stop() throws IOException, InterruptedException {
+ if (zooKeeperProcess != null)
+ zooKeeperProcess.destroy();
+ if (loggerProcess != null)
+ loggerProcess.destroy();
+ if (masterProcess != null)
+ masterProcess.destroy();
+ if (tabletServerProcesses != null) {
+ for (Process tserver : tabletServerProcesses) {
+ tserver.destroy();
+ }
+ }
+
+ for (LogWriter lw : logWriters)
+ lw.flush();
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/49201bd2/src/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
----------------------------------------------------------------------
diff --git a/src/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java b/src/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
new file mode 100644
index 0000000..e0a9e94
--- /dev/null
+++ b/src/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.minicluster;
+
+import java.io.File;
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * Holds configuration for {@link MiniAccumuloCluster}. Required configurations must be passed to constructor and all other configurations are optional.
+ *
+ * @since 1.5.0, 1.4.4
+ */
+
+public class MiniAccumuloConfig {
+
+ private File dir = null;
+ private String rootPassword = null;
+ private Map<String,String> siteConfig = Collections.emptyMap();
+ private int numTservers = 2;
+
+ /**
+ * @param dir
+ * An empty or nonexistant temp directoy that Accumulo and Zookeeper can store data in. Creating the directory is left to the user. Java 7, Guava,
+ * and Junit provide methods for creating temporary directories.
+ * @param rootPassword
+ * The initial password for the Accumulo root user
+ */
+
+ public MiniAccumuloConfig(File dir, String rootPassword) {
+ this.dir = dir;
+ this.rootPassword = rootPassword;
+ }
+
+ public File getDir() {
+ return dir;
+ }
+
+ public String getRootPassword() {
+ return rootPassword;
+ }
+
+ public int getNumTservers() {
+ return numTservers;
+ }
+
+ /**
+ * Calling this method is optional. If not set, it defaults to two.
+ *
+ * @param numTservers
+ * the number of tablet servers that mini accumulo cluster should start
+ */
+
+ public MiniAccumuloConfig setNumTservers(int numTservers) {
+ if (numTservers < 1)
+ throw new IllegalArgumentException("Must have at least one tablet server");
+ this.numTservers = numTservers;
+ return this;
+ }
+
+ public Map<String,String> getSiteConfig() {
+ return siteConfig;
+ }
+
+ /**
+ * Calling this method is optional. If not set, it defautls to an empty map.
+ *
+ * @param siteConfig
+ * key/values that you normally put in accumulo-site.xml can be put here
+ */
+ public MiniAccumuloConfig setSiteConfig(Map<String,String> siteConfig) {
+ this.siteConfig = siteConfig;
+ return this;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/49201bd2/src/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterTest.java
----------------------------------------------------------------------
diff --git a/src/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterTest.java b/src/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterTest.java
new file mode 100644
index 0000000..7f1e441
--- /dev/null
+++ b/src/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterTest.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.minicluster;
+
+import java.util.Collections;
+import java.util.Map.Entry;
+import java.util.UUID;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.LongCombiner;
+import org.apache.accumulo.core.iterators.user.SummingCombiner;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+public class MiniAccumuloClusterTest {
+
+ private static final Logger logger = Logger.getLogger(MiniAccumuloClusterTest.class);
+
+ private static TemporaryFolder tmpDir = new TemporaryFolder();
+ private static MiniAccumuloCluster accumulo;
+
+ @BeforeClass
+ public static void setupMiniCluster() throws Exception {
+
+ tmpDir.create();
+ logger.info("MiniCluster started @ " + tmpDir.getRoot());
+ Logger.getLogger("org.apache.zookeeper").setLevel(Level.ERROR);
+
+ accumulo = new MiniAccumuloCluster(tmpDir.getRoot(), "superSecret");
+ accumulo.start();
+ }
+
+ @Test(timeout = 30000)
+ public void test() throws Exception {
+ Connector conn = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers()).getConnector("root", "superSecret".getBytes());
+
+ conn.tableOperations().create("table1");
+
+ conn.securityOperations().createUser("user1", "pass1".getBytes(), new Authorizations("A", "B"));
+ conn.securityOperations().grantTablePermission("user1", "table1", TablePermission.WRITE);
+ conn.securityOperations().grantTablePermission("user1", "table1", TablePermission.READ);
+
+ IteratorSetting is = new IteratorSetting(10, SummingCombiner.class);
+ SummingCombiner.setEncodingType(is, LongCombiner.Type.STRING);
+ SummingCombiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("META", "COUNT")));
+
+ conn.tableOperations().attachIterator("table1", is);
+
+ Connector uconn = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers()).getConnector("user1", "pass1".getBytes());
+
+ BatchWriter bw = uconn.createBatchWriter("table1", 10000, 1000000, 2);
+
+ UUID uuid = UUID.randomUUID();
+
+ Mutation m = new Mutation(uuid.toString());
+ m.put("META", "SIZE", new ColumnVisibility("A|B"), "8");
+ m.put("META", "CRC", new ColumnVisibility("A|B"), "456");
+ m.put("META", "COUNT", new ColumnVisibility("A|B"), "1");
+ m.put("DATA", "IMG", new ColumnVisibility("A&B"), "ABCDEFGH");
+
+ bw.addMutation(m);
+ bw.flush();
+
+ m = new Mutation(uuid.toString());
+ m.put("META", "COUNT", new ColumnVisibility("A|B"), "1");
+ m.put("META", "CRC", new ColumnVisibility("A|B"), "123");
+ bw.addMutation(m);
+
+ bw.close();
+
+ int count = 0;
+ Scanner scanner = uconn.createScanner("table1", new Authorizations("A"));
+ for (Entry<Key,Value> entry : scanner) {
+ if (entry.getKey().getColumnQualifierData().toString().equals("COUNT")) {
+ Assert.assertEquals("2", entry.getValue().toString());
+ } else if (entry.getKey().getColumnQualifierData().toString().equals("SIZE")) {
+ Assert.assertEquals("8", entry.getValue().toString());
+ } else if (entry.getKey().getColumnQualifierData().toString().equals("CRC")) {
+ Assert.assertEquals("123", entry.getValue().toString());
+ } else {
+ Assert.assertTrue(false);
+ }
+ count++;
+ }
+
+ Assert.assertEquals(3, count);
+
+ count = 0;
+ scanner = uconn.createScanner("table1", new Authorizations("A", "B"));
+ for (Entry<Key,Value> entry : scanner) {
+ if (entry.getKey().getColumnQualifierData().toString().equals("IMG")) {
+ Assert.assertEquals("ABCDEFGH", entry.getValue().toString());
+ }
+ count++;
+ }
+
+ Assert.assertEquals(4, count);
+
+ conn.tableOperations().delete("table1");
+ }
+
+ @Test(timeout = 20000)
+ public void testMultipleTabletServersRunning() throws AccumuloException, AccumuloSecurityException {
+
+ Connector conn = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers()).getConnector("root", "superSecret".getBytes());
+
+ while (conn.instanceOperations().getTabletServers().size() != 2) {
+ UtilWaitThread.sleep(500);
+ }
+ }
+
+ @AfterClass
+ public static void tearDownMiniCluster() throws Exception {
+ accumulo.stop();
+ tmpDir.delete();
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/49201bd2/src/proxy/pom.xml
----------------------------------------------------------------------
diff --git a/src/proxy/pom.xml b/src/proxy/pom.xml
index 5cfde7b..cc54f82 100644
--- a/src/proxy/pom.xml
+++ b/src/proxy/pom.xml
@@ -77,6 +77,11 @@
<scope>compile</scope>
</dependency>
<dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-minicluster</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
<scope>compile</scope>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/49201bd2/src/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java
----------------------------------------------------------------------
diff --git a/src/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java b/src/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java
index 7b106dd..a65e5b6 100644
--- a/src/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java
+++ b/src/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java
@@ -23,7 +23,7 @@ import java.lang.reflect.Constructor;
import java.util.Properties;
import org.apache.accumulo.proxy.thrift.AccumuloProxy;
-import org.apache.accumulo.server.mini.MiniAccumuloCluster;
+import org.apache.accumulo.minicluster.MiniAccumuloCluster;
import org.apache.log4j.Logger;
import org.apache.thrift.TProcessor;
import org.apache.thrift.protocol.TCompactProtocol;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/49201bd2/src/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
----------------------------------------------------------------------
diff --git a/src/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java b/src/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
index 8a898f4..dd1bb19 100644
--- a/src/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
+++ b/src/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
@@ -49,6 +49,7 @@ import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.iterators.user.SummingCombiner;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.examples.simple.constraints.NumericValueConstraint;
+import org.apache.accumulo.minicluster.MiniAccumuloCluster;
import org.apache.accumulo.proxy.thrift.AccumuloException;
import org.apache.accumulo.proxy.thrift.AccumuloProxy.Client;
import org.apache.accumulo.proxy.thrift.AccumuloSecurityException;
@@ -74,7 +75,6 @@ import org.apache.accumulo.proxy.thrift.TimeType;
import org.apache.accumulo.proxy.thrift.UnknownScanner;
import org.apache.accumulo.proxy.thrift.UnknownWriter;
import org.apache.accumulo.proxy.thrift.WriterOptions;
-import org.apache.accumulo.server.mini.MiniAccumuloCluster;
import org.apache.accumulo.server.test.functional.SlowIterator;
import org.apache.accumulo.server.util.PortUtils;
import org.apache.hadoop.conf.Configuration;
@@ -133,7 +133,7 @@ public class SimpleTest {
Properties props = new Properties();
props.put("instance", accumulo.getInstanceName());
props.put("zookeepers", accumulo.getZooKeepers());
-
+
protocolClass = getRandomProtocol();
System.out.println(protocolClass.getName());
@@ -325,7 +325,7 @@ public class SimpleTest {
client.hasNext(scanner);
fail("exception not thrown");
} catch (UnknownScanner us) {}
-
+
try {
client.closeScanner(scanner);
fail("exception not thrown");
@@ -365,7 +365,7 @@ public class SimpleTest {
// this is a oneway call, so it does not throw exceptions
client.update(writer, mutation("row2", "cf", "cq", "value2"));
-
+
try {
client.flush(writer);
fail("exception not thrown");
http://git-wip-us.apache.org/repos/asf/accumulo/blob/49201bd2/src/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloCluster.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloCluster.java b/src/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloCluster.java
deleted file mode 100644
index 2cb3ae2..0000000
--- a/src/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloCluster.java
+++ /dev/null
@@ -1,364 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.server.mini;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Properties;
-import java.util.TimerTask;
-
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.util.UtilWaitThread;
-import org.apache.accumulo.server.logger.LogService;
-import org.apache.accumulo.server.master.Master;
-import org.apache.accumulo.server.tabletserver.TabletServer;
-import org.apache.accumulo.server.util.Initialize;
-import org.apache.accumulo.server.util.PortUtils;
-import org.apache.accumulo.server.util.time.SimpleTimer;
-import org.apache.accumulo.start.Main;
-import org.apache.zookeeper.server.ZooKeeperServerMain;
-
-/**
- * A utility class that will create Zookeeper and Accumulo processes that write all of their data to a single local directory. This class makes it easy to test
- * code against a real Accumulo instance. Its much more accurate for testing than MockAccumulo, but much slower than MockAccumulo.
- *
- * @since 1.5.0, 1.4.4
- */
-public class MiniAccumuloCluster {
-
- private static final String INSTANCE_SECRET = "DONTTELL";
- private static final String INSTANCE_NAME = "miniInstance";
-
- private static class LogWriter extends Thread {
- private BufferedReader in;
- private BufferedWriter out;
-
- /**
- * @throws java.io.IOException
- */
- public LogWriter(InputStream stream, File logFile) throws IOException {
- this.setDaemon(true);
- this.in = new BufferedReader(new InputStreamReader(stream));
- out = new BufferedWriter(new FileWriter(logFile));
-
- SimpleTimer.getInstance().schedule(new TimerTask() {
- @Override
- public void run() {
- try {
- flush();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- }, 1000, 1000);
- }
-
- public synchronized void flush() throws IOException {
- if (out != null)
- out.flush();
- }
-
- @Override
- public void run() {
- String line;
-
- try {
- while ((line = in.readLine()) != null) {
- out.append(line);
- out.append("\n");
- }
-
- synchronized (this) {
- out.close();
- out = null;
- in.close();
- }
-
- } catch (IOException e) {
- }
- }
- }
-
- private File libDir;
- private File confDir;
- private File zooKeeperDir;
- private File accumuloDir;
- private File zooCfgFile;
- private File logDir;
- private File walogDir;
-
- private Process zooKeeperProcess;
- private Process masterProcess;
- private Process loggerProcess;
-
- private int zooKeeperPort;
-
- private List<LogWriter> logWriters = new ArrayList<LogWriter>();
-
- private MiniAccumuloConfig config;
- private Process[] tabletServerProcesses;
-
- Process exec(Class<? extends Object> clazz, String... args) throws IOException {
- String javaHome = System.getProperty("java.home");
- String javaBin = javaHome + File.separator + "bin" + File.separator + "java";
- String classpath = System.getProperty("java.class.path");
-
- classpath = confDir.getAbsolutePath() + File.pathSeparator + classpath;
-
- String className = clazz.getCanonicalName();
-
- ArrayList<String> argList = new ArrayList<String>();
-
- argList.addAll(Arrays.asList(javaBin, "-cp", classpath, "-Xmx128m", "-XX:+UseConcMarkSweepGC", "-XX:CMSInitiatingOccupancyFraction=75",
- Main.class.getName(), className));
-
- argList.addAll(Arrays.asList(args));
-
- ProcessBuilder builder = new ProcessBuilder(argList);
-
- builder.environment().put("ACCUMULO_HOME", config.getDir().getAbsolutePath());
- builder.environment().put("ACCUMULO_LOG_DIR", logDir.getAbsolutePath());
-
- // if we're running under accumulo.start, we forward these env vars
- String env = System.getenv("HADOOP_PREFIX");
- if (env != null)
- builder.environment().put("HADOOP_PREFIX", env);
- env = System.getenv("ZOOKEEPER_HOME");
- if (env != null)
- builder.environment().put("ZOOKEEPER_HOME", env);
-
- Process process = builder.start();
-
- LogWriter lw;
- lw = new LogWriter(process.getErrorStream(), new File(logDir, clazz.getSimpleName() + "_" + process.hashCode() + ".err"));
- logWriters.add(lw);
- lw.start();
- lw = new LogWriter(process.getInputStream(), new File(logDir, clazz.getSimpleName() + "_" + process.hashCode() + ".out"));
- logWriters.add(lw);
- lw.start();
-
- return process;
- }
-
- private void appendProp(FileWriter fileWriter, Property key, String value, Map<String,String> siteConfig) throws IOException {
- appendProp(fileWriter, key.getKey(), value, siteConfig);
- }
-
- private void appendProp(FileWriter fileWriter, String key, String value, Map<String,String> siteConfig) throws IOException {
- if (!siteConfig.containsKey(key))
- fileWriter.append("<property><name>" + key + "</name><value>" + value + "</value></property>\n");
- }
-
- /**
- * @param dir
- * An empty or nonexistant temp directoy that Accumulo and Zookeeper can store data in. Creating the directory is left to the user. Java 7, Guava,
- * and Junit provide methods for creating temporary directories.
- * @param rootPassword
- * Initial root password for instance.
- * @throws IOException
- */
- public MiniAccumuloCluster(File dir, String rootPassword) throws IOException {
- this(new MiniAccumuloConfig(dir, rootPassword));
- }
-
- /**
- * @param config
- * initial configuration
- * @throws IOException
- */
-
- public MiniAccumuloCluster(MiniAccumuloConfig config) throws IOException {
-
- if (config.getDir().exists() && !config.getDir().isDirectory())
- throw new IllegalArgumentException("Must pass in directory, " + config.getDir() + " is a file");
-
- if (config.getDir().exists() && config.getDir().list().length != 0)
- throw new IllegalArgumentException("Directory " + config.getDir() + " is not empty");
-
- this.config = config;
-
- libDir = new File(config.getDir(), "lib");
- confDir = new File(config.getDir(), "conf");
- accumuloDir = new File(config.getDir(), "accumulo");
- zooKeeperDir = new File(config.getDir(), "zookeeper");
- logDir = new File(config.getDir(), "logs");
- walogDir = new File(config.getDir(), "walogs");
-
- confDir.mkdirs();
- accumuloDir.mkdirs();
- zooKeeperDir.mkdirs();
- logDir.mkdirs();
- walogDir.mkdirs();
- libDir.mkdirs();
-
- zooKeeperPort = PortUtils.getRandomFreePort();
-
- File siteFile = new File(confDir, "accumulo-site.xml");
-
- FileWriter fileWriter = new FileWriter(siteFile);
- fileWriter.append("<configuration>\n");
-
- HashMap<String,String> siteConfig = new HashMap<String,String>(config.getSiteConfig());
-
- appendProp(fileWriter, Property.INSTANCE_DFS_URI, "file:///", siteConfig);
- appendProp(fileWriter, Property.INSTANCE_DFS_DIR, accumuloDir.getAbsolutePath(), siteConfig);
- appendProp(fileWriter, Property.INSTANCE_ZK_HOST, "localhost:" + zooKeeperPort, siteConfig);
- appendProp(fileWriter, Property.INSTANCE_SECRET, INSTANCE_SECRET, siteConfig);
- appendProp(fileWriter, Property.MASTER_CLIENTPORT, "" + PortUtils.getRandomFreePort(), siteConfig);
- appendProp(fileWriter, Property.TSERV_CLIENTPORT, "" + PortUtils.getRandomFreePort(), siteConfig);
- appendProp(fileWriter, Property.TSERV_PORTSEARCH, "true", siteConfig);
- appendProp(fileWriter, Property.LOGGER_DIR, walogDir.getAbsolutePath(), siteConfig);
- appendProp(fileWriter, Property.TSERV_DATACACHE_SIZE, "10M", siteConfig);
- appendProp(fileWriter, Property.TSERV_INDEXCACHE_SIZE, "10M", siteConfig);
- appendProp(fileWriter, Property.TSERV_MAXMEM, "50M", siteConfig);
- appendProp(fileWriter, Property.TSERV_WALOG_MAX_SIZE, "100M", siteConfig);
- appendProp(fileWriter, Property.TSERV_NATIVEMAP_ENABLED, "false", siteConfig);
- appendProp(fileWriter, Property.TRACE_PORT, "" + PortUtils.getRandomFreePort(), siteConfig);
- appendProp(fileWriter, Property.LOGGER_SORT_BUFFER_SIZE, "50M", siteConfig);
- appendProp(fileWriter, Property.LOGGER_PORTSEARCH, "true", siteConfig);
-
- // since there is a small amount of memory, check more frequently for majc... setting may not be needed in 1.5
- appendProp(fileWriter, Property.TSERV_MAJC_DELAY, "3", siteConfig);
- String cp = System.getenv("ACCUMULO_HOME") + "/lib/.*.jar," + "$ZOOKEEPER_HOME/zookeeper[^.].*.jar," + "$HADOOP_HOME/[^.].*.jar,"
- + "$HADOOP_HOME/lib/[^.].*.jar," + "$HADOOP_PREFIX/share/hadoop/common/.*.jar," + "$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,"
- + "$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar," + "$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar";
- appendProp(fileWriter, Property.GENERAL_CLASSPATHS, cp, siteConfig);
- appendProp(fileWriter, Property.GENERAL_DYNAMIC_CLASSPATHS, libDir.getAbsolutePath(), siteConfig);
-
- for (Entry<String,String> entry : siteConfig.entrySet())
- fileWriter.append("<property><name>" + entry.getKey() + "</name><value>" + entry.getValue() + "</value></property>\n");
- fileWriter.append("</configuration>\n");
- fileWriter.close();
-
- zooCfgFile = new File(confDir, "zoo.cfg");
- fileWriter = new FileWriter(zooCfgFile);
-
- // zookeeper uses Properties to read its config, so use that to write in order to properly escape things like Windows paths
- Properties zooCfg = new Properties();
- zooCfg.setProperty("tickTime", "1000");
- zooCfg.setProperty("initLimit", "10");
- zooCfg.setProperty("syncLimit", "5");
- zooCfg.setProperty("clientPort", zooKeeperPort + "");
- zooCfg.setProperty("maxClientCnxns", "100");
- zooCfg.setProperty("dataDir", zooKeeperDir.getAbsolutePath());
- zooCfg.store(fileWriter, null);
-
- fileWriter.close();
- }
-
- /**
- * Starts Accumulo and Zookeeper processes. Can only be called once.
- *
- * @throws IOException
- * @throws InterruptedException
- * @throws IllegalStateException
- * if already started
- */
- public void start() throws IOException, InterruptedException {
- if (zooKeeperProcess != null)
- throw new IllegalStateException("Already started");
-
- Runtime.getRuntime().addShutdownHook(new Thread() {
- @Override
- public void run() {
- try {
- MiniAccumuloCluster.this.stop();
- } catch (IOException e) {
- e.printStackTrace();
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- });
-
- zooKeeperProcess = exec(Main.class, ZooKeeperServerMain.class.getName(), zooCfgFile.getAbsolutePath());
-
- // sleep a little bit to let zookeeper come up before calling init, seems to work better
- UtilWaitThread.sleep(250);
-
- // TODO initialization could probably be done in process
- Process initProcess = exec(Initialize.class);
- initProcess.getOutputStream().write((INSTANCE_NAME + "\n").getBytes());
- initProcess.getOutputStream().write((config.getRootPassword() + "\n").getBytes());
- initProcess.getOutputStream().write((config.getRootPassword() + "\n").getBytes());
- initProcess.getOutputStream().flush();
- int ret = initProcess.waitFor();
- if (ret != 0) {
- throw new RuntimeException("Initialize process returned " + ret);
- }
-
- tabletServerProcesses = new Process[config.getNumTservers()];
- for (int i = 0; i < config.getNumTservers(); i++) {
- tabletServerProcesses[i] = exec(TabletServer.class);
- }
-
- loggerProcess = exec(LogService.class);
- masterProcess = exec(Master.class);
- }
-
- /**
- * @return Accumulo instance name
- */
-
- public String getInstanceName() {
- return INSTANCE_NAME;
- }
-
- /**
- * @return zookeeper connection string
- */
-
- public String getZooKeepers() {
- return "localhost:" + zooKeeperPort;
- }
-
- /**
- * Stops Accumulo and Zookeeper processes. If stop is not called, there is a shutdown hook that is setup to kill the processes. Howerver its probably best to
- * call stop in a finally block as soon as possible.
- *
- * @throws IOException
- * @throws InterruptedException
- */
-
- public void stop() throws IOException, InterruptedException {
- if (zooKeeperProcess != null)
- zooKeeperProcess.destroy();
- if (loggerProcess != null)
- loggerProcess.destroy();
- if (masterProcess != null)
- masterProcess.destroy();
- if (tabletServerProcesses != null) {
- for (Process tserver : tabletServerProcesses) {
- tserver.destroy();
- }
- }
-
- for (LogWriter lw : logWriters)
- lw.flush();
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/49201bd2/src/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloConfig.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloConfig.java b/src/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloConfig.java
deleted file mode 100644
index a82f277..0000000
--- a/src/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloConfig.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.server.mini;
-
-import java.io.File;
-import java.util.Collections;
-import java.util.Map;
-
-/**
- * Holds configuration for {@link MiniAccumuloCluster}. Required configurations must be passed to constructor and all other configurations are optional.
- *
- * @since 1.5.0, 1.4.4
- */
-
-public class MiniAccumuloConfig {
-
- private File dir = null;
- private String rootPassword = null;
- private Map<String,String> siteConfig = Collections.emptyMap();
- private int numTservers = 2;
-
- /**
- * @param dir
- * An empty or nonexistant temp directoy that Accumulo and Zookeeper can store data in. Creating the directory is left to the user. Java 7, Guava,
- * and Junit provide methods for creating temporary directories.
- * @param rootPassword
- * The initial password for the Accumulo root user
- */
-
- public MiniAccumuloConfig(File dir, String rootPassword) {
- this.dir = dir;
- this.rootPassword = rootPassword;
- }
-
- public File getDir() {
- return dir;
- }
-
- public String getRootPassword() {
- return rootPassword;
- }
-
- public int getNumTservers() {
- return numTservers;
- }
-
- /**
- * Calling this method is optional. If not set, it defaults to two.
- *
- * @param numTservers
- * the number of tablet servers that mini accumulo cluster should start
- */
-
- public MiniAccumuloConfig setNumTservers(int numTservers) {
- if (numTservers < 1)
- throw new IllegalArgumentException("Must have at least one tablet server");
- this.numTservers = numTservers;
- return this;
- }
-
- public Map<String,String> getSiteConfig() {
- return siteConfig;
- }
-
- /**
- * Calling this method is optional. If not set, it defautls to an empty map.
- *
- * @param siteConfig
- * key/values that you normally put in accumulo-site.xml can be put here
- */
- public MiniAccumuloConfig setSiteConfig(Map<String,String> siteConfig) {
- this.siteConfig = siteConfig;
- return this;
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/49201bd2/src/server/src/test/java/org/apache/accumulo/server/mini/MiniAccumuloClusterTest.java
----------------------------------------------------------------------
diff --git a/src/server/src/test/java/org/apache/accumulo/server/mini/MiniAccumuloClusterTest.java b/src/server/src/test/java/org/apache/accumulo/server/mini/MiniAccumuloClusterTest.java
deleted file mode 100644
index 083d4b8..0000000
--- a/src/server/src/test/java/org/apache/accumulo/server/mini/MiniAccumuloClusterTest.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.server.mini;
-
-import java.util.Collections;
-import java.util.Map.Entry;
-import java.util.UUID;
-
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.IteratorSetting;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.client.ZooKeeperInstance;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.iterators.LongCombiner;
-import org.apache.accumulo.core.iterators.user.SummingCombiner;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.security.ColumnVisibility;
-import org.apache.accumulo.core.security.TablePermission;
-import org.apache.accumulo.core.util.UtilWaitThread;
-import org.apache.accumulo.server.mini.MiniAccumuloCluster;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-public class MiniAccumuloClusterTest {
-
- private static final Logger logger = Logger.getLogger(MiniAccumuloClusterTest.class);
-
- private static TemporaryFolder tmpDir = new TemporaryFolder();
- private static MiniAccumuloCluster accumulo;
-
- @BeforeClass
- public static void setupMiniCluster() throws Exception {
-
- tmpDir.create();
- logger.info("MiniCluster started @ " + tmpDir.getRoot());
- Logger.getLogger("org.apache.zookeeper").setLevel(Level.ERROR);
-
- accumulo = new MiniAccumuloCluster(tmpDir.getRoot(), "superSecret");
- accumulo.start();
- }
-
- @Test(timeout = 30000)
- public void test() throws Exception {
- Connector conn = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers()).getConnector("root", "superSecret".getBytes());
-
- conn.tableOperations().create("table1");
-
- conn.securityOperations().createUser("user1", "pass1".getBytes(), new Authorizations("A", "B"));
- conn.securityOperations().grantTablePermission("user1", "table1", TablePermission.WRITE);
- conn.securityOperations().grantTablePermission("user1", "table1", TablePermission.READ);
-
- IteratorSetting is = new IteratorSetting(10, SummingCombiner.class);
- SummingCombiner.setEncodingType(is, LongCombiner.Type.STRING);
- SummingCombiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("META", "COUNT")));
-
- conn.tableOperations().attachIterator("table1", is);
-
- Connector uconn = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers()).getConnector("user1", "pass1".getBytes());
-
- BatchWriter bw = uconn.createBatchWriter("table1", 10000, 1000000, 2);
-
- UUID uuid = UUID.randomUUID();
-
- Mutation m = new Mutation(uuid.toString());
- m.put("META", "SIZE", new ColumnVisibility("A|B"), "8");
- m.put("META", "CRC", new ColumnVisibility("A|B"), "456");
- m.put("META", "COUNT", new ColumnVisibility("A|B"), "1");
- m.put("DATA", "IMG", new ColumnVisibility("A&B"), "ABCDEFGH");
-
- bw.addMutation(m);
- bw.flush();
-
- m = new Mutation(uuid.toString());
- m.put("META", "COUNT", new ColumnVisibility("A|B"), "1");
- m.put("META", "CRC", new ColumnVisibility("A|B"), "123");
- bw.addMutation(m);
-
- bw.close();
-
- int count = 0;
- Scanner scanner = uconn.createScanner("table1", new Authorizations("A"));
- for (Entry<Key,Value> entry : scanner) {
- if (entry.getKey().getColumnQualifierData().toString().equals("COUNT")) {
- Assert.assertEquals("2", entry.getValue().toString());
- } else if (entry.getKey().getColumnQualifierData().toString().equals("SIZE")) {
- Assert.assertEquals("8", entry.getValue().toString());
- } else if (entry.getKey().getColumnQualifierData().toString().equals("CRC")) {
- Assert.assertEquals("123", entry.getValue().toString());
- } else {
- Assert.assertTrue(false);
- }
- count++;
- }
-
- Assert.assertEquals(3, count);
-
- count = 0;
- scanner = uconn.createScanner("table1", new Authorizations("A", "B"));
- for (Entry<Key,Value> entry : scanner) {
- if (entry.getKey().getColumnQualifierData().toString().equals("IMG")) {
- Assert.assertEquals("ABCDEFGH", entry.getValue().toString());
- }
- count++;
- }
-
- Assert.assertEquals(4, count);
-
- conn.tableOperations().delete("table1");
- }
-
- @Test(timeout = 20000)
- public void testMultipleTabletServersRunning() throws AccumuloException, AccumuloSecurityException {
-
- Connector conn = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers()).getConnector("root", "superSecret".getBytes());
-
- while (conn.instanceOperations().getTabletServers().size() != 2) {
- UtilWaitThread.sleep(500);
- }
- }
-
- @AfterClass
- public static void tearDownMiniCluster() throws Exception {
- accumulo.stop();
- tmpDir.delete();
- }
-}
[30/50] [abbrv] git commit: ACCUMULO-1526 provide an xml report format
Posted by el...@apache.org.
ACCUMULO-1526 provide an xml report format
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.4@1495439 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a329a7a5
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a329a7a5
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a329a7a5
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: a329a7a56b8d0f821ac2db93c4610d89ade3dbb1
Parents: a91173b
Author: Eric C. Newton <ec...@apache.org>
Authored: Fri Jun 21 13:45:37 2013 +0000
Committer: Eric C. Newton <ec...@apache.org>
Committed: Fri Jun 21 13:45:37 2013 +0000
----------------------------------------------------------------------
test/system/auto/run.py | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a329a7a5/test/system/auto/run.py
----------------------------------------------------------------------
diff --git a/test/system/auto/run.py b/test/system/auto/run.py
index 214cf3a..5a0678b 100755
--- a/test/system/auto/run.py
+++ b/test/system/auto/run.py
@@ -241,6 +241,8 @@ def main():
help='Do not clean up at the end of the test.')
parser.add_option('-s', '--start', dest='start', default=None,
help='Start the test list at the given test name')
+ parser.add_option('-x', '--xml', dest='xmlreport', default=False, action='store_true',
+ help='Output tests results to xml (jenkins conpatible)')
allTests = getTests()
options = parseArguments(parser, allTests)
@@ -268,7 +270,12 @@ def main():
print "ZOOKEEPER_HOME needs to be set"
sys.exit(1)
- runner = TestRunner()
+ if options.xmlreport:
+ import xmlrunner
+ runner = xmlrunner.XMLTestRunner(output='test-reports')
+ else:
+ runner = TestRunner()
+
suite = unittest.TestSuite()
map(suite.addTest, filtered)
[40/50] [abbrv] git commit: ACCUMULO-1550 committing Jonathan Hsieh's
patch
Posted by el...@apache.org.
ACCUMULO-1550 committing Jonathan Hsieh's patch
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/cd8ebfe8
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/cd8ebfe8
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/cd8ebfe8
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: cd8ebfe895726695378644fad55e886022d67fc6
Parents: 8113ca6
Author: Eric Newton <er...@gmail.com>
Authored: Wed Jul 17 11:56:29 2013 -0400
Committer: Eric Newton <er...@gmail.com>
Committed: Wed Jul 17 11:56:29 2013 -0400
----------------------------------------------------------------------
README | 10 ++--
bin/accumulo | 63 +++++++++++++-------
bin/config.sh | 61 +++++++++++++------
bin/start-all.sh | 8 +--
bin/start-here.sh | 6 +-
bin/start-server.sh | 2 +-
bin/stop-all.sh | 4 +-
bin/stop-here.sh | 4 +-
bin/tdown.sh | 2 +-
bin/tup.sh | 2 +-
.../1GB/native-standalone/accumulo-env.sh | 6 +-
conf/examples/1GB/standalone/accumulo-env.sh | 6 +-
.../2GB/native-standalone/accumulo-env.sh | 6 +-
conf/examples/2GB/standalone/accumulo-env.sh | 6 +-
.../3GB/native-standalone/accumulo-env.sh | 6 +-
conf/examples/3GB/standalone/accumulo-env.sh | 6 +-
.../512MB/native-standalone/accumulo-env.sh | 6 +-
conf/examples/512MB/standalone/accumulo-env.sh | 6 +-
docs/config.html | 2 +-
.../org/apache/accumulo/server/Accumulo.java | 4 +-
.../start/classloader/AccumuloClassLoader.java | 11 +++-
.../java/org/apache/accumulo/start/Test.java | 1 +
test/system/auto/TestUtils.py | 2 +-
test/system/continuous/agitator.pl | 2 +-
test/system/continuous/magitator.pl | 6 +-
test/system/continuous/mapred-setup.sh | 2 +-
test/system/continuous/start-stats.sh | 2 +-
test/system/randomwalk/README | 2 +-
test/system/randomwalk/bin/reset-cluster.sh | 10 ++--
test/system/randomwalk/bin/start-all.sh | 4 +-
test/system/randomwalk/bin/start-local.sh | 4 +-
test/system/scalability/run.py | 2 +-
32 files changed, 158 insertions(+), 106 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd8ebfe8/README
----------------------------------------------------------------------
diff --git a/README b/README
index 1f514de..8cfbd53 100644
--- a/README
+++ b/README
@@ -82,12 +82,12 @@ accumulo client code. These directories must be at the same location on every
node in the cluster.
If you are configuring a larger cluster you will need to create the configuration
-files yourself and propogate the changes to the $ACCUMULO_HOME/conf directories:
+files yourself and propogate the changes to the $ACCUMULO_CONF_DIR directories:
- Create a "slaves" file in $ACCUMULO_HOME/conf/. This is a list of machines
+ Create a "slaves" file in $ACCUMULO_CONF_DIR/. This is a list of machines
where tablet servers and loggers will run.
- Create a "masters" file in $ACCUMULO_HOME/conf/. This is a list of
+ Create a "masters" file in $ACCUMULO_CONF_DIR/. This is a list of
machines where the master server will run.
Create conf/accumulo-env.sh following the template of
@@ -202,7 +202,7 @@ server, but you can also glob them if you wish.
kadmin.local -q "xst -k accumulo.keytab -glob accumulo*"
-Place this file in $ACCUMULO_HOME/conf for every host. It should be owned by
+Place this file in $ACCUMULO_CONF_DIR for every host. It should be owned by
the accumulo user and chmodded to 400. Add the following to the accumulo-env.sh
In the accumulo-site.xml file on each node, add settings for general.kerberos.keytab
@@ -213,7 +213,7 @@ _HOST in lieu of your individual host names.
<property>
<name>general.kerberos.keytab</name>
- <value>$ACCUMULO_HOME/conf/accumulo.keytab</value>
+ <value>$ACCUMULO_CONF_DIR/accumulo.keytab</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd8ebfe8/bin/accumulo
----------------------------------------------------------------------
diff --git a/bin/accumulo b/bin/accumulo
index e1d269c..42074b6 100755
--- a/bin/accumulo
+++ b/bin/accumulo
@@ -15,12 +15,54 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+# The main Accumulo command script.
+#
+# Environment variables parameters:
+#
+# JAVA_HOME The java implenetation to use.
+# ACCUMULO_CONF_DIR Location where accumulo-env.sh, accumulo-site.xml and friends will be read from
+# ACCUMULO_LOG_DIR Directory for accmulo daemon logs. defaults to $ACCUMULO_HOME/logs
+# ACCUMULO_VERSION This Accumulo's version number
+# HADOOP_HOME The home diretcory for HADOOP. Defaults to bin/hadoop's parent.
+#
+# Java jvm paramters: (-Xmx et al)
+#
+# ACCUMULO_TSERVER_OPTS
+# ACCUMULO_MASTER_OPTS
+# ACCUMULO_GC_OPTS
+# ACCUMULO_MONITOR_OPTS
+# ACCUMULO_LOGGER_OPTS
+# ACCUMULO_GENERAL_OPTS
+# ACCUMULO_OTHER_OPTS
+#
+# Environment variables overriden by config.sh
+#
+# ACCUMULO_HOME Always the parent dir of bin/accumulo
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
. "$bin"/config.sh
+if [ -z "$JAVA_HOME" -o ! -d "$JAVA_HOME" ]; then
+ echo "JAVA_HOME is not set. Please make sure it's set globally or in conf/accumulo-env.sh"
+ exit 1
+fi
+if [ -z "$HADOOP_HOME" -o ! -d "$HADOOP_HOME" ]; then
+ echo "HADOOP_HOME is not set. Please make sure it's set globally or in conf/accumulo-env.sh"
+ exit 1
+fi
+if [ -z "$ZOOKEEPER_HOME" -o ! -d "$ZOOKEEPER_HOME" ]; then
+ echo "ZOOKEEPER_HOME is not set. Please make sure it's set globally or in conf/accumulo-env.sh"
+ exit 1
+fi
+
+#Export the variables just in case they are not exported
+#This makes them available to java
+export JAVA_HOME=$JAVA_HOME
+export HADOOP_HOME=$HADOOP_HOME
+export ZOOKEEPER_HOME=$ZOOKEEPER_HOME
+
START_JAR=$ACCUMULO_HOME/lib/accumulo-start-$ACCUMULO_VERSION.jar
COMMONS_JCI_JARS=$ACCUMULO_HOME/lib/commons-jci-core-1.0.jar:$ACCUMULO_HOME/lib/commons-jci-fam-1.0.jar:$ACCUMULO_HOME/lib/log4j-1.2.16.jar:$ACCUMULO_HOME/lib/commons-logging-1.0.4.jar:$ACCUMULO_HOME/lib/commons-logging-api-1.0.4.jar
START_CLASSES_DIR=$ACCUMULO_HOME/src/start/target/classes
@@ -81,28 +123,9 @@ else
export ACCUMULO_OPTS="${ACCUMULO_GENERAL_OPTS} ${ACCUMULO_OTHER_OPTS}"
fi
-XML_FILES=${ACCUMULO_HOME}/conf
+XML_FILES=${ACCUMULO_CONF_DIR}
CLASSPATH=${XML_FILES}:${START_JAR}:${COMMONS_JCI_JARS}
-if [ -z $JAVA_HOME -o ! -d $JAVA_HOME ]; then
- echo "JAVA_HOME is not set. Please make sure it's set globally or in conf/accumulo-env.sh"
- exit 1
-fi
-if [ -z $HADOOP_HOME -o ! -d $HADOOP_HOME ]; then
- echo "HADOOP_HOME is not set. Please make sure it's set globally or in conf/accumulo-env.sh"
- exit 1
-fi
-if [ -z $ZOOKEEPER_HOME -o ! -d $ZOOKEEPER_HOME ]; then
- echo "ZOOKEEPER_HOME is not set. Please make sure it's set globally or in conf/accumulo-env.sh"
- exit 1
-fi
-
-#Export the variables just in case they are not exported
-#This makes them available to java
-export JAVA_HOME=$JAVA_HOME
-export HADOOP_HOME=$HADOOP_HOME
-export ZOOKEEPER_HOME=$ZOOKEEPER_HOME
-
JAVA=$JAVA_HOME/bin/java
PLATFORM="`$JAVA -cp $CLASSPATH org.apache.accumulo.start.Platform`"
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd8ebfe8/bin/config.sh
----------------------------------------------------------------------
diff --git a/bin/config.sh b/bin/config.sh
index 6a88e61..a89115a 100755
--- a/bin/config.sh
+++ b/bin/config.sh
@@ -15,6 +15,20 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+# Guarantees that Accumulo and its environment variables are set.
+#
+# Values set by script that can be user provided. If not provided script attempts to infer.
+# ACCUMULO_CONF_DIR Location where accumulo-env.sh, accumulo-site.xml and friends will be read from
+# ACCUMULO_HOME Home directory for Accumulo
+# ACCUMULO_LOG_DIR Directory for Accumulo daemon logs
+# ACCUMULO_VERSION Accumulo version name
+# HADOOP_HOME Home dir for hadoop.
+#
+# Values always set by script.
+# GC Machine to rn GC daemon on. Used by start-here.sh script
+# MONITOR Machine to run monitor daemon on. Used by start-here.sh script
+# SSH Default ssh parameters used to start daemons
+
this="$0"
while [ -h "$this" ]; do
ls=`ls -ld "$this"`
@@ -33,8 +47,17 @@ this="$bin/$script"
ACCUMULO_HOME=`dirname "$this"`/..
export ACCUMULO_HOME=`cd $ACCUMULO_HOME; pwd`
-if [ -f $ACCUMULO_HOME/conf/accumulo-env.sh ] ; then
-. $ACCUMULO_HOME/conf/accumulo-env.sh
+ACCUMULO_CONF_DIR="${ACCUMULO_CONF_DIR:-$ACCUMULO_HOME/conf}"
+export ACCUMULO_CONF_DIR
+if [ -z "$ACCUMULO_CONF_DIR" -o ! -d "$ACCUMULO_CONF_DIR" ]
+then
+ echo "ACCUMULO_CONF_DIR=$ACCUMULO_CONF_DIR is not a valid directory. Please make sure it exists"
+ exit 1
+fi
+
+
+if [ -f $ACCUMULO_CONF_DIR/accumulo-env.sh ] ; then
+. $ACCUMULO_CONF_DIR/accumulo-env.sh
fi
if [ -z ${ACCUMULO_LOG_DIR} ]; then
@@ -67,17 +90,17 @@ then
fi
export HADOOP_HOME
-if [ ! -f "$ACCUMULO_HOME/conf/masters" -o ! -f "$ACCUMULO_HOME/conf/slaves" ]
+if [ ! -f "$ACCUMULO_CONF_DIR/masters" -o ! -f "$ACCUMULO_CONF_DIR/slaves" ]
then
- if [ ! -f "$ACCUMULO_HOME/conf/masters" -a ! -f "$ACCUMULO_HOME/conf/slaves" ]
+ if [ ! -f "$ACCUMULO_CONF_DIR/masters" -a ! -f "$ACCUMULO_CONF_DIR/slaves" ]
then
echo "STANDALONE: Missing both conf/masters and conf/slaves files"
echo "STANDALONE: Assuming single-node (localhost only) instance"
- echo "STANDALONE: echo "`hostname`" > $ACCUMULO_HOME/conf/masters"
- echo `hostname` > "$ACCUMULO_HOME/conf/masters"
- echo "STANDALONE: echo "`hostname`" > $ACCUMULO_HOME/conf/slaves"
- echo `hostname` > "$ACCUMULO_HOME/conf/slaves"
- fgrep -s logger.dir.walog "$ACCUMULO_HOME/conf/accumulo-site.xml" > /dev/null
+ echo "STANDALONE: echo "`hostname`" > $ACCUMULO_CONF_DIR/masters"
+ echo `hostname` > "$ACCUMULO_CONF_DIR/masters"
+ echo "STANDALONE: echo "`hostname`" > $ACCUMULO_CONF_DIR/slaves"
+ echo `hostname` > "$ACCUMULO_CONF_DIR/slaves"
+ fgrep -s logger.dir.walog "$ACCUMULO_CONF_DIR/accumulo-site.xml" > /dev/null
WALOG_CONFIGURED=$?
if [ $WALOG_CONFIGURED -ne 0 -a ! -e "$ACCUMULO_HOME/walogs" ]
then
@@ -85,28 +108,28 @@ then
mkdir "$ACCUMULO_HOME/walogs"
echo "STANDALONE: mkdir \"$ACCUMULO_HOME/walogs\""
fi
- if [ ! -e "$ACCUMULO_HOME/conf/accumulo-metrics.xml" ]
+ if [ ! -e "$ACCUMULO_CONF_DIR/accumulo-metrics.xml" ]
then
echo "STANDALONE: Creating default metrics configuration"
- cp "$ACCUMULO_HOME/conf/accumulo-metrics.xml.example" "$ACCUMULO_HOME/conf/accumulo-metrics.xml"
+ cp "$ACCUMULO_CONF_DIR/accumulo-metrics.xml.example" "$ACCUMULO_CONF_DIR/accumulo-metrics.xml"
fi
else
- echo "You are missing either $ACCUMULO_HOME/conf/masters or $ACCUMULO_HOME/conf/slaves"
+ echo "You are missing either $ACCUMULO_CONF_DIR/masters or $ACCUMULO_CONF_DIR/slaves"
echo "Please configure them both for a multi-node instance, or delete them both for a single-node (localhost only) instance"
exit 1
fi
fi
-MASTER1=`grep -v '^#' "$ACCUMULO_HOME/conf/masters" | head -1`
+MASTER1=`grep -v '^#' "$ACCUMULO_CONF_DIR/masters" | head -1`
GC=$MASTER1
MONITOR=$MASTER1
-if [ -f "$ACCUMULO_HOME/conf/gc" ]; then
- GC=`grep -v '^#' "$ACCUMULO_HOME/conf/gc" | head -1`
+if [ -f "$ACCUMULO_CONF_DIR/gc" ]; then
+ GC=`grep -v '^#' "$ACCUMULO_CONF_DIR/gc" | head -1`
fi
-if [ -f "$ACCUMULO_HOME/conf/monitor" ]; then
- MONITOR=`grep -v '^#' "$ACCUMULO_HOME/conf/monitor" | head -1`
+if [ -f "$ACCUMULO_CONF_DIR/monitor" ]; then
+ MONITOR=`grep -v '^#' "$ACCUMULO_CONF_DIR/monitor" | head -1`
fi
-if [ ! -f "$ACCUMULO_HOME/conf/tracers" ]; then
- echo "$MASTER1" > "$ACCUMULO_HOME/conf/tracers"
+if [ ! -f "$ACCUMULO_CONF_DIR/tracers" ]; then
+ echo "$MASTER1" > "$ACCUMULO_CONF_DIR/tracers"
fi
SSH='ssh -qnf -o ConnectTimeout=2'
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd8ebfe8/bin/start-all.sh
----------------------------------------------------------------------
diff --git a/bin/start-all.sh b/bin/start-all.sh
index 28cf6fc..5bee329 100755
--- a/bin/start-all.sh
+++ b/bin/start-all.sh
@@ -22,8 +22,8 @@ bin=`cd "$bin"; pwd`
. "$bin"/config.sh
unset DISPLAY
-if [ ! -f $ACCUMULO_HOME/conf/accumulo-env.sh ] ; then
- echo "${ACCUMULO_HOME}/conf/accumulo-env.sh does not exist. Please make sure you configure Accumulo before you run anything"
+if [ ! -f $ACCUMULO_CONF_DIR/accumulo-env.sh ] ; then
+ echo "${ACCUMULO_CONF_DIR}/accumulo-env.sh does not exist. Please make sure you configure Accumulo before you run anything"
echo "We provide examples you can copy in ${ACCUMULO_HOME}/conf/examples/ which are set up for your memory footprint"
exit 1
fi
@@ -47,7 +47,7 @@ if [ "$1" != "--notSlaves" ] ; then
fi
${bin}/accumulo org.apache.accumulo.server.master.state.SetGoalState NORMAL
-for master in `grep -v '^#' "$ACCUMULO_HOME/conf/masters"`
+for master in `grep -v '^#' "$ACCUMULO_CONF_DIR/masters"`
do
${bin}/start-server.sh $master master
done
@@ -56,7 +56,7 @@ ${bin}/start-server.sh $GC gc "garbage collector"
${bin}/start-server.sh $MONITOR monitor
-for tracer in `grep -v '^#' "$ACCUMULO_HOME/conf/tracers"`
+for tracer in `grep -v '^#' "$ACCUMULO_CONF_DIR/tracers"`
do
${bin}/start-server.sh $tracer tracer
done
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd8ebfe8/bin/start-here.sh
----------------------------------------------------------------------
diff --git a/bin/start-here.sh b/bin/start-here.sh
index b3d494e..22c8a04 100755
--- a/bin/start-here.sh
+++ b/bin/start-here.sh
@@ -27,7 +27,7 @@ bin=`cd "$bin"; pwd`
HOSTS="`hostname -a` `hostname` localhost"
for host in $HOSTS
do
- if grep -q "^${host}\$" $ACCUMULO_HOME/conf/slaves
+ if grep -q "^${host}\$" $ACCUMULO_CONF_DIR/slaves
then
${bin}/start-server.sh $host logger
${bin}/start-server.sh $host tserver "tablet server"
@@ -37,7 +37,7 @@ done
for host in $HOSTS
do
- if grep -q "^${host}\$" $ACCUMULO_HOME/conf/masters
+ if grep -q "^${host}\$" $ACCUMULO_CONF_DIR/masters
then
${bin}/accumulo org.apache.accumulo.server.master.state.SetGoalState NORMAL
${bin}/start-server.sh $host master
@@ -65,7 +65,7 @@ done
for host in $HOSTS
do
- if grep -q "^${host}\$" $ACCUMULO_HOME/conf/tracers
+ if grep -q "^${host}\$" $ACCUMULO_CONF_DIR/tracers
then
${bin}/start-server.sh $host tracer
break
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd8ebfe8/bin/start-server.sh
----------------------------------------------------------------------
diff --git a/bin/start-server.sh b/bin/start-server.sh
index 59cf1a3..7655319 100755
--- a/bin/start-server.sh
+++ b/bin/start-server.sh
@@ -35,7 +35,7 @@ if [ -z "$LONGNAME" ]
then
LONGNAME="$2"
fi
-SLAVES=`wc -l < ${ACCUMULO_HOME}/conf/slaves`
+SLAVES=`wc -l < ${ACCUMULO_CONF_DIR}/slaves`
IFCONFIG=/sbin/ifconfig
if [ ! -x $IFCONFIG ]
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd8ebfe8/bin/stop-all.sh
----------------------------------------------------------------------
diff --git a/bin/stop-all.sh b/bin/stop-all.sh
index f318397..a717e42 100755
--- a/bin/stop-all.sh
+++ b/bin/stop-all.sh
@@ -37,7 +37,7 @@ sleep 5
#look for master and gc processes not killed by 'admin stopAll'
for signal in TERM KILL
do
- for master in `grep -v '^#' "$ACCUMULO_HOME/conf/masters"`
+ for master in `grep -v '^#' "$ACCUMULO_CONF_DIR/masters"`
do
${bin}/stop-server.sh $master "$ACCUMULO_HOME/.*/accumulo-start.*.jar" master $signal
done
@@ -46,7 +46,7 @@ do
${bin}/stop-server.sh $MONITOR "$ACCUMULO_HOME/.*/accumulo-start.*.jar" monitor $signal
- for tracer in `grep -v '^#' "$ACCUMULO_HOME/conf/tracers"`
+ for tracer in `grep -v '^#' "$ACCUMULO_CONF_DIR/tracers"`
do
${bin}/stop-server.sh $tracer "$ACCUMULO_HOME/.*/accumulo-start.*.jar" tracer $signal
done
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd8ebfe8/bin/stop-here.sh
----------------------------------------------------------------------
diff --git a/bin/stop-here.sh b/bin/stop-here.sh
index 9532e22..f7fbad0 100755
--- a/bin/stop-here.sh
+++ b/bin/stop-here.sh
@@ -26,13 +26,13 @@ bin=`cd "$bin"; pwd`
ACCUMULO="$ACCUMULO_HOME/.*/accumulo-start.*.jar"
-if grep -q localhost $ACCUMULO_HOME/conf/slaves
+if grep -q localhost $ACCUMULO_CONF_DIR/slaves
then
$bin/accumulo admin stop localhost
else
for host in `hostname -a`
do
- if grep -q $host $ACCUMULO_HOME/conf/slaves
+ if grep -q $host $ACCUMULO_CONF_DIR/slaves
then
$bin/accumulo admin stop $host
fi
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd8ebfe8/bin/tdown.sh
----------------------------------------------------------------------
diff --git a/bin/tdown.sh b/bin/tdown.sh
index 1283799..12148f5 100755
--- a/bin/tdown.sh
+++ b/bin/tdown.sh
@@ -23,7 +23,7 @@ bin=`cd "$bin"; pwd`
HADOOP_CMD=$HADOOP_LOCATION/bin/hadoop
-SLAVES=$ACCUMULO_HOME/conf/slaves
+SLAVES=$ACCUMULO_CONF_DIR/slaves
echo 'stopping unresponsive tablet servers (if any) ...'
for server in `cat $SLAVES | grep -v '^#' `; do
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd8ebfe8/bin/tup.sh
----------------------------------------------------------------------
diff --git a/bin/tup.sh b/bin/tup.sh
index 53fd9a4..bf52feb 100755
--- a/bin/tup.sh
+++ b/bin/tup.sh
@@ -21,7 +21,7 @@ bin=`cd "$bin"; pwd`
. "$bin"/config.sh
-SLAVES=$ACCUMULO_HOME/conf/slaves
+SLAVES=$ACCUMULO_CONF_DIR/slaves
echo -n "Starting tablet servers and loggers ..."
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd8ebfe8/conf/examples/1GB/native-standalone/accumulo-env.sh
----------------------------------------------------------------------
diff --git a/conf/examples/1GB/native-standalone/accumulo-env.sh b/conf/examples/1GB/native-standalone/accumulo-env.sh
index 37eb2fc..5410cf9 100755
--- a/conf/examples/1GB/native-standalone/accumulo-env.sh
+++ b/conf/examples/1GB/native-standalone/accumulo-env.sh
@@ -32,9 +32,9 @@ test -z "$JAVA_HOME" && export JAVA_HOME=/path/to/java
test -z "$HADOOP_HOME" && export HADOOP_HOME=/path/to/hadoop
test -z "$ZOOKEEPER_HOME" && export ZOOKEEPER_HOME=/path/to/zookeeper
test -z "$ACCUMULO_LOG_DIR" && export ACCUMULO_LOG_DIR=$ACCUMULO_HOME/logs
-if [ -f ${ACCUMULO_HOME}/conf/accumulo.policy ]
+if [ -f ${ACCUMULO_CONF_DIR}/accumulo.policy ]
then
- POLICY="-Djava.security.manager -Djava.security.policy=${ACCUMULO_HOME}/conf/accumulo.policy"
+ POLICY="-Djava.security.manager -Djava.security.policy=${ACCUMULO_CONF_DIR}/accumulo.policy"
fi
test -z "$ACCUMULO_TSERVER_OPTS" && export ACCUMULO_TSERVER_OPTS="${POLICY} -Xmx128m -Xms128m "
test -z "$ACCUMULO_MASTER_OPTS" && export ACCUMULO_MASTER_OPTS="${POLICY} -Xmx128m -Xms128m"
@@ -43,4 +43,4 @@ test -z "$ACCUMULO_GC_OPTS" && export ACCUMULO_GC_OPTS="-Xmx64m -Xms64m"
test -z "$ACCUMULO_LOGGER_OPTS" && export ACCUMULO_LOGGER_OPTS="-Xmx384m -Xms256m"
test -z "$ACCUMULO_GENERAL_OPTS" && export ACCUMULO_GENERAL_OPTS="-XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75"
test -z "$ACCUMULO_OTHER_OPTS" && export ACCUMULO_OTHER_OPTS="-Xmx128m -Xms64m"
-export ACCUMULO_LOG_HOST=`(grep -v '^#' $ACCUMULO_HOME/conf/masters ; echo localhost ) 2>/dev/null | head -1`
+export ACCUMULO_LOG_HOST=`(grep -v '^#' $ACCUMULO_CONF_DIR/masters ; echo localhost ) 2>/dev/null | head -1`
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd8ebfe8/conf/examples/1GB/standalone/accumulo-env.sh
----------------------------------------------------------------------
diff --git a/conf/examples/1GB/standalone/accumulo-env.sh b/conf/examples/1GB/standalone/accumulo-env.sh
index f38e5b1..6214cf7 100755
--- a/conf/examples/1GB/standalone/accumulo-env.sh
+++ b/conf/examples/1GB/standalone/accumulo-env.sh
@@ -32,9 +32,9 @@ test -z "$JAVA_HOME" && export JAVA_HOME=/path/to/java
test -z "$HADOOP_HOME" && export HADOOP_HOME=/path/to/hadoop
test -z "$ZOOKEEPER_HOME" && export ZOOKEEPER_HOME=/path/to/zookeeper
test -z "$ACCUMULO_LOG_DIR" && export ACCUMULO_LOG_DIR=$ACCUMULO_HOME/logs
-if [ -f ${ACCUMULO_HOME}/conf/accumulo.policy ]
+if [ -f ${ACCUMULO_CONF_DIR}/accumulo.policy ]
then
- POLICY="-Djava.security.manager -Djava.security.policy=${ACCUMULO_HOME}/conf/accumulo.policy"
+ POLICY="-Djava.security.manager -Djava.security.policy=${ACCUMULO_CONF_DIR}/accumulo.policy"
fi
test -z "$ACCUMULO_TSERVER_OPTS" && export ACCUMULO_TSERVER_OPTS="${POLICY} -Xmx384m -Xms384m "
test -z "$ACCUMULO_MASTER_OPTS" && export ACCUMULO_MASTER_OPTS="${POLICY} -Xmx128m -Xms128m"
@@ -43,4 +43,4 @@ test -z "$ACCUMULO_GC_OPTS" && export ACCUMULO_GC_OPTS="-Xmx64m -Xms64m"
test -z "$ACCUMULO_LOGGER_OPTS" && export ACCUMULO_LOGGER_OPTS="-Xmx384m -Xms256m"
test -z "$ACCUMULO_GENERAL_OPTS" && export ACCUMULO_GENERAL_OPTS="-XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75"
test -z "$ACCUMULO_OTHER_OPTS" && export ACCUMULO_OTHER_OPTS="-Xmx128m -Xms64m"
-export ACCUMULO_LOG_HOST=`(grep -v '^#' $ACCUMULO_HOME/conf/masters ; echo localhost ) 2>/dev/null | head -1`
+export ACCUMULO_LOG_HOST=`(grep -v '^#' $ACCUMULO_CONF_DIR/masters ; echo localhost ) 2>/dev/null | head -1`
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd8ebfe8/conf/examples/2GB/native-standalone/accumulo-env.sh
----------------------------------------------------------------------
diff --git a/conf/examples/2GB/native-standalone/accumulo-env.sh b/conf/examples/2GB/native-standalone/accumulo-env.sh
index 1fd641f..77cc243 100755
--- a/conf/examples/2GB/native-standalone/accumulo-env.sh
+++ b/conf/examples/2GB/native-standalone/accumulo-env.sh
@@ -32,9 +32,9 @@ test -z "$JAVA_HOME" && export JAVA_HOME=/path/to/java
test -z "$HADOOP_HOME" && export HADOOP_HOME=/path/to/hadoop
test -z "$ZOOKEEPER_HOME" && export ZOOKEEPER_HOME=/path/to/zookeeper
test -z "$ACCUMULO_LOG_DIR" && export ACCUMULO_LOG_DIR=$ACCUMULO_HOME/logs
-if [ -f ${ACCUMULO_HOME}/conf/accumulo.policy ]
+if [ -f ${ACCUMULO_CONF_DIR}/accumulo.policy ]
then
- POLICY="-Djava.security.manager -Djava.security.policy=${ACCUMULO_HOME}/conf/accumulo.policy"
+ POLICY="-Djava.security.manager -Djava.security.policy=${ACCUMULO_CONF_DIR}/accumulo.policy"
fi
test -z "$ACCUMULO_TSERVER_OPTS" && export ACCUMULO_TSERVER_OPTS="${POLICY} -Xmx256m -Xms256m "
test -z "$ACCUMULO_MASTER_OPTS" && export ACCUMULO_MASTER_OPTS="${POLICY} -Xmx256m -Xms256m"
@@ -43,4 +43,4 @@ test -z "$ACCUMULO_GC_OPTS" && export ACCUMULO_GC_OPTS="-Xmx128m -Xms128m"
test -z "$ACCUMULO_LOGGER_OPTS" && export ACCUMULO_LOGGER_OPTS="-Xmx768m -Xms256m"
test -z "$ACCUMULO_GENERAL_OPTS" && export ACCUMULO_GENERAL_OPTS="-XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75"
test -z "$ACCUMULO_OTHER_OPTS" && export ACCUMULO_OTHER_OPTS="-Xmx256m -Xms64m"
-export ACCUMULO_LOG_HOST=`(grep -v '^#' $ACCUMULO_HOME/conf/masters ; echo localhost ) 2>/dev/null | head -1`
+export ACCUMULO_LOG_HOST=`(grep -v '^#' $ACCUMULO_CONF_DIR/masters ; echo localhost ) 2>/dev/null | head -1`
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd8ebfe8/conf/examples/2GB/standalone/accumulo-env.sh
----------------------------------------------------------------------
diff --git a/conf/examples/2GB/standalone/accumulo-env.sh b/conf/examples/2GB/standalone/accumulo-env.sh
index 290d81e..dee9937 100755
--- a/conf/examples/2GB/standalone/accumulo-env.sh
+++ b/conf/examples/2GB/standalone/accumulo-env.sh
@@ -32,9 +32,9 @@ test -z "$JAVA_HOME" && export JAVA_HOME=/path/to/java
test -z "$HADOOP_HOME" && export HADOOP_HOME=/path/to/hadoop
test -z "$ZOOKEEPER_HOME" && export ZOOKEEPER_HOME=/path/to/zookeeper
test -z "$ACCUMULO_LOG_DIR" && export ACCUMULO_LOG_DIR=$ACCUMULO_HOME/logs
-if [ -f ${ACCUMULO_HOME}/conf/accumulo.policy ]
+if [ -f ${ACCUMULO_CONF_DIR}/accumulo.policy ]
then
- POLICY="-Djava.security.manager -Djava.security.policy=${ACCUMULO_HOME}/conf/accumulo.policy"
+ POLICY="-Djava.security.manager -Djava.security.policy=${ACCUMULO_CONF_DIR}/accumulo.policy"
fi
test -z "$ACCUMULO_TSERVER_OPTS" && export ACCUMULO_TSERVER_OPTS="${POLICY} -Xmx768m -Xms768m "
test -z "$ACCUMULO_MASTER_OPTS" && export ACCUMULO_MASTER_OPTS="${POLICY} -Xmx256m -Xms256m"
@@ -43,4 +43,4 @@ test -z "$ACCUMULO_GC_OPTS" && export ACCUMULO_GC_OPTS="-Xmx128m -Xms128m"
test -z "$ACCUMULO_LOGGER_OPTS" && export ACCUMULO_LOGGER_OPTS="-Xmx768m -Xms256m"
test -z "$ACCUMULO_GENERAL_OPTS" && export ACCUMULO_GENERAL_OPTS="-XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75"
test -z "$ACCUMULO_OTHER_OPTS" && export ACCUMULO_OTHER_OPTS="-Xmx256m -Xms64m"
-export ACCUMULO_LOG_HOST=`(grep -v '^#' $ACCUMULO_HOME/conf/masters ; echo localhost ) 2>/dev/null | head -1`
+export ACCUMULO_LOG_HOST=`(grep -v '^#' $ACCUMULO_CONF_DIR/masters ; echo localhost ) 2>/dev/null | head -1`
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd8ebfe8/conf/examples/3GB/native-standalone/accumulo-env.sh
----------------------------------------------------------------------
diff --git a/conf/examples/3GB/native-standalone/accumulo-env.sh b/conf/examples/3GB/native-standalone/accumulo-env.sh
index c537161..3014554 100755
--- a/conf/examples/3GB/native-standalone/accumulo-env.sh
+++ b/conf/examples/3GB/native-standalone/accumulo-env.sh
@@ -32,9 +32,9 @@ test -z "$JAVA_HOME" && export JAVA_HOME=/path/to/java
test -z "$HADOOP_HOME" && export HADOOP_HOME=/path/to/hadoop
test -z "$ZOOKEEPER_HOME" && export ZOOKEEPER_HOME=/path/to/zookeeper
test -z "$ACCUMULO_LOG_DIR" && export ACCUMULO_LOG_DIR=$ACCUMULO_HOME/logs
-if [ -f ${ACCUMULO_HOME}/conf/accumulo.policy ]
+if [ -f ${ACCUMULO_CONF_DIR}/accumulo.policy ]
then
- POLICY="-Djava.security.manager -Djava.security.policy=${ACCUMULO_HOME}/conf/accumulo.policy"
+ POLICY="-Djava.security.manager -Djava.security.policy=${ACCUMULO_CONF_DIR}/accumulo.policy"
fi
test -z "$ACCUMULO_TSERVER_OPTS" && export ACCUMULO_TSERVER_OPTS="${POLICY} -Xmx1g -Xms1g -XX:NewSize=500m -XX:MaxNewSize=500m "
test -z "$ACCUMULO_MASTER_OPTS" && export ACCUMULO_MASTER_OPTS="${POLICY} -Xmx1g -Xms1g"
@@ -43,4 +43,4 @@ test -z "$ACCUMULO_GC_OPTS" && export ACCUMULO_GC_OPTS="-Xmx256m -Xms256m"
test -z "$ACCUMULO_LOGGER_OPTS" && export ACCUMULO_LOGGER_OPTS="-Xmx1g -Xms256m"
test -z "$ACCUMULO_GENERAL_OPTS" && export ACCUMULO_GENERAL_OPTS="-XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75"
test -z "$ACCUMULO_OTHER_OPTS" && export ACCUMULO_OTHER_OPTS="-Xmx1g -Xms256m"
-export ACCUMULO_LOG_HOST=`(grep -v '^#' $ACCUMULO_HOME/conf/masters ; echo localhost ) 2>/dev/null | head -1`
+export ACCUMULO_LOG_HOST=`(grep -v '^#' $ACCUMULO_CONF_DIR/masters ; echo localhost ) 2>/dev/null | head -1`
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd8ebfe8/conf/examples/3GB/standalone/accumulo-env.sh
----------------------------------------------------------------------
diff --git a/conf/examples/3GB/standalone/accumulo-env.sh b/conf/examples/3GB/standalone/accumulo-env.sh
index eaeed4e..23080d3 100755
--- a/conf/examples/3GB/standalone/accumulo-env.sh
+++ b/conf/examples/3GB/standalone/accumulo-env.sh
@@ -32,9 +32,9 @@ test -z "$JAVA_HOME" && export JAVA_HOME=/path/to/java
test -z "$HADOOP_HOME" && export HADOOP_HOME=/path/to/hadoop
test -z "$ZOOKEEPER_HOME" && export ZOOKEEPER_HOME=/path/to/zookeeper
test -z "$ACCUMULO_LOG_DIR" && export ACCUMULO_LOG_DIR=$ACCUMULO_HOME/logs
-if [ -f ${ACCUMULO_HOME}/conf/accumulo.policy ]
+if [ -f ${ACCUMULO_CONF_DIR}/accumulo.policy ]
then
- POLICY="-Djava.security.manager -Djava.security.policy=${ACCUMULO_HOME}/conf/accumulo.policy"
+ POLICY="-Djava.security.manager -Djava.security.policy=${ACCUMULO_CONF_DIR}/accumulo.policy"
fi
test -z "$ACCUMULO_TSERVER_OPTS" && export ACCUMULO_TSERVER_OPTS="${POLICY} -Xmx2g -Xms2g -XX:NewSize=1G -XX:MaxNewSize=1G "
test -z "$ACCUMULO_MASTER_OPTS" && export ACCUMULO_MASTER_OPTS="${POLICY} -Xmx1g -Xms1g"
@@ -43,4 +43,4 @@ test -z "$ACCUMULO_GC_OPTS" && export ACCUMULO_GC_OPTS="-Xmx256m -Xms256m"
test -z "$ACCUMULO_LOGGER_OPTS" && export ACCUMULO_LOGGER_OPTS="-Xmx1g -Xms256m"
test -z "$ACCUMULO_GENERAL_OPTS" && export ACCUMULO_GENERAL_OPTS="-XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75"
test -z "$ACCUMULO_OTHER_OPTS" && export ACCUMULO_OTHER_OPTS="-Xmx1g -Xms256m"
-export ACCUMULO_LOG_HOST=`(grep -v '^#' $ACCUMULO_HOME/conf/masters ; echo localhost ) 2>/dev/null | head -1`
+export ACCUMULO_LOG_HOST=`(grep -v '^#' $ACCUMULO_CONF_DIR/masters ; echo localhost ) 2>/dev/null | head -1`
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd8ebfe8/conf/examples/512MB/native-standalone/accumulo-env.sh
----------------------------------------------------------------------
diff --git a/conf/examples/512MB/native-standalone/accumulo-env.sh b/conf/examples/512MB/native-standalone/accumulo-env.sh
index 0a59dda..49ec64d 100755
--- a/conf/examples/512MB/native-standalone/accumulo-env.sh
+++ b/conf/examples/512MB/native-standalone/accumulo-env.sh
@@ -32,9 +32,9 @@ test -z "$JAVA_HOME" && export JAVA_HOME=/path/to/java
test -z "$HADOOP_HOME" && export HADOOP_HOME=/path/to/hadoop
test -z "$ZOOKEEPER_HOME" && export ZOOKEEPER_HOME=/path/to/zookeeper
test -z "$ACCUMULO_LOG_DIR" && export ACCUMULO_LOG_DIR=$ACCUMULO_HOME/logs
-if [ -f ${ACCUMULO_HOME}/conf/accumulo.policy ]
+if [ -f ${ACCUMULO_CONF_DIR}/accumulo.policy ]
then
- POLICY="-Djava.security.manager -Djava.security.policy=${ACCUMULO_HOME}/conf/accumulo.policy"
+ POLICY="-Djava.security.manager -Djava.security.policy=${ACCUMULO_CONF_DIR}/accumulo.policy"
fi
test -z "$ACCUMULO_TSERVER_OPTS" && export ACCUMULO_TSERVER_OPTS="${POLICY} -Xmx48m -Xms48m "
test -z "$ACCUMULO_MASTER_OPTS" && export ACCUMULO_MASTER_OPTS="${POLICY} -Xmx128m -Xms128m"
@@ -43,4 +43,4 @@ test -z "$ACCUMULO_GC_OPTS" && export ACCUMULO_GC_OPTS="-Xmx64m -Xms64m"
test -z "$ACCUMULO_LOGGER_OPTS" && export ACCUMULO_LOGGER_OPTS="-Xmx128m -Xms128m"
test -z "$ACCUMULO_GENERAL_OPTS" && export ACCUMULO_GENERAL_OPTS="-XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75"
test -z "$ACCUMULO_OTHER_OPTS" && export ACCUMULO_OTHER_OPTS="-Xmx128m -Xms64m"
-export ACCUMULO_LOG_HOST=`(grep -v '^#' $ACCUMULO_HOME/conf/masters ; echo localhost ) 2>/dev/null | head -1`
+export ACCUMULO_LOG_HOST=`(grep -v '^#' $ACCUMULO_CONF_DIR/masters ; echo localhost ) 2>/dev/null | head -1`
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd8ebfe8/conf/examples/512MB/standalone/accumulo-env.sh
----------------------------------------------------------------------
diff --git a/conf/examples/512MB/standalone/accumulo-env.sh b/conf/examples/512MB/standalone/accumulo-env.sh
index e01f088..e0ae73e 100755
--- a/conf/examples/512MB/standalone/accumulo-env.sh
+++ b/conf/examples/512MB/standalone/accumulo-env.sh
@@ -32,9 +32,9 @@ test -z "$JAVA_HOME" && export JAVA_HOME=/path/to/java
test -z "$HADOOP_HOME" && export HADOOP_HOME=/path/to/hadoop
test -z "$ZOOKEEPER_HOME" && export ZOOKEEPER_HOME=/path/to/zookeeper
test -z "$ACCUMULO_LOG_DIR" && export ACCUMULO_LOG_DIR=$ACCUMULO_HOME/logs
-if [ -f ${ACCUMULO_HOME}/conf/accumulo.policy ]
+if [ -f ${ACCUMULO_CONF_DIR}/accumulo.policy ]
then
- POLICY="-Djava.security.manager -Djava.security.policy=${ACCUMULO_HOME}/conf/accumulo.policy"
+ POLICY="-Djava.security.manager -Djava.security.policy=${ACCUMULO_CONF_DIR}/accumulo.policy"
fi
test -z "$ACCUMULO_TSERVER_OPTS" && export ACCUMULO_TSERVER_OPTS="${POLICY} -Xmx128m -Xms128m "
test -z "$ACCUMULO_MASTER_OPTS" && export ACCUMULO_MASTER_OPTS="${POLICY} -Xmx128m -Xms128m"
@@ -43,4 +43,4 @@ test -z "$ACCUMULO_GC_OPTS" && export ACCUMULO_GC_OPTS="-Xmx64m -Xms64m"
test -z "$ACCUMULO_LOGGER_OPTS" && export ACCUMULO_LOGGER_OPTS="-Xmx128m -Xms128m"
test -z "$ACCUMULO_GENERAL_OPTS" && export ACCUMULO_GENERAL_OPTS="-XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75"
test -z "$ACCUMULO_OTHER_OPTS" && export ACCUMULO_OTHER_OPTS="-Xmx128m -Xms64m"
-export ACCUMULO_LOG_HOST=`(grep -v '^#' $ACCUMULO_HOME/conf/masters ; echo localhost ) 2>/dev/null | head -1`
+export ACCUMULO_LOG_HOST=`(grep -v '^#' $ACCUMULO_CONF_DIR/masters ; echo localhost ) 2>/dev/null | head -1`
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd8ebfe8/docs/config.html
----------------------------------------------------------------------
diff --git a/docs/config.html b/docs/config.html
index 70f4855..97c2c6b 100644
--- a/docs/config.html
+++ b/docs/config.html
@@ -136,7 +136,7 @@
<td>general.classpaths</td>
<td><b><a href='#STRING'>string</a></b></td>
<td>no</td>
- <td><pre>$ACCUMULO_HOME/conf,
+ <td><pre>$ACCUMULO_CONF_DIR,
$ACCUMULO_HOME/lib/[^.].$ACCUMULO_VERSION.jar,
$ACCUMULO_HOME/lib/[^.].*.jar,
$ZOOKEEPER_HOME/zookeeper[^.].*.jar,
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd8ebfe8/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java b/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java
index 60f4788..b2feb5c 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java
@@ -117,10 +117,10 @@ public class Accumulo {
System.setProperty("org.apache.accumulo.core.host.log", localhost);
// Use a specific log config, if it exists
- String logConfig = String.format("%s/conf/%s_logger.xml", System.getenv("ACCUMULO_HOME"), application);
+ String logConfig = String.format("%s/%s_logger.xml", System.getenv("ACCUMULO_CONF_DIR"), application);
if (!new File(logConfig).exists()) {
// otherwise, use the generic config
- logConfig = String.format("%s/conf/generic_logger.xml", System.getenv("ACCUMULO_HOME"));
+ logConfig = String.format("%s/generic_logger.xml", System.getenv("ACCUMULO_CONF_DIR"));
}
// Turn off messages about not being able to reach the remote logger... we protect against that.
LogLog.setQuietMode(true);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd8ebfe8/src/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java
----------------------------------------------------------------------
diff --git a/src/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java b/src/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java
index 0e5597f..8d217d7 100644
--- a/src/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java
+++ b/src/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java
@@ -106,7 +106,9 @@ public class AccumuloClassLoader {
public static final String DYNAMIC_CLASSPATH_PROPERTY_NAME = "general.dynamic.classpaths";
- public static final String ACCUMULO_CLASSPATH_VALUE = "$ACCUMULO_HOME/conf,\n" + "$ACCUMULO_HOME/lib/[^.].$ACCUMULO_VERSION.jar,\n"
+ public static final String ACCUMULO_CLASSPATH_VALUE =
+ "$ACCUMULO_CONF_DIR,\n"
+ + "$ACCUMULO_HOME/lib/[^.].$ACCUMULO_VERSION.jar,\n"
+ "$ACCUMULO_HOME/lib/[^.].*.jar,\n" + "$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n" + "$HADOOP_HOME/[^.].*.jar,\n" + "$HADOOP_HOME/conf,\n"
+ "$HADOOP_HOME/lib/[^.].*.jar,\n";
@@ -120,8 +122,11 @@ public class AccumuloClassLoader {
private static final String SITE_CONF;
static {
String configFile = System.getProperty("org.apache.accumulo.config.file", "accumulo-site.xml");
- if (System.getenv("ACCUMULO_HOME") != null) {
- // accumulo home should be set
+ if (System.getenv("ACCUMULO_CONF_DIR") != null) {
+ // accumulo conf dir should be set.
+ SITE_CONF = System.getenv("ACCUMULO_CONF_DIR") + "/" + configFile;
+ } else if (System.getenv("ACCUMULO_HOME") != null) {
+ // if no accumulo conf dir, try accumulo home default
SITE_CONF = System.getenv("ACCUMULO_HOME") + "/conf/" + configFile;
} else {
/*
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd8ebfe8/src/start/src/test/java/org/apache/accumulo/start/Test.java
----------------------------------------------------------------------
diff --git a/src/start/src/test/java/org/apache/accumulo/start/Test.java b/src/start/src/test/java/org/apache/accumulo/start/Test.java
index 376d563..7da905c 100644
--- a/src/start/src/test/java/org/apache/accumulo/start/Test.java
+++ b/src/start/src/test/java/org/apache/accumulo/start/Test.java
@@ -181,6 +181,7 @@ public class Test extends TestCase {
public void testChangingDirectory() throws Exception {
String configFile = System.getProperty("org.apache.accumulo.config.file", "accumulo-site.xml");
+ // This does not use ACCUMULO_CONF_DIR since it is used to get to the conf/examples dirs
String CONF_DIR = System.getenv("ACCUMULO_HOME") + "/conf/";
String SITE_CONF = CONF_DIR + configFile;
File oldConf = new File(SITE_CONF);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd8ebfe8/test/system/auto/TestUtils.py
----------------------------------------------------------------------
diff --git a/test/system/auto/TestUtils.py b/test/system/auto/TestUtils.py
index 42c137b..4b5efdf 100755
--- a/test/system/auto/TestUtils.py
+++ b/test/system/auto/TestUtils.py
@@ -48,7 +48,7 @@ WALOG = os.path.join(ACCUMULO_HOME, 'walogs', ID)
LOG_PROPERTIES= os.path.join(ACCUMULO_HOME, 'conf', 'log4j.properties')
LOG_GENERIC = os.path.join(ACCUMULO_HOME, 'conf', 'generic_logger.xml')
LOG_MONITOR = os.path.join(ACCUMULO_HOME, 'conf', 'monitor_logger.xml')
-General_CLASSPATH = ("$ACCUMULO_HOME/lib/[^.].$ACCUMULO_VERSION.jar, $ACCUMULO_HOME/lib/[^.].*.jar, $ZOOKEEPER_HOME/zookeeper[^.].*.jar,"
+General_CLASSPATH = ("$ACCUMULO_CONF_DIR,$ACCUMULO_HOME/lib/[^.].$ACCUMULO_VERSION.jar, $ACCUMULO_HOME/lib/[^.].*.jar, $ZOOKEEPER_HOME/zookeeper[^.].*.jar,"
"$HADOOP_HOME/conf,$HADOOP_HOME/[^.].*.jar, $HADOOP_HOME/lib/[^.].*.jar")
log = logging.getLogger('test.auto')
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd8ebfe8/test/system/continuous/agitator.pl
----------------------------------------------------------------------
diff --git a/test/system/continuous/agitator.pl b/test/system/continuous/agitator.pl
index be3a0c6..c074665 100755
--- a/test/system/continuous/agitator.pl
+++ b/test/system/continuous/agitator.pl
@@ -40,7 +40,7 @@ if($minKill > $maxKill){
die("minKill > maxKill $minKill > $maxKill");
}
-@slavesRaw = `cat $ACCUMULO_HOME/conf/slaves`;
+@slavesRaw = `cat $ACCUMULO_CONF_DIR/slaves`;
chomp(@slavesRaw);
for $slave (@slavesRaw){
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd8ebfe8/test/system/continuous/magitator.pl
----------------------------------------------------------------------
diff --git a/test/system/continuous/magitator.pl b/test/system/continuous/magitator.pl
index 8b22505..a0cab60 100755
--- a/test/system/continuous/magitator.pl
+++ b/test/system/continuous/magitator.pl
@@ -29,7 +29,7 @@ $ACCUMULO_HOME="../../..";
$sleep1 = $ARGV[0];
$sleep2 = $ARGV[1];
-@mastersRaw = `cat $ACCUMULO_HOME/conf/masters`;
+@mastersRaw = `cat $ACCUMULO_CONF_DIR/masters`;
chomp(@mastersRaw);
for $master (@mastersRaw){
@@ -52,10 +52,10 @@ while(1){
system($cmd);
}else{
print STDERR "$t Killing all masters\n";
- $cmd = "pssh -h $ACCUMULO_HOME/conf/masters \"pkill -f '[ ]org.apache.accumulo.start.*master'\" < /dev/null";
+ $cmd = "pssh -h $ACCUMULO_CONF_DIR/masters \"pkill -f '[ ]org.apache.accumulo.start.*master'\" < /dev/null";
print "$t $cmd\n";
system($cmd);
- $cmd = "pssh -h $ACCUMULO_HOME/conf/masters \"pkill -f '[ ]org.apache.accumulo.start.*gc'\" < /dev/null";
+ $cmd = "pssh -h $ACCUMULO_CONF_DIR/masters \"pkill -f '[ ]org.apache.accumulo.start.*gc'\" < /dev/null";
print "$t $cmd\n";
system($cmd);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd8ebfe8/test/system/continuous/mapred-setup.sh
----------------------------------------------------------------------
diff --git a/test/system/continuous/mapred-setup.sh b/test/system/continuous/mapred-setup.sh
index b714137..3e5b227 100755
--- a/test/system/continuous/mapred-setup.sh
+++ b/test/system/continuous/mapred-setup.sh
@@ -19,7 +19,7 @@
# for running a map reduce job
. continuous-env.sh
-. $ACCUMULO_HOME/conf/accumulo-env.sh
+. $ACCUMULO_CONF_DIR/accumulo-env.sh
SERVER_CMD='ls -1 $ACCUMULO_HOME/lib/accumulo-server-*[!javadoc\|sources].jar'
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd8ebfe8/test/system/continuous/start-stats.sh
----------------------------------------------------------------------
diff --git a/test/system/continuous/start-stats.sh b/test/system/continuous/start-stats.sh
index ae757bc..04b16a2 100755
--- a/test/system/continuous/start-stats.sh
+++ b/test/system/continuous/start-stats.sh
@@ -22,7 +22,7 @@ mkdir -p $CONTINUOUS_LOG_DIR
CONFIG_OUT=$CONTINUOUS_LOG_DIR/`date +%Y%m%d%H%M%S`_`hostname`_config.out
-cat $ACCUMULO_HOME/conf/accumulo-env.sh > $CONFIG_OUT
+cat $ACCUMULO_CONF_DIR/accumulo-env.sh > $CONFIG_OUT
echo >> $CONFIG_OUT
echo -e "config -np\nconfig -t $TABLE -np\nquit" | $ACCUMULO_HOME/bin/accumulo shell -u $USER -p $PASS >> $CONFIG_OUT
echo >> $CONFIG_OUT
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd8ebfe8/test/system/randomwalk/README
----------------------------------------------------------------------
diff --git a/test/system/randomwalk/README b/test/system/randomwalk/README
index 0c33310..78a50c6 100644
--- a/test/system/randomwalk/README
+++ b/test/system/randomwalk/README
@@ -4,7 +4,7 @@ The randomwalk framework needs to be configured for your Accumulo instance by
doing the following steps:
1. Make sure you have both ACCUMULO_HOME and HADOOP_HOME set in your
- $ACCUMULO_HOME/conf/accumulo-env.sh.
+ $ACCUMULO_CONF_DIR/accumulo-env.sh.
2. Create 'randomwalk.conf' file in the conf directory containing settings
needed by walkers to connect to Accumulo.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd8ebfe8/test/system/randomwalk/bin/reset-cluster.sh
----------------------------------------------------------------------
diff --git a/test/system/randomwalk/bin/reset-cluster.sh b/test/system/randomwalk/bin/reset-cluster.sh
index 3ffea0c..83dfcee 100755
--- a/test/system/randomwalk/bin/reset-cluster.sh
+++ b/test/system/randomwalk/bin/reset-cluster.sh
@@ -34,8 +34,8 @@ this="$bin/$script"
ACCUMULO_HOME=`dirname "$this"`/../../../..
export ACCUMULO_HOME=`cd $ACCUMULO_HOME; pwd`
-if [ -f $ACCUMULO_HOME/conf/accumulo-env.sh ] ; then
-. $ACCUMULO_HOME/conf/accumulo-env.sh
+if [ -f $ACCUMULO_CONF_DIR/accumulo-env.sh ] ; then
+. $ACCUMULO_CONF_DIR/accumulo-env.sh
fi
if [ -z $HADOOP_HOME ] ; then
@@ -49,7 +49,7 @@ if [ "$1" = "" ] ; then
fi
echo 'killing accumulo'
-pssh -h $ACCUMULO_HOME/conf/slaves "pkill -f org.apache.accumulo.start" < /dev/null
+pssh -h $ACCUMULO_CONF_DIR/slaves "pkill -f org.apache.accumulo.start" < /dev/null
pkill -f org.apache.accumulo.start
pkill -f agitator.pl
@@ -64,10 +64,10 @@ rm -f $ACCUMULO_HOME/test/system/continuous/logs/*
rm -f ~/rwlogs/*
echo 'removing old code'
-pssh -h $ACCUMULO_HOME/conf/slaves "rm -rf $ACCUMULO_HOME" < /dev/null
+pssh -h $ACCUMULO_CONF_DIR/slaves "rm -rf $ACCUMULO_HOME" < /dev/null
echo 'pushing new code'
-prsync -r -h $ACCUMULO_HOME/conf/slaves $ACCUMULO_HOME /opt/dev
+prsync -r -h $ACCUMULO_CONF_DIR/slaves $ACCUMULO_HOME /opt/dev
echo 'removing /accumulo dir'
$HADOOP_HOME/bin/hadoop fs -rmr /accumulo
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd8ebfe8/test/system/randomwalk/bin/start-all.sh
----------------------------------------------------------------------
diff --git a/test/system/randomwalk/bin/start-all.sh b/test/system/randomwalk/bin/start-all.sh
index 660c87e..67296de 100755
--- a/test/system/randomwalk/bin/start-all.sh
+++ b/test/system/randomwalk/bin/start-all.sh
@@ -34,8 +34,8 @@ this="$bin/$script"
ACCUMULO_HOME=`dirname "$this"`/../../../..
export ACCUMULO_HOME=`cd $ACCUMULO_HOME; pwd`
-if [ -f $ACCUMULO_HOME/conf/accumulo-env.sh ] ; then
-. $ACCUMULO_HOME/conf/accumulo-env.sh
+if [ -f $ACCUMULO_CONF_DIR/accumulo-env.sh ] ; then
+. $ACCUMULO_CONF_DIR/accumulo-env.sh
fi
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd8ebfe8/test/system/randomwalk/bin/start-local.sh
----------------------------------------------------------------------
diff --git a/test/system/randomwalk/bin/start-local.sh b/test/system/randomwalk/bin/start-local.sh
index a27523e..c02b942 100755
--- a/test/system/randomwalk/bin/start-local.sh
+++ b/test/system/randomwalk/bin/start-local.sh
@@ -35,8 +35,8 @@ this="$bin/$script"
ACCUMULO_HOME=`dirname "$this"`/../../../..
export ACCUMULO_HOME=`cd $ACCUMULO_HOME; pwd`
-if [ -f $ACCUMULO_HOME/conf/accumulo-env.sh ] ; then
-. $ACCUMULO_HOME/conf/accumulo-env.sh
+if [ -f $ACCUMULO_CONF_DIR/accumulo-env.sh ] ; then
+. $ACCUMULO_CONF_DIR/accumulo-env.sh
fi
if [ -z "$HADOOP_HOME" ] ; then
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd8ebfe8/test/system/scalability/run.py
----------------------------------------------------------------------
diff --git a/test/system/scalability/run.py b/test/system/scalability/run.py
index e74fbf9..d0f9f51 100755
--- a/test/system/scalability/run.py
+++ b/test/system/scalability/run.py
@@ -57,7 +57,7 @@ def runTest(testName, siteConfig, testDir, numNodes, fdata):
syscall('head -n %d %s > %s' % (numNodes,slavesPath,nodesPath))
log('Copying slaves file to accumulo config')
- syscall('cp '+nodesPath+' $ACCUMULO_HOME/conf/slaves');
+ syscall('cp '+nodesPath+' $ACCUMULO_CONF_DIR/slaves');
log('Removing /accumulo directory in HDFS')
syscall("hadoop fs -rmr /accumulo")
[32/50] [abbrv] git commit: ACCUMULO-1267 Added check to see if a
user actually sets a regex before applying the RegExFilter to the record
reader's scanner.
Posted by el...@apache.org.
ACCUMULO-1267 Added check to see if a user actually sets a regex before applying the RegExFilter to the record reader's scanner.
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.4@1499198 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/1725ec3a
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/1725ec3a
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/1725ec3a
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 1725ec3a6f8a7f56fb167e7b24686b1848520aa9
Parents: 4fd1066
Author: Bill Slacum <uj...@apache.org>
Authored: Wed Jul 3 04:04:20 2013 +0000
Committer: Bill Slacum <uj...@apache.org>
Committed: Wed Jul 3 04:04:20 2013 +0000
----------------------------------------------------------------------
.../core/client/mapreduce/InputFormatBase.java | 10 ++++--
.../mapreduce/AccumuloInputFormatTest.java | 33 ++++++++++++++++++++
2 files changed, 40 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/1725ec3a/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java b/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
index 174df66..8e238f1 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
@@ -1131,9 +1131,13 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
scanner = new ClientSideIteratorScanner(scanner);
}
setupMaxVersions(conf, scanner);
- IteratorSetting is = new IteratorSetting(50, RegExFilter.class);
- RegExFilter.setRegexs(is, conf.get(ROW_REGEX), conf.get(COLUMN_FAMILY_REGEX), conf.get(COLUMN_QUALIFIER_REGEX), null, false);
- scanner.addScanIterator(is);
+ if (conf.get(ROW_REGEX) != null || conf.get(COLUMN_FAMILY_REGEX) != null || conf.get(COLUMN_QUALIFIER_REGEX) != null ||
+ conf.get(VALUE_REGEX) != null) {
+ IteratorSetting is = new IteratorSetting(50, RegExFilter.class);
+ RegExFilter.setRegexs(is, conf.get(ROW_REGEX), conf.get(COLUMN_FAMILY_REGEX), conf.get(COLUMN_QUALIFIER_REGEX),
+ conf.get(VALUE_REGEX), false);
+ scanner.addScanIterator(is);
+ }
setupIterators(conf, scanner);
} catch (Exception e) {
throw new IOException(e);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/1725ec3a/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
----------------------------------------------------------------------
diff --git a/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java b/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
index d6df099..ba647e9 100644
--- a/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
+++ b/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
@@ -21,6 +21,7 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.List;
+import java.util.regex.Pattern;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.Connector;
@@ -46,6 +47,7 @@ import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Test;
public class AccumuloInputFormatTest {
@@ -353,4 +355,35 @@ public class AccumuloInputFormatTest {
mapper.map(rr.getCurrentKey(), rr.getCurrentValue(), context);
}
}
+
+ @SuppressWarnings("deprecation")
+ @Test
+ public void testRegex() throws Exception {
+ MockInstance mockInstance = new MockInstance("testmapinstance");
+ Connector c = mockInstance.getConnector("root", new byte[] {});
+ c.tableOperations().create("testtable3");
+ BatchWriter bw = c.createBatchWriter("testtable3", 10000L, 1000L, 4);
+ for (int i = 0; i < 100; i++) {
+ Mutation m = new Mutation(new Text(String.format("%09x", i + 1)));
+ m.put(new Text(), new Text(), new Value(String.format("%09x", i).getBytes()));
+ bw.addMutation(m);
+ }
+ bw.close();
+
+ JobContext job = new JobContext(new Configuration(), new JobID());
+ AccumuloInputFormat.setInputInfo(job.getConfiguration(), "root", "".getBytes(), "testtable3", new Authorizations());
+ AccumuloInputFormat.setMockInstance(job.getConfiguration(), "testmapinstance");
+ final String regex = ".*1.*";
+ AccumuloInputFormat.setRegex(job, RegexType.ROW, regex);
+ AccumuloInputFormat input = new AccumuloInputFormat();
+ RangeInputSplit ris = new RangeInputSplit();
+ TaskAttemptContext tac = new TaskAttemptContext(job.getConfiguration(), new TaskAttemptID());
+ RecordReader<Key,Value> rr = input.createRecordReader(ris, tac);
+ rr.initialize(ris, tac);
+
+ Pattern p = Pattern.compile(regex);
+ while (rr.nextKeyValue()) {
+ Assert.assertTrue( p.matcher( rr.getCurrentKey().getRow().toString()).matches());
+ }
+ }
}
[11/50] [abbrv] git commit: ACCUMULO-1380 mock root user's password
is empty
Posted by el...@apache.org.
ACCUMULO-1380 mock root user's password is empty
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.4@1482370 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/854b91f9
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/854b91f9
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/854b91f9
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 854b91f996f4161788357a4f09e7c51de5883b44
Parents: 452bffc
Author: Keith Turner <kt...@apache.org>
Authored: Tue May 14 15:01:51 2013 +0000
Committer: Keith Turner <kt...@apache.org>
Committed: Tue May 14 15:01:51 2013 +0000
----------------------------------------------------------------------
docs/src/user_manual/chapters/development_clients.tex | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/854b91f9/docs/src/user_manual/chapters/development_clients.tex
----------------------------------------------------------------------
diff --git a/docs/src/user_manual/chapters/development_clients.tex b/docs/src/user_manual/chapters/development_clients.tex
index 7482a23..f79172a 100644
--- a/docs/src/user_manual/chapters/development_clients.tex
+++ b/docs/src/user_manual/chapters/development_clients.tex
@@ -50,7 +50,7 @@ MockAccumulo:
\small
\begin{verbatim}
-$ ./bin/accumulo shell --fake -u root -p nonsense
+$ ./bin/accumulo shell --fake -u root -p ''
Shell - Apache Accumulo Interactive Shell
-
[33/50] [abbrv] git commit: ACCUMULO-895 Applying Nguessan Kouame's
patch, modulo movement of the minicluster source.
Posted by el...@apache.org.
ACCUMULO-895 Applying Nguessan Kouame's patch, modulo movement of the minicluster source.
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.4@1500320 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d5da59ac
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d5da59ac
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d5da59ac
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: d5da59ac8c35a846374537b0ef35226c07428fea
Parents: 1725ec3
Author: Mike Drob <md...@apache.org>
Authored: Sat Jul 6 20:29:56 2013 +0000
Committer: Mike Drob <md...@apache.org>
Committed: Sat Jul 6 20:29:56 2013 +0000
----------------------------------------------------------------------
.../main/java/org/apache/accumulo/core/conf/PropertyType.java | 2 +-
.../org/apache/accumulo/minicluster/MiniAccumuloCluster.java | 5 +++--
.../main/java/org/apache/accumulo/server/logger/LogService.java | 4 ++--
3 files changed, 6 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d5da59ac/src/core/src/main/java/org/apache/accumulo/core/conf/PropertyType.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/conf/PropertyType.java b/src/core/src/main/java/org/apache/accumulo/core/conf/PropertyType.java
index 67bd6c4..9e64548 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/conf/PropertyType.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/conf/PropertyType.java
@@ -49,7 +49,7 @@ public enum PropertyType {
PATH("path", ".*",
"A string that represents a filesystem path, which can be either relative or absolute to some directory. The filesystem depends on the property."),
- ABSOLUTEPATH("absolute path", "[/].*",
+ ABSOLUTEPATH("absolute path", "([/]|[A-Z]:[\\\\]).*",
"An absolute filesystem path. The filesystem depends on the property. This is the same as path, but enforces that its root is explicitly specified."),
CLASSNAME("java class", "[\\w$.]*", "A fully qualified java class name representing a class on the classpath.<br />"
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d5da59ac/src/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
----------------------------------------------------------------------
diff --git a/src/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java b/src/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
index 2335560..3a4a1fd 100644
--- a/src/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
+++ b/src/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
@@ -134,7 +134,7 @@ public class MiniAccumuloCluster {
ArrayList<String> argList = new ArrayList<String>();
- argList.addAll(Arrays.asList(javaBin, "-cp", classpath, "-Xmx128m", "-XX:+UseConcMarkSweepGC", "-XX:CMSInitiatingOccupancyFraction=75",
+ argList.addAll(Arrays.asList(javaBin, "-cp", classpath, "-Xmx128m", "-XX:+UseConcMarkSweepGC", "-XX:CMSInitiatingOccupancyFraction=75", "-Djline.WindowsTerminal.directConsole=false",
Main.class.getName(), className));
argList.addAll(Arrays.asList(args));
@@ -143,6 +143,7 @@ public class MiniAccumuloCluster {
builder.environment().put("ACCUMULO_HOME", config.getDir().getAbsolutePath());
builder.environment().put("ACCUMULO_LOG_DIR", logDir.getAbsolutePath());
+
// if we're running under accumulo.start, we forward these env vars
String env = System.getenv("HADOOP_PREFIX");
@@ -296,7 +297,7 @@ public class MiniAccumuloCluster {
}
});
- zooKeeperProcess = exec(Main.class, ZooKeeperServerMain.class.getName(), zooCfgFile.getAbsolutePath());
+ zooKeeperProcess = exec(ZooKeeperServerMain.class, zooCfgFile.getAbsolutePath());
// sleep a little bit to let zookeeper come up before calling init, seems to work better
UtilWaitThread.sleep(250);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d5da59ac/src/server/src/main/java/org/apache/accumulo/server/logger/LogService.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/logger/LogService.java b/src/server/src/main/java/org/apache/accumulo/server/logger/LogService.java
index b83b49b..b386234 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/logger/LogService.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/logger/LogService.java
@@ -157,8 +157,8 @@ public class LogService implements MutationLogger.Iface, Watcher {
}
final Set<String> rootDirs = new HashSet<String>();
for (String root : ServerConfiguration.getSystemConfiguration().get(Property.LOGGER_DIR).split(",")) {
- if (!root.startsWith("/"))
- root = System.getenv("ACCUMULO_HOME") + "/" + root;
+ if (!(new File(root).isAbsolute()))
+ root = System.getenv("ACCUMULO_HOME") + File.separator + root;
else if (root.equals(""))
root = System.getProperty("org.apache.accumulo.core.dir.log");
else if (root == null || root.isEmpty()) {
[38/50] [abbrv] git commit: ACCUMULO-1560 updating pom to produce
x86_64 arch rpm
Posted by el...@apache.org.
ACCUMULO-1560 updating pom to produce x86_64 arch rpm
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.4@1501624 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e6fe91ab
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e6fe91ab
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e6fe91ab
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: e6fe91abab2ac5a3c113cab297e19158d01f165a
Parents: e4f34c4
Author: Mike Drob <md...@apache.org>
Authored: Wed Jul 10 00:59:40 2013 +0000
Committer: Mike Drob <md...@apache.org>
Committed: Wed Jul 10 00:59:40 2013 +0000
----------------------------------------------------------------------
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e6fe91ab/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 9ed57b9..addb43e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -201,7 +201,7 @@
</description>
<copyright>2011 The Apache Software Foundation.</copyright>
<url>http://accumulo.apache.org</url>
- <needarch>true</needarch>
+ <needarch>x86_64</needarch>
<group>Utilities</group>
<requires>
<require>jdk</require>
[04/50] [abbrv] git commit: ACCUMULO-14 move MAC to server module
Posted by el...@apache.org.
ACCUMULO-14 move MAC to server module
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.4@1478977 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d52636c4
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d52636c4
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d52636c4
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: d52636c48372750b0d29b34f00c36522bd6efa4c
Parents: 7f5d017
Author: Keith Turner <kt...@apache.org>
Authored: Fri May 3 21:00:23 2013 +0000
Committer: Keith Turner <kt...@apache.org>
Committed: Fri May 3 21:00:23 2013 +0000
----------------------------------------------------------------------
pom.xml | 1 -
src/proxy/pom.xml | 6 -
.../java/org/apache/accumulo/proxy/Proxy.java | 2 +-
.../org/apache/accumulo/proxy/SimpleTest.java | 2 +-
.../server/test/MiniAccumuloCluster.java | 364 +++++++++++++++++++
.../server/test/MiniAccumuloConfig.java | 89 +++++
.../server/test/MiniAccumuloClusterTest.java | 150 ++++++++
src/server/src/test/resources/log4j.properties | 8 +
src/test/pom.xml | 59 ---
.../accumulo/test/MiniAccumuloCluster.java | 364 -------------------
.../accumulo/test/MiniAccumuloConfig.java | 89 -----
src/test/src/main/resources/log4j.properties | 8 -
.../accumulo/test/MiniAccumuloClusterTest.java | 149 --------
13 files changed, 613 insertions(+), 678 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d52636c4/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 0b8840f..5dcc0c2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -37,7 +37,6 @@
<module>src/start</module>
<module>src/examples</module>
<module>src/proxy</module>
- <module>src/test</module>
</modules>
<build>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d52636c4/src/proxy/pom.xml
----------------------------------------------------------------------
diff --git a/src/proxy/pom.xml b/src/proxy/pom.xml
index 297fafd..5cfde7b 100644
--- a/src/proxy/pom.xml
+++ b/src/proxy/pom.xml
@@ -77,12 +77,6 @@
<scope>compile</scope>
</dependency>
<dependency>
- <groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-test</artifactId>
- <version>${project.version}</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
<scope>compile</scope>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d52636c4/src/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java
----------------------------------------------------------------------
diff --git a/src/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java b/src/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java
index 78d7a66..8fcd4a8 100644
--- a/src/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java
+++ b/src/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java
@@ -23,7 +23,7 @@ import java.lang.reflect.Constructor;
import java.util.Properties;
import org.apache.accumulo.proxy.thrift.AccumuloProxy;
-import org.apache.accumulo.test.MiniAccumuloCluster;
+import org.apache.accumulo.server.test.MiniAccumuloCluster;
import org.apache.log4j.Logger;
import org.apache.thrift.TProcessor;
import org.apache.thrift.protocol.TCompactProtocol;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d52636c4/src/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
----------------------------------------------------------------------
diff --git a/src/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java b/src/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
index a841bd2..20b6279 100644
--- a/src/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
+++ b/src/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
@@ -74,9 +74,9 @@ import org.apache.accumulo.proxy.thrift.TimeType;
import org.apache.accumulo.proxy.thrift.UnknownScanner;
import org.apache.accumulo.proxy.thrift.UnknownWriter;
import org.apache.accumulo.proxy.thrift.WriterOptions;
+import org.apache.accumulo.server.test.MiniAccumuloCluster;
import org.apache.accumulo.server.test.functional.SlowIterator;
import org.apache.accumulo.server.util.PortUtils;
-import org.apache.accumulo.test.MiniAccumuloCluster;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d52636c4/src/server/src/main/java/org/apache/accumulo/server/test/MiniAccumuloCluster.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/test/MiniAccumuloCluster.java b/src/server/src/main/java/org/apache/accumulo/server/test/MiniAccumuloCluster.java
new file mode 100644
index 0000000..4f74cb2
--- /dev/null
+++ b/src/server/src/main/java/org/apache/accumulo/server/test/MiniAccumuloCluster.java
@@ -0,0 +1,364 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.test;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.TimerTask;
+
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.server.logger.LogService;
+import org.apache.accumulo.server.master.Master;
+import org.apache.accumulo.server.tabletserver.TabletServer;
+import org.apache.accumulo.server.util.Initialize;
+import org.apache.accumulo.server.util.PortUtils;
+import org.apache.accumulo.server.util.time.SimpleTimer;
+import org.apache.accumulo.start.Main;
+import org.apache.zookeeper.server.ZooKeeperServerMain;
+
+/**
+ * A utility class that will create Zookeeper and Accumulo processes that write all of their data to a single local directory. This class makes it easy to test
+ * code against a real Accumulo instance. Its much more accurate for testing than MockAccumulo, but much slower than MockAccumulo.
+ *
+ * @since 1.5.0, 1.4.4
+ */
+public class MiniAccumuloCluster {
+
+ private static final String INSTANCE_SECRET = "DONTTELL";
+ private static final String INSTANCE_NAME = "miniInstance";
+
+ private static class LogWriter extends Thread {
+ private BufferedReader in;
+ private BufferedWriter out;
+
+ /**
+ * @throws java.io.IOException
+ */
+ public LogWriter(InputStream stream, File logFile) throws IOException {
+ this.setDaemon(true);
+ this.in = new BufferedReader(new InputStreamReader(stream));
+ out = new BufferedWriter(new FileWriter(logFile));
+
+ SimpleTimer.getInstance().schedule(new TimerTask() {
+ @Override
+ public void run() {
+ try {
+ flush();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }, 1000, 1000);
+ }
+
+ public synchronized void flush() throws IOException {
+ if (out != null)
+ out.flush();
+ }
+
+ @Override
+ public void run() {
+ String line;
+
+ try {
+ while ((line = in.readLine()) != null) {
+ out.append(line);
+ out.append("\n");
+ }
+
+ synchronized (this) {
+ out.close();
+ out = null;
+ in.close();
+ }
+
+ } catch (IOException e) {
+ }
+ }
+ }
+
+ private File libDir;
+ private File confDir;
+ private File zooKeeperDir;
+ private File accumuloDir;
+ private File zooCfgFile;
+ private File logDir;
+ private File walogDir;
+
+ private Process zooKeeperProcess;
+ private Process masterProcess;
+ private Process loggerProcess;
+
+ private int zooKeeperPort;
+
+ private List<LogWriter> logWriters = new ArrayList<LogWriter>();
+
+ private MiniAccumuloConfig config;
+ private Process[] tabletServerProcesses;
+
+ Process exec(Class<? extends Object> clazz, String... args) throws IOException {
+ String javaHome = System.getProperty("java.home");
+ String javaBin = javaHome + File.separator + "bin" + File.separator + "java";
+ String classpath = System.getProperty("java.class.path");
+
+ classpath = confDir.getAbsolutePath() + File.pathSeparator + classpath;
+
+ String className = clazz.getCanonicalName();
+
+ ArrayList<String> argList = new ArrayList<String>();
+
+ argList.addAll(Arrays.asList(javaBin, "-cp", classpath, "-Xmx128m", "-XX:+UseConcMarkSweepGC", "-XX:CMSInitiatingOccupancyFraction=75",
+ Main.class.getName(), className));
+
+ argList.addAll(Arrays.asList(args));
+
+ ProcessBuilder builder = new ProcessBuilder(argList);
+
+ builder.environment().put("ACCUMULO_HOME", config.getDir().getAbsolutePath());
+ builder.environment().put("ACCUMULO_LOG_DIR", logDir.getAbsolutePath());
+
+ // if we're running under accumulo.start, we forward these env vars
+ String env = System.getenv("HADOOP_PREFIX");
+ if (env != null)
+ builder.environment().put("HADOOP_PREFIX", env);
+ env = System.getenv("ZOOKEEPER_HOME");
+ if (env != null)
+ builder.environment().put("ZOOKEEPER_HOME", env);
+
+ Process process = builder.start();
+
+ LogWriter lw;
+ lw = new LogWriter(process.getErrorStream(), new File(logDir, clazz.getSimpleName() + "_" + process.hashCode() + ".err"));
+ logWriters.add(lw);
+ lw.start();
+ lw = new LogWriter(process.getInputStream(), new File(logDir, clazz.getSimpleName() + "_" + process.hashCode() + ".out"));
+ logWriters.add(lw);
+ lw.start();
+
+ return process;
+ }
+
+ private void appendProp(FileWriter fileWriter, Property key, String value, Map<String,String> siteConfig) throws IOException {
+ appendProp(fileWriter, key.getKey(), value, siteConfig);
+ }
+
+ private void appendProp(FileWriter fileWriter, String key, String value, Map<String,String> siteConfig) throws IOException {
+ if (!siteConfig.containsKey(key))
+ fileWriter.append("<property><name>" + key + "</name><value>" + value + "</value></property>\n");
+ }
+
+ /**
+ * @param dir
+ * An empty or nonexistant temp directoy that Accumulo and Zookeeper can store data in. Creating the directory is left to the user. Java 7, Guava,
+ * and Junit provide methods for creating temporary directories.
+ * @param rootPassword
+ * Initial root password for instance.
+ * @throws IOException
+ */
+ public MiniAccumuloCluster(File dir, String rootPassword) throws IOException {
+ this(new MiniAccumuloConfig(dir, rootPassword));
+ }
+
+ /**
+ * @param config
+ * initial configuration
+ * @throws IOException
+ */
+
+ public MiniAccumuloCluster(MiniAccumuloConfig config) throws IOException {
+
+ if (config.getDir().exists() && !config.getDir().isDirectory())
+ throw new IllegalArgumentException("Must pass in directory, " + config.getDir() + " is a file");
+
+ if (config.getDir().exists() && config.getDir().list().length != 0)
+ throw new IllegalArgumentException("Directory " + config.getDir() + " is not empty");
+
+ this.config = config;
+
+ libDir = new File(config.getDir(), "lib");
+ confDir = new File(config.getDir(), "conf");
+ accumuloDir = new File(config.getDir(), "accumulo");
+ zooKeeperDir = new File(config.getDir(), "zookeeper");
+ logDir = new File(config.getDir(), "logs");
+ walogDir = new File(config.getDir(), "walogs");
+
+ confDir.mkdirs();
+ accumuloDir.mkdirs();
+ zooKeeperDir.mkdirs();
+ logDir.mkdirs();
+ walogDir.mkdirs();
+ libDir.mkdirs();
+
+ zooKeeperPort = PortUtils.getRandomFreePort();
+
+ File siteFile = new File(confDir, "accumulo-site.xml");
+
+ FileWriter fileWriter = new FileWriter(siteFile);
+ fileWriter.append("<configuration>\n");
+
+ HashMap<String,String> siteConfig = new HashMap<String,String>(config.getSiteConfig());
+
+ appendProp(fileWriter, Property.INSTANCE_DFS_URI, "file:///", siteConfig);
+ appendProp(fileWriter, Property.INSTANCE_DFS_DIR, accumuloDir.getAbsolutePath(), siteConfig);
+ appendProp(fileWriter, Property.INSTANCE_ZK_HOST, "localhost:" + zooKeeperPort, siteConfig);
+ appendProp(fileWriter, Property.INSTANCE_SECRET, INSTANCE_SECRET, siteConfig);
+ appendProp(fileWriter, Property.MASTER_CLIENTPORT, "" + PortUtils.getRandomFreePort(), siteConfig);
+ appendProp(fileWriter, Property.TSERV_CLIENTPORT, "" + PortUtils.getRandomFreePort(), siteConfig);
+ appendProp(fileWriter, Property.TSERV_PORTSEARCH, "true", siteConfig);
+ appendProp(fileWriter, Property.LOGGER_DIR, walogDir.getAbsolutePath(), siteConfig);
+ appendProp(fileWriter, Property.TSERV_DATACACHE_SIZE, "10M", siteConfig);
+ appendProp(fileWriter, Property.TSERV_INDEXCACHE_SIZE, "10M", siteConfig);
+ appendProp(fileWriter, Property.TSERV_MAXMEM, "50M", siteConfig);
+ appendProp(fileWriter, Property.TSERV_WALOG_MAX_SIZE, "100M", siteConfig);
+ appendProp(fileWriter, Property.TSERV_NATIVEMAP_ENABLED, "false", siteConfig);
+ appendProp(fileWriter, Property.TRACE_PORT, "" + PortUtils.getRandomFreePort(), siteConfig);
+ appendProp(fileWriter, Property.LOGGER_SORT_BUFFER_SIZE, "50M", siteConfig);
+ appendProp(fileWriter, Property.LOGGER_PORTSEARCH, "true", siteConfig);
+
+ // since there is a small amount of memory, check more frequently for majc... setting may not be needed in 1.5
+ appendProp(fileWriter, Property.TSERV_MAJC_DELAY, "3", siteConfig);
+ String cp = System.getenv("ACCUMULO_HOME") + "/lib/.*.jar," + "$ZOOKEEPER_HOME/zookeeper[^.].*.jar," + "$HADOOP_HOME/[^.].*.jar,"
+ + "$HADOOP_HOME/lib/[^.].*.jar," + "$HADOOP_PREFIX/share/hadoop/common/.*.jar," + "$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,"
+ + "$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar," + "$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar";
+ appendProp(fileWriter, Property.GENERAL_CLASSPATHS, cp, siteConfig);
+ appendProp(fileWriter, Property.GENERAL_DYNAMIC_CLASSPATHS, libDir.getAbsolutePath(), siteConfig);
+
+ for (Entry<String,String> entry : siteConfig.entrySet())
+ fileWriter.append("<property><name>" + entry.getKey() + "</name><value>" + entry.getValue() + "</value></property>\n");
+ fileWriter.append("</configuration>\n");
+ fileWriter.close();
+
+ zooCfgFile = new File(confDir, "zoo.cfg");
+ fileWriter = new FileWriter(zooCfgFile);
+
+ // zookeeper uses Properties to read its config, so use that to write in order to properly escape things like Windows paths
+ Properties zooCfg = new Properties();
+ zooCfg.setProperty("tickTime", "1000");
+ zooCfg.setProperty("initLimit", "10");
+ zooCfg.setProperty("syncLimit", "5");
+ zooCfg.setProperty("clientPort", zooKeeperPort + "");
+ zooCfg.setProperty("maxClientCnxns", "100");
+ zooCfg.setProperty("dataDir", zooKeeperDir.getAbsolutePath());
+ zooCfg.store(fileWriter, null);
+
+ fileWriter.close();
+ }
+
+ /**
+ * Starts Accumulo and Zookeeper processes. Can only be called once.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws IllegalStateException
+ * if already started
+ */
+ public void start() throws IOException, InterruptedException {
+ if (zooKeeperProcess != null)
+ throw new IllegalStateException("Already started");
+
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ try {
+ MiniAccumuloCluster.this.stop();
+ } catch (IOException e) {
+ e.printStackTrace();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ });
+
+ zooKeeperProcess = exec(Main.class, ZooKeeperServerMain.class.getName(), zooCfgFile.getAbsolutePath());
+
+ // sleep a little bit to let zookeeper come up before calling init, seems to work better
+ UtilWaitThread.sleep(250);
+
+ // TODO initialization could probably be done in process
+ Process initProcess = exec(Initialize.class);
+ initProcess.getOutputStream().write((INSTANCE_NAME + "\n").getBytes());
+ initProcess.getOutputStream().write((config.getRootPassword() + "\n").getBytes());
+ initProcess.getOutputStream().write((config.getRootPassword() + "\n").getBytes());
+ initProcess.getOutputStream().flush();
+ int ret = initProcess.waitFor();
+ if (ret != 0) {
+ throw new RuntimeException("Initialize process returned " + ret);
+ }
+
+ tabletServerProcesses = new Process[config.getNumTservers()];
+ for (int i = 0; i < config.getNumTservers(); i++) {
+ tabletServerProcesses[i] = exec(TabletServer.class);
+ }
+
+ loggerProcess = exec(LogService.class);
+ masterProcess = exec(Master.class);
+ }
+
+ /**
+ * @return Accumulo instance name
+ */
+
+ public String getInstanceName() {
+ return INSTANCE_NAME;
+ }
+
+ /**
+ * @return zookeeper connection string
+ */
+
+ public String getZooKeepers() {
+ return "localhost:" + zooKeeperPort;
+ }
+
+ /**
+ * Stops Accumulo and Zookeeper processes. If stop is not called, there is a shutdown hook that is setup to kill the processes. Howerver its probably best to
+ * call stop in a finally block as soon as possible.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ */
+
+ public void stop() throws IOException, InterruptedException {
+ if (zooKeeperProcess != null)
+ zooKeeperProcess.destroy();
+ if (loggerProcess != null)
+ loggerProcess.destroy();
+ if (masterProcess != null)
+ masterProcess.destroy();
+ if (tabletServerProcesses != null) {
+ for (Process tserver : tabletServerProcesses) {
+ tserver.destroy();
+ }
+ }
+
+ for (LogWriter lw : logWriters)
+ lw.flush();
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d52636c4/src/server/src/main/java/org/apache/accumulo/server/test/MiniAccumuloConfig.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/test/MiniAccumuloConfig.java b/src/server/src/main/java/org/apache/accumulo/server/test/MiniAccumuloConfig.java
new file mode 100644
index 0000000..6b3b211
--- /dev/null
+++ b/src/server/src/main/java/org/apache/accumulo/server/test/MiniAccumuloConfig.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.test;
+
+import java.io.File;
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * Holds configuration for {@link MiniAccumuloCluster}. Required configurations must be passed to constructor and all other configurations are optional.
+ *
+ * @since 1.5.0, 1.4.4
+ */
+
+public class MiniAccumuloConfig {
+
+ private File dir = null;
+ private String rootPassword = null;
+ private Map<String,String> siteConfig = Collections.emptyMap();
+ private int numTservers = 2;
+
+ /**
+ * @param dir
+ * An empty or nonexistant temp directoy that Accumulo and Zookeeper can store data in. Creating the directory is left to the user. Java 7, Guava,
+ * and Junit provide methods for creating temporary directories.
+ * @param rootPassword
+ * The initial password for the Accumulo root user
+ */
+
+ public MiniAccumuloConfig(File dir, String rootPassword) {
+ this.dir = dir;
+ this.rootPassword = rootPassword;
+ }
+
+ public File getDir() {
+ return dir;
+ }
+
+ public String getRootPassword() {
+ return rootPassword;
+ }
+
+ public int getNumTservers() {
+ return numTservers;
+ }
+
+ /**
+ * Calling this method is optional. If not set, it defaults to two.
+ *
+ * @param numTservers
+ * the number of tablet servers that mini accumulo cluster should start
+ */
+
+ public MiniAccumuloConfig setNumTservers(int numTservers) {
+ if (numTservers < 1)
+ throw new IllegalArgumentException("Must have at least one tablet server");
+ this.numTservers = numTservers;
+ return this;
+ }
+
+ public Map<String,String> getSiteConfig() {
+ return siteConfig;
+ }
+
+ /**
+ * Calling this method is optional. If not set, it defautls to an empty map.
+ *
+ * @param siteConfig
+ * key/values that you normally put in accumulo-site.xml can be put here
+ */
+ public MiniAccumuloConfig setSiteConfig(Map<String,String> siteConfig) {
+ this.siteConfig = siteConfig;
+ return this;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d52636c4/src/server/src/test/java/org/apache/accumulo/server/test/MiniAccumuloClusterTest.java
----------------------------------------------------------------------
diff --git a/src/server/src/test/java/org/apache/accumulo/server/test/MiniAccumuloClusterTest.java b/src/server/src/test/java/org/apache/accumulo/server/test/MiniAccumuloClusterTest.java
new file mode 100644
index 0000000..a469fc5
--- /dev/null
+++ b/src/server/src/test/java/org/apache/accumulo/server/test/MiniAccumuloClusterTest.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.test;
+
+import java.util.Collections;
+import java.util.Map.Entry;
+import java.util.UUID;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.LongCombiner;
+import org.apache.accumulo.core.iterators.user.SummingCombiner;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.server.test.MiniAccumuloCluster;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+public class MiniAccumuloClusterTest {
+
+ private static final Logger logger = Logger.getLogger(MiniAccumuloClusterTest.class);
+
+ private static TemporaryFolder tmpDir = new TemporaryFolder();
+ private static MiniAccumuloCluster accumulo;
+
+ @BeforeClass
+ public static void setupMiniCluster() throws Exception {
+
+ tmpDir.create();
+ logger.info("MiniCluster started @ " + tmpDir.getRoot());
+ Logger.getLogger("org.apache.zookeeper").setLevel(Level.ERROR);
+
+ accumulo = new MiniAccumuloCluster(tmpDir.getRoot(), "superSecret");
+ accumulo.start();
+ }
+
+ @Test(timeout = 30000)
+ public void test() throws Exception {
+ Connector conn = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers()).getConnector("root", "superSecret".getBytes());
+
+ conn.tableOperations().create("table1");
+
+ conn.securityOperations().createUser("user1", "pass1".getBytes(), new Authorizations("A", "B"));
+ conn.securityOperations().grantTablePermission("user1", "table1", TablePermission.WRITE);
+ conn.securityOperations().grantTablePermission("user1", "table1", TablePermission.READ);
+
+ IteratorSetting is = new IteratorSetting(10, SummingCombiner.class);
+ SummingCombiner.setEncodingType(is, LongCombiner.Type.STRING);
+ SummingCombiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("META", "COUNT")));
+
+ conn.tableOperations().attachIterator("table1", is);
+
+ Connector uconn = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers()).getConnector("user1", "pass1".getBytes());
+
+ BatchWriter bw = uconn.createBatchWriter("table1", 10000, 1000000, 2);
+
+ UUID uuid = UUID.randomUUID();
+
+ Mutation m = new Mutation(uuid.toString());
+ m.put("META", "SIZE", new ColumnVisibility("A|B"), "8");
+ m.put("META", "CRC", new ColumnVisibility("A|B"), "456");
+ m.put("META", "COUNT", new ColumnVisibility("A|B"), "1");
+ m.put("DATA", "IMG", new ColumnVisibility("A&B"), "ABCDEFGH");
+
+ bw.addMutation(m);
+ bw.flush();
+
+ m = new Mutation(uuid.toString());
+ m.put("META", "COUNT", new ColumnVisibility("A|B"), "1");
+ m.put("META", "CRC", new ColumnVisibility("A|B"), "123");
+ bw.addMutation(m);
+
+ bw.close();
+
+ int count = 0;
+ Scanner scanner = uconn.createScanner("table1", new Authorizations("A"));
+ for (Entry<Key,Value> entry : scanner) {
+ if (entry.getKey().getColumnQualifierData().toString().equals("COUNT")) {
+ Assert.assertEquals("2", entry.getValue().toString());
+ } else if (entry.getKey().getColumnQualifierData().toString().equals("SIZE")) {
+ Assert.assertEquals("8", entry.getValue().toString());
+ } else if (entry.getKey().getColumnQualifierData().toString().equals("CRC")) {
+ Assert.assertEquals("123", entry.getValue().toString());
+ } else {
+ Assert.assertTrue(false);
+ }
+ count++;
+ }
+
+ Assert.assertEquals(3, count);
+
+ count = 0;
+ scanner = uconn.createScanner("table1", new Authorizations("A", "B"));
+ for (Entry<Key,Value> entry : scanner) {
+ if (entry.getKey().getColumnQualifierData().toString().equals("IMG")) {
+ Assert.assertEquals("ABCDEFGH", entry.getValue().toString());
+ }
+ count++;
+ }
+
+ Assert.assertEquals(4, count);
+
+ conn.tableOperations().delete("table1");
+ }
+
+ @Test(timeout = 20000)
+ public void testMultipleTabletServersRunning() throws AccumuloException, AccumuloSecurityException {
+
+ Connector conn = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers()).getConnector("root", "superSecret".getBytes());
+
+ while (conn.instanceOperations().getTabletServers().size() != 2) {
+ UtilWaitThread.sleep(500);
+ }
+ }
+
+ @AfterClass
+ public static void tearDownMiniCluster() throws Exception {
+ accumulo.stop();
+ tmpDir.delete();
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d52636c4/src/server/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/src/server/src/test/resources/log4j.properties b/src/server/src/test/resources/log4j.properties
new file mode 100644
index 0000000..0b17793
--- /dev/null
+++ b/src/server/src/test/resources/log4j.properties
@@ -0,0 +1,8 @@
+log4j.rootLogger=INFO, CA
+log4j.appender.CA=org.apache.log4j.ConsoleAppender
+log4j.appender.CA.layout=org.apache.log4j.PatternLayout
+log4j.appender.CA.layout.ConversionPattern=[%t] %-5p %c %x - %m%n
+
+log4j.logger.org.apache.zookeeper=ERROR,CA
+log4j.logger.org.apache.accumulo.core.client.impl.ServerClient=ERROR
+log4j.logger.org.apache.accumulo.server.security.Auditor=off
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d52636c4/src/test/pom.xml
----------------------------------------------------------------------
diff --git a/src/test/pom.xml b/src/test/pom.xml
deleted file mode 100644
index 12c7e38..0000000
--- a/src/test/pom.xml
+++ /dev/null
@@ -1,59 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-
- <parent>
- <groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo</artifactId>
- <version>1.4.4-SNAPSHOT</version>
- <relativePath>../../</relativePath>
- </parent>
-
- <modelVersion>4.0.0</modelVersion>
- <artifactId>accumulo-test</artifactId>
- <name>accumulo-test</name>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-server</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>zookeeper</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- </dependency>
- <dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
- </dependencies>
-
-</project>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d52636c4/src/test/src/main/java/org/apache/accumulo/test/MiniAccumuloCluster.java
----------------------------------------------------------------------
diff --git a/src/test/src/main/java/org/apache/accumulo/test/MiniAccumuloCluster.java b/src/test/src/main/java/org/apache/accumulo/test/MiniAccumuloCluster.java
deleted file mode 100644
index e0e49d4..0000000
--- a/src/test/src/main/java/org/apache/accumulo/test/MiniAccumuloCluster.java
+++ /dev/null
@@ -1,364 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Properties;
-import java.util.TimerTask;
-
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.util.UtilWaitThread;
-import org.apache.accumulo.server.logger.LogService;
-import org.apache.accumulo.server.master.Master;
-import org.apache.accumulo.server.tabletserver.TabletServer;
-import org.apache.accumulo.server.util.Initialize;
-import org.apache.accumulo.server.util.PortUtils;
-import org.apache.accumulo.server.util.time.SimpleTimer;
-import org.apache.accumulo.start.Main;
-import org.apache.zookeeper.server.ZooKeeperServerMain;
-
-/**
- * A utility class that will create Zookeeper and Accumulo processes that write all of their data to a single local directory. This class makes it easy to test
- * code against a real Accumulo instance. Its much more accurate for testing than MockAccumulo, but much slower than MockAccumulo.
- *
- * @since 1.5.0, 1.4.4
- */
-public class MiniAccumuloCluster {
-
- private static final String INSTANCE_SECRET = "DONTTELL";
- private static final String INSTANCE_NAME = "miniInstance";
-
- private static class LogWriter extends Thread {
- private BufferedReader in;
- private BufferedWriter out;
-
- /**
- * @throws java.io.IOException
- */
- public LogWriter(InputStream stream, File logFile) throws IOException {
- this.setDaemon(true);
- this.in = new BufferedReader(new InputStreamReader(stream));
- out = new BufferedWriter(new FileWriter(logFile));
-
- SimpleTimer.getInstance().schedule(new TimerTask() {
- @Override
- public void run() {
- try {
- flush();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- }, 1000, 1000);
- }
-
- public synchronized void flush() throws IOException {
- if (out != null)
- out.flush();
- }
-
- @Override
- public void run() {
- String line;
-
- try {
- while ((line = in.readLine()) != null) {
- out.append(line);
- out.append("\n");
- }
-
- synchronized (this) {
- out.close();
- out = null;
- in.close();
- }
-
- } catch (IOException e) {
- }
- }
- }
-
- private File libDir;
- private File confDir;
- private File zooKeeperDir;
- private File accumuloDir;
- private File zooCfgFile;
- private File logDir;
- private File walogDir;
-
- private Process zooKeeperProcess;
- private Process masterProcess;
- private Process loggerProcess;
-
- private int zooKeeperPort;
-
- private List<LogWriter> logWriters = new ArrayList<LogWriter>();
-
- private MiniAccumuloConfig config;
- private Process[] tabletServerProcesses;
-
- Process exec(Class<? extends Object> clazz, String... args) throws IOException {
- String javaHome = System.getProperty("java.home");
- String javaBin = javaHome + File.separator + "bin" + File.separator + "java";
- String classpath = System.getProperty("java.class.path");
-
- classpath = confDir.getAbsolutePath() + File.pathSeparator + classpath;
-
- String className = clazz.getCanonicalName();
-
- ArrayList<String> argList = new ArrayList<String>();
-
- argList.addAll(Arrays.asList(javaBin, "-cp", classpath, "-Xmx128m", "-XX:+UseConcMarkSweepGC", "-XX:CMSInitiatingOccupancyFraction=75",
- Main.class.getName(), className));
-
- argList.addAll(Arrays.asList(args));
-
- ProcessBuilder builder = new ProcessBuilder(argList);
-
- builder.environment().put("ACCUMULO_HOME", config.getDir().getAbsolutePath());
- builder.environment().put("ACCUMULO_LOG_DIR", logDir.getAbsolutePath());
-
- // if we're running under accumulo.start, we forward these env vars
- String env = System.getenv("HADOOP_PREFIX");
- if (env != null)
- builder.environment().put("HADOOP_PREFIX", env);
- env = System.getenv("ZOOKEEPER_HOME");
- if (env != null)
- builder.environment().put("ZOOKEEPER_HOME", env);
-
- Process process = builder.start();
-
- LogWriter lw;
- lw = new LogWriter(process.getErrorStream(), new File(logDir, clazz.getSimpleName() + "_" + process.hashCode() + ".err"));
- logWriters.add(lw);
- lw.start();
- lw = new LogWriter(process.getInputStream(), new File(logDir, clazz.getSimpleName() + "_" + process.hashCode() + ".out"));
- logWriters.add(lw);
- lw.start();
-
- return process;
- }
-
- private void appendProp(FileWriter fileWriter, Property key, String value, Map<String,String> siteConfig) throws IOException {
- appendProp(fileWriter, key.getKey(), value, siteConfig);
- }
-
- private void appendProp(FileWriter fileWriter, String key, String value, Map<String,String> siteConfig) throws IOException {
- if (!siteConfig.containsKey(key))
- fileWriter.append("<property><name>" + key + "</name><value>" + value + "</value></property>\n");
- }
-
- /**
- * @param dir
- * An empty or nonexistant temp directoy that Accumulo and Zookeeper can store data in. Creating the directory is left to the user. Java 7, Guava,
- * and Junit provide methods for creating temporary directories.
- * @param rootPassword
- * Initial root password for instance.
- * @throws IOException
- */
- public MiniAccumuloCluster(File dir, String rootPassword) throws IOException {
- this(new MiniAccumuloConfig(dir, rootPassword));
- }
-
- /**
- * @param config
- * initial configuration
- * @throws IOException
- */
-
- public MiniAccumuloCluster(MiniAccumuloConfig config) throws IOException {
-
- if (config.getDir().exists() && !config.getDir().isDirectory())
- throw new IllegalArgumentException("Must pass in directory, " + config.getDir() + " is a file");
-
- if (config.getDir().exists() && config.getDir().list().length != 0)
- throw new IllegalArgumentException("Directory " + config.getDir() + " is not empty");
-
- this.config = config;
-
- libDir = new File(config.getDir(), "lib");
- confDir = new File(config.getDir(), "conf");
- accumuloDir = new File(config.getDir(), "accumulo");
- zooKeeperDir = new File(config.getDir(), "zookeeper");
- logDir = new File(config.getDir(), "logs");
- walogDir = new File(config.getDir(), "walogs");
-
- confDir.mkdirs();
- accumuloDir.mkdirs();
- zooKeeperDir.mkdirs();
- logDir.mkdirs();
- walogDir.mkdirs();
- libDir.mkdirs();
-
- zooKeeperPort = PortUtils.getRandomFreePort();
-
- File siteFile = new File(confDir, "accumulo-site.xml");
-
- FileWriter fileWriter = new FileWriter(siteFile);
- fileWriter.append("<configuration>\n");
-
- HashMap<String,String> siteConfig = new HashMap<String,String>(config.getSiteConfig());
-
- appendProp(fileWriter, Property.INSTANCE_DFS_URI, "file:///", siteConfig);
- appendProp(fileWriter, Property.INSTANCE_DFS_DIR, accumuloDir.getAbsolutePath(), siteConfig);
- appendProp(fileWriter, Property.INSTANCE_ZK_HOST, "localhost:" + zooKeeperPort, siteConfig);
- appendProp(fileWriter, Property.INSTANCE_SECRET, INSTANCE_SECRET, siteConfig);
- appendProp(fileWriter, Property.MASTER_CLIENTPORT, "" + PortUtils.getRandomFreePort(), siteConfig);
- appendProp(fileWriter, Property.TSERV_CLIENTPORT, "" + PortUtils.getRandomFreePort(), siteConfig);
- appendProp(fileWriter, Property.TSERV_PORTSEARCH, "true", siteConfig);
- appendProp(fileWriter, Property.LOGGER_DIR, walogDir.getAbsolutePath(), siteConfig);
- appendProp(fileWriter, Property.TSERV_DATACACHE_SIZE, "10M", siteConfig);
- appendProp(fileWriter, Property.TSERV_INDEXCACHE_SIZE, "10M", siteConfig);
- appendProp(fileWriter, Property.TSERV_MAXMEM, "50M", siteConfig);
- appendProp(fileWriter, Property.TSERV_WALOG_MAX_SIZE, "100M", siteConfig);
- appendProp(fileWriter, Property.TSERV_NATIVEMAP_ENABLED, "false", siteConfig);
- appendProp(fileWriter, Property.TRACE_PORT, "" + PortUtils.getRandomFreePort(), siteConfig);
- appendProp(fileWriter, Property.LOGGER_SORT_BUFFER_SIZE, "50M", siteConfig);
- appendProp(fileWriter, Property.LOGGER_PORTSEARCH, "true", siteConfig);
-
- // since there is a small amount of memory, check more frequently for majc... setting may not be needed in 1.5
- appendProp(fileWriter, Property.TSERV_MAJC_DELAY, "3", siteConfig);
- String cp = System.getenv("ACCUMULO_HOME") + "/lib/.*.jar," + "$ZOOKEEPER_HOME/zookeeper[^.].*.jar," + "$HADOOP_HOME/[^.].*.jar,"
- + "$HADOOP_HOME/lib/[^.].*.jar," + "$HADOOP_PREFIX/share/hadoop/common/.*.jar," + "$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,"
- + "$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar," + "$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar";
- appendProp(fileWriter, Property.GENERAL_CLASSPATHS, cp, siteConfig);
- appendProp(fileWriter, Property.GENERAL_DYNAMIC_CLASSPATHS, libDir.getAbsolutePath(), siteConfig);
-
- for (Entry<String,String> entry : siteConfig.entrySet())
- fileWriter.append("<property><name>" + entry.getKey() + "</name><value>" + entry.getValue() + "</value></property>\n");
- fileWriter.append("</configuration>\n");
- fileWriter.close();
-
- zooCfgFile = new File(confDir, "zoo.cfg");
- fileWriter = new FileWriter(zooCfgFile);
-
- // zookeeper uses Properties to read its config, so use that to write in order to properly escape things like Windows paths
- Properties zooCfg = new Properties();
- zooCfg.setProperty("tickTime", "1000");
- zooCfg.setProperty("initLimit", "10");
- zooCfg.setProperty("syncLimit", "5");
- zooCfg.setProperty("clientPort", zooKeeperPort + "");
- zooCfg.setProperty("maxClientCnxns", "100");
- zooCfg.setProperty("dataDir", zooKeeperDir.getAbsolutePath());
- zooCfg.store(fileWriter, null);
-
- fileWriter.close();
- }
-
- /**
- * Starts Accumulo and Zookeeper processes. Can only be called once.
- *
- * @throws IOException
- * @throws InterruptedException
- * @throws IllegalStateException
- * if already started
- */
- public void start() throws IOException, InterruptedException {
- if (zooKeeperProcess != null)
- throw new IllegalStateException("Already started");
-
- Runtime.getRuntime().addShutdownHook(new Thread() {
- @Override
- public void run() {
- try {
- MiniAccumuloCluster.this.stop();
- } catch (IOException e) {
- e.printStackTrace();
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- });
-
- zooKeeperProcess = exec(Main.class, ZooKeeperServerMain.class.getName(), zooCfgFile.getAbsolutePath());
-
- // sleep a little bit to let zookeeper come up before calling init, seems to work better
- UtilWaitThread.sleep(250);
-
- // TODO initialization could probably be done in process
- Process initProcess = exec(Initialize.class);
- initProcess.getOutputStream().write((INSTANCE_NAME + "\n").getBytes());
- initProcess.getOutputStream().write((config.getRootPassword() + "\n").getBytes());
- initProcess.getOutputStream().write((config.getRootPassword() + "\n").getBytes());
- initProcess.getOutputStream().flush();
- int ret = initProcess.waitFor();
- if (ret != 0) {
- throw new RuntimeException("Initialize process returned " + ret);
- }
-
- tabletServerProcesses = new Process[config.getNumTservers()];
- for (int i = 0; i < config.getNumTservers(); i++) {
- tabletServerProcesses[i] = exec(TabletServer.class);
- }
-
- loggerProcess = exec(LogService.class);
- masterProcess = exec(Master.class);
- }
-
- /**
- * @return Accumulo instance name
- */
-
- public String getInstanceName() {
- return INSTANCE_NAME;
- }
-
- /**
- * @return zookeeper connection string
- */
-
- public String getZooKeepers() {
- return "localhost:" + zooKeeperPort;
- }
-
- /**
- * Stops Accumulo and Zookeeper processes. If stop is not called, there is a shutdown hook that is setup to kill the processes. Howerver its probably best to
- * call stop in a finally block as soon as possible.
- *
- * @throws IOException
- * @throws InterruptedException
- */
-
- public void stop() throws IOException, InterruptedException {
- if (zooKeeperProcess != null)
- zooKeeperProcess.destroy();
- if (loggerProcess != null)
- loggerProcess.destroy();
- if (masterProcess != null)
- masterProcess.destroy();
- if (tabletServerProcesses != null) {
- for (Process tserver : tabletServerProcesses) {
- tserver.destroy();
- }
- }
-
- for (LogWriter lw : logWriters)
- lw.flush();
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d52636c4/src/test/src/main/java/org/apache/accumulo/test/MiniAccumuloConfig.java
----------------------------------------------------------------------
diff --git a/src/test/src/main/java/org/apache/accumulo/test/MiniAccumuloConfig.java b/src/test/src/main/java/org/apache/accumulo/test/MiniAccumuloConfig.java
deleted file mode 100644
index dc478f3..0000000
--- a/src/test/src/main/java/org/apache/accumulo/test/MiniAccumuloConfig.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test;
-
-import java.io.File;
-import java.util.Collections;
-import java.util.Map;
-
-/**
- * Holds configuration for {@link MiniAccumuloCluster}. Required configurations must be passed to constructor and all other configurations are optional.
- *
- * @since 1.5.0, 1.4.4
- */
-
-public class MiniAccumuloConfig {
-
- private File dir = null;
- private String rootPassword = null;
- private Map<String,String> siteConfig = Collections.emptyMap();
- private int numTservers = 2;
-
- /**
- * @param dir
- * An empty or nonexistant temp directoy that Accumulo and Zookeeper can store data in. Creating the directory is left to the user. Java 7, Guava,
- * and Junit provide methods for creating temporary directories.
- * @param rootPassword
- * The initial password for the Accumulo root user
- */
-
- public MiniAccumuloConfig(File dir, String rootPassword) {
- this.dir = dir;
- this.rootPassword = rootPassword;
- }
-
- public File getDir() {
- return dir;
- }
-
- public String getRootPassword() {
- return rootPassword;
- }
-
- public int getNumTservers() {
- return numTservers;
- }
-
- /**
- * Calling this method is optional. If not set, it defaults to two.
- *
- * @param numTservers
- * the number of tablet servers that mini accumulo cluster should start
- */
-
- public MiniAccumuloConfig setNumTservers(int numTservers) {
- if (numTservers < 1)
- throw new IllegalArgumentException("Must have at least one tablet server");
- this.numTservers = numTservers;
- return this;
- }
-
- public Map<String,String> getSiteConfig() {
- return siteConfig;
- }
-
- /**
- * Calling this method is optional. If not set, it defautls to an empty map.
- *
- * @param siteConfig
- * key/values that you normally put in accumulo-site.xml can be put here
- */
- public MiniAccumuloConfig setSiteConfig(Map<String,String> siteConfig) {
- this.siteConfig = siteConfig;
- return this;
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d52636c4/src/test/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/src/test/src/main/resources/log4j.properties b/src/test/src/main/resources/log4j.properties
deleted file mode 100644
index 0b17793..0000000
--- a/src/test/src/main/resources/log4j.properties
+++ /dev/null
@@ -1,8 +0,0 @@
-log4j.rootLogger=INFO, CA
-log4j.appender.CA=org.apache.log4j.ConsoleAppender
-log4j.appender.CA.layout=org.apache.log4j.PatternLayout
-log4j.appender.CA.layout.ConversionPattern=[%t] %-5p %c %x - %m%n
-
-log4j.logger.org.apache.zookeeper=ERROR,CA
-log4j.logger.org.apache.accumulo.core.client.impl.ServerClient=ERROR
-log4j.logger.org.apache.accumulo.server.security.Auditor=off
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d52636c4/src/test/src/test/java/org/apache/accumulo/test/MiniAccumuloClusterTest.java
----------------------------------------------------------------------
diff --git a/src/test/src/test/java/org/apache/accumulo/test/MiniAccumuloClusterTest.java b/src/test/src/test/java/org/apache/accumulo/test/MiniAccumuloClusterTest.java
deleted file mode 100644
index 5316c3e..0000000
--- a/src/test/src/test/java/org/apache/accumulo/test/MiniAccumuloClusterTest.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test;
-
-import java.util.Collections;
-import java.util.Map.Entry;
-import java.util.UUID;
-
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.IteratorSetting;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.client.ZooKeeperInstance;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.iterators.LongCombiner;
-import org.apache.accumulo.core.iterators.user.SummingCombiner;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.security.ColumnVisibility;
-import org.apache.accumulo.core.security.TablePermission;
-import org.apache.accumulo.core.util.UtilWaitThread;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-public class MiniAccumuloClusterTest {
-
- private static final Logger logger = Logger.getLogger(MiniAccumuloClusterTest.class);
-
- private static TemporaryFolder tmpDir = new TemporaryFolder();
- private static MiniAccumuloCluster accumulo;
-
- @BeforeClass
- public static void setupMiniCluster() throws Exception {
-
- tmpDir.create();
- logger.info("MiniCluster started @ " + tmpDir.getRoot());
- Logger.getLogger("org.apache.zookeeper").setLevel(Level.ERROR);
-
- accumulo = new MiniAccumuloCluster(tmpDir.getRoot(), "superSecret");
- accumulo.start();
- }
-
- @Test(timeout = 30000)
- public void test() throws Exception {
- Connector conn = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers()).getConnector("root", "superSecret".getBytes());
-
- conn.tableOperations().create("table1");
-
- conn.securityOperations().createUser("user1", "pass1".getBytes(), new Authorizations("A", "B"));
- conn.securityOperations().grantTablePermission("user1", "table1", TablePermission.WRITE);
- conn.securityOperations().grantTablePermission("user1", "table1", TablePermission.READ);
-
- IteratorSetting is = new IteratorSetting(10, SummingCombiner.class);
- SummingCombiner.setEncodingType(is, LongCombiner.Type.STRING);
- SummingCombiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("META", "COUNT")));
-
- conn.tableOperations().attachIterator("table1", is);
-
- Connector uconn = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers()).getConnector("user1", "pass1".getBytes());
-
- BatchWriter bw = uconn.createBatchWriter("table1", 10000, 1000000, 2);
-
- UUID uuid = UUID.randomUUID();
-
- Mutation m = new Mutation(uuid.toString());
- m.put("META", "SIZE", new ColumnVisibility("A|B"), "8");
- m.put("META", "CRC", new ColumnVisibility("A|B"), "456");
- m.put("META", "COUNT", new ColumnVisibility("A|B"), "1");
- m.put("DATA", "IMG", new ColumnVisibility("A&B"), "ABCDEFGH");
-
- bw.addMutation(m);
- bw.flush();
-
- m = new Mutation(uuid.toString());
- m.put("META", "COUNT", new ColumnVisibility("A|B"), "1");
- m.put("META", "CRC", new ColumnVisibility("A|B"), "123");
- bw.addMutation(m);
-
- bw.close();
-
- int count = 0;
- Scanner scanner = uconn.createScanner("table1", new Authorizations("A"));
- for (Entry<Key,Value> entry : scanner) {
- if (entry.getKey().getColumnQualifierData().toString().equals("COUNT")) {
- Assert.assertEquals("2", entry.getValue().toString());
- } else if (entry.getKey().getColumnQualifierData().toString().equals("SIZE")) {
- Assert.assertEquals("8", entry.getValue().toString());
- } else if (entry.getKey().getColumnQualifierData().toString().equals("CRC")) {
- Assert.assertEquals("123", entry.getValue().toString());
- } else {
- Assert.assertTrue(false);
- }
- count++;
- }
-
- Assert.assertEquals(3, count);
-
- count = 0;
- scanner = uconn.createScanner("table1", new Authorizations("A", "B"));
- for (Entry<Key,Value> entry : scanner) {
- if (entry.getKey().getColumnQualifierData().toString().equals("IMG")) {
- Assert.assertEquals("ABCDEFGH", entry.getValue().toString());
- }
- count++;
- }
-
- Assert.assertEquals(4, count);
-
- conn.tableOperations().delete("table1");
- }
-
- @Test(timeout = 20000)
- public void testMultipleTabletServersRunning() throws AccumuloException, AccumuloSecurityException {
-
- Connector conn = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers()).getConnector("root", "superSecret".getBytes());
-
- while (conn.instanceOperations().getTabletServers().size() != 2) {
- UtilWaitThread.sleep(500);
- }
- }
-
- @AfterClass
- public static void tearDownMiniCluster() throws Exception {
- accumulo.stop();
- tmpDir.delete();
- }
-}
[35/50] [abbrv] git commit: ACCUMULO-1552 applying Jonathan Hsieh's
patch to fix the typo
Posted by el...@apache.org.
ACCUMULO-1552 applying Jonathan Hsieh's patch to fix the typo
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.4@1500718 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e3d247dc
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e3d247dc
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e3d247dc
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: e3d247dcc7a64308fc2c60c02f745481b4e180ef
Parents: 732e881
Author: Eric C. Newton <ec...@apache.org>
Authored: Mon Jul 8 13:36:02 2013 +0000
Committer: Eric C. Newton <ec...@apache.org>
Committed: Mon Jul 8 13:36:02 2013 +0000
----------------------------------------------------------------------
test/system/continuous/continuous-env.sh.example | 2 +-
test/system/continuous/run-verify.sh | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e3d247dc/test/system/continuous/continuous-env.sh.example
----------------------------------------------------------------------
diff --git a/test/system/continuous/continuous-env.sh.example b/test/system/continuous/continuous-env.sh.example
index 156aa75..54ee0e3 100644
--- a/test/system/continuous/continuous-env.sh.example
+++ b/test/system/continuous/continuous-env.sh.example
@@ -87,7 +87,7 @@ MASTER_KILL_SLEEP_TIME=60
MASTER_RESTART_SLEEP_TIME=2
#settings for the verification map reduce job
-VERFIY_OUT=/tmp/continuous_verify
+VERIFY_OUT=/tmp/continuous_verify
VERIFY_MAX_MAPS=64
VERIFY_REDUCERS=64
SCAN_OFFLINE=false
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e3d247dc/test/system/continuous/run-verify.sh
----------------------------------------------------------------------
diff --git a/test/system/continuous/run-verify.sh b/test/system/continuous/run-verify.sh
index fbd259f..edf58b7 100755
--- a/test/system/continuous/run-verify.sh
+++ b/test/system/continuous/run-verify.sh
@@ -24,5 +24,5 @@ if [ -n "$VERIFY_AUTHS" ] ; then
AUTH_OPT="--auths $VERIFY_AUTHS";
fi
-$ACCUMULO_HOME/bin/tool.sh "$SERVER_LIBJAR" org.apache.accumulo.server.test.continuous.ContinuousVerify -libjars "$SERVER_LIBJAR" $AUTH_OPT $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $VERFIY_OUT $VERIFY_MAX_MAPS $VERIFY_REDUCERS $SCAN_OFFLINE
+$ACCUMULO_HOME/bin/tool.sh "$SERVER_LIBJAR" org.apache.accumulo.server.test.continuous.ContinuousVerify -libjars "$SERVER_LIBJAR" $AUTH_OPT $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $VERIFY_OUT $VERIFY_MAX_MAPS $VERIFY_REDUCERS $SCAN_OFFLINE
[50/50] [abbrv] git commit: Merge branch '1.5.1-SNAPSHOT' into
1.5.1-SNAPSHOT-1.4.4-SNAPSHOT-merged
Posted by el...@apache.org.
Merge branch '1.5.1-SNAPSHOT' into 1.5.1-SNAPSHOT-1.4.4-SNAPSHOT-merged
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/8622b4d6
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/8622b4d6
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/8622b4d6
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 8622b4d670ecb3fde53022cb6588df649357d2c9
Parents: 8e1e2a5 1c44069
Author: Josh Elser <el...@apache.org>
Authored: Tue Jul 30 20:03:40 2013 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Tue Jul 30 20:03:40 2013 -0400
----------------------------------------------------------------------
assemble/scripts/init.d/accumulo-tserver | 2 +
.../core/client/admin/TableOperationsImpl.java | 14 +-
.../apache/accumulo/server/master/Master.java | 7 +-
.../accumulo/server/tabletserver/Compactor.java | 9 +-
.../org/apache/accumulo/test/CloneTest.java | 133 +++++++++++++++++++
5 files changed, 158 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
[41/50] [abbrv] git commit: ACCUMULO-1572 ignore connection lost;
eventually we'll get an session lost event
Posted by el...@apache.org.
ACCUMULO-1572 ignore connection lost; eventually we'll get an session lost event
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/333062d2
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/333062d2
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/333062d2
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 333062d27e25ee227365357bdca237b0c6912f68
Parents: cd8ebfe
Author: Eric Newton <er...@gmail.com>
Authored: Wed Jul 17 14:03:39 2013 -0400
Committer: Eric Newton <er...@gmail.com>
Committed: Wed Jul 17 14:03:39 2013 -0400
----------------------------------------------------------------------
.../main/java/org/apache/accumulo/server/zookeeper/ZooLock.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/333062d2/src/server/src/main/java/org/apache/accumulo/server/zookeeper/ZooLock.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/zookeeper/ZooLock.java b/src/server/src/main/java/org/apache/accumulo/server/zookeeper/ZooLock.java
index 277dad5..03e84a7 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/zookeeper/ZooLock.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/zookeeper/ZooLock.java
@@ -230,7 +230,7 @@ public class ZooLock implements Watcher {
lostLock(LockLossReason.LOCK_DELETED);
} else if (asyncLock != null && event.getType() == EventType.NodeDeleted && event.getPath().equals(path + "/" + asyncLock)) {
failedToAcquireLock();
- } else if (event.getState() != KeeperState.Expired && (lock != null || asyncLock != null)) {
+ } else if (event.getState() != KeeperState.Disconnected && event.getState() != KeeperState.Expired && (lock != null || asyncLock != null)) {
log.debug("Unexpected event watching lock node "+event+" "+asyncLockPath);
try {
Stat stat2 = zooKeeper.getStatus(asyncLockPath, this);
[16/50] [abbrv] git commit: ACCUMULO-1438 Adding Mini Module
Posted by el...@apache.org.
ACCUMULO-1438 Adding Mini Module
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.4@1484636 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9060cb74
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9060cb74
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9060cb74
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 9060cb74555af860d9dee40d52b77b28e4291e10
Parents: 4f1ffc8
Author: Corey J. Nolet <cj...@apache.org>
Authored: Tue May 21 01:28:39 2013 +0000
Committer: Corey J. Nolet <cj...@apache.org>
Committed: Tue May 21 01:28:39 2013 +0000
----------------------------------------------------------------------
src/mini/pom.xml | 1 +
.../accumulo/mini/MiniAccumuloCluster.java | 364 +++++++++++++++++++
.../accumulo/mini/MiniAccumuloConfig.java | 89 +++++
.../accumulo/mini/MiniAccumuloClusterTest.java | 1 +
src/mini/src/test/resources/log4j.properties | 1 +
5 files changed, 456 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9060cb74/src/mini/pom.xml
----------------------------------------------------------------------
diff --git a/src/mini/pom.xml b/src/mini/pom.xml
new file mode 100644
index 0000000..a6177fa
--- /dev/null
+++ b/src/mini/pom.xml
@@ -0,0 +1 @@
+<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 ht
tp://maven.apache.org/maven-v4_0_0.xsd">
<parent>
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo</artifactId>
<version>1.4.4-SNAPSHOT</version>
<relativePath>../../</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>accumulo-mini</artifactId>
<name>accumulo-mini</name>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<configuration>
<archive>
<manifestSections>
<manifestSection>
<name>accumulo/mini/</name>
<manifestEntries>
<Sealed>true</Sealed>
</manifestEntries>
</manifestSection>
</manifestSections>
</archive>
</configuration>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>zookeeper</artifactId>
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-core</artifactId>
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-start</artifactId>
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-server</artifactId>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</dependency>
<dependency>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
<dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
</dependency>
</dependencies>
</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9060cb74/src/mini/src/main/java/org/apache/accumulo/mini/MiniAccumuloCluster.java
----------------------------------------------------------------------
diff --git a/src/mini/src/main/java/org/apache/accumulo/mini/MiniAccumuloCluster.java b/src/mini/src/main/java/org/apache/accumulo/mini/MiniAccumuloCluster.java
new file mode 100644
index 0000000..ed50755
--- /dev/null
+++ b/src/mini/src/main/java/org/apache/accumulo/mini/MiniAccumuloCluster.java
@@ -0,0 +1,364 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.mini;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.TimerTask;
+
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.server.logger.LogService;
+import org.apache.accumulo.server.master.Master;
+import org.apache.accumulo.server.tabletserver.TabletServer;
+import org.apache.accumulo.server.util.Initialize;
+import org.apache.accumulo.server.util.PortUtils;
+import org.apache.accumulo.server.util.time.SimpleTimer;
+import org.apache.accumulo.start.Main;
+import org.apache.zookeeper.server.ZooKeeperServerMain;
+
+/**
+ * A utility class that will create Zookeeper and Accumulo processes that write all of their data to a single local directory. This class makes it easy to test
+ * code against a real Accumulo instance. Its much more accurate for testing than MockAccumulo, but much slower than MockAccumulo.
+ *
+ * @since 1.5.0, 1.4.4
+ */
+public class MiniAccumuloCluster {
+
+ private static final String INSTANCE_SECRET = "DONTTELL";
+ private static final String INSTANCE_NAME = "miniInstance";
+
+ private static class LogWriter extends Thread {
+ private BufferedReader in;
+ private BufferedWriter out;
+
+ /**
+ * @throws java.io.IOException
+ */
+ public LogWriter(InputStream stream, File logFile) throws IOException {
+ this.setDaemon(true);
+ this.in = new BufferedReader(new InputStreamReader(stream));
+ out = new BufferedWriter(new FileWriter(logFile));
+
+ SimpleTimer.getInstance().schedule(new TimerTask() {
+ @Override
+ public void run() {
+ try {
+ flush();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }, 1000, 1000);
+ }
+
+ public synchronized void flush() throws IOException {
+ if (out != null)
+ out.flush();
+ }
+
+ @Override
+ public void run() {
+ String line;
+
+ try {
+ while ((line = in.readLine()) != null) {
+ out.append(line);
+ out.append("\n");
+ }
+
+ synchronized (this) {
+ out.close();
+ out = null;
+ in.close();
+ }
+
+ } catch (IOException e) {
+ }
+ }
+ }
+
+ private File libDir;
+ private File confDir;
+ private File zooKeeperDir;
+ private File accumuloDir;
+ private File zooCfgFile;
+ private File logDir;
+ private File walogDir;
+
+ private Process zooKeeperProcess;
+ private Process masterProcess;
+ private Process loggerProcess;
+
+ private int zooKeeperPort;
+
+ private List<LogWriter> logWriters = new ArrayList<LogWriter>();
+
+ private MiniAccumuloConfig config;
+ private Process[] tabletServerProcesses;
+
+ Process exec(Class<? extends Object> clazz, String... args) throws IOException {
+ String javaHome = System.getProperty("java.home");
+ String javaBin = javaHome + File.separator + "bin" + File.separator + "java";
+ String classpath = System.getProperty("java.class.path");
+
+ classpath = confDir.getAbsolutePath() + File.pathSeparator + classpath;
+
+ String className = clazz.getCanonicalName();
+
+ ArrayList<String> argList = new ArrayList<String>();
+
+ argList.addAll(Arrays.asList(javaBin, "-cp", classpath, "-Xmx128m", "-XX:+UseConcMarkSweepGC", "-XX:CMSInitiatingOccupancyFraction=75",
+ Main.class.getName(), className));
+
+ argList.addAll(Arrays.asList(args));
+
+ ProcessBuilder builder = new ProcessBuilder(argList);
+
+ builder.environment().put("ACCUMULO_HOME", config.getDir().getAbsolutePath());
+ builder.environment().put("ACCUMULO_LOG_DIR", logDir.getAbsolutePath());
+
+ // if we're running under accumulo.start, we forward these env vars
+ String env = System.getenv("HADOOP_PREFIX");
+ if (env != null)
+ builder.environment().put("HADOOP_PREFIX", env);
+ env = System.getenv("ZOOKEEPER_HOME");
+ if (env != null)
+ builder.environment().put("ZOOKEEPER_HOME", env);
+
+ Process process = builder.start();
+
+ LogWriter lw;
+ lw = new LogWriter(process.getErrorStream(), new File(logDir, clazz.getSimpleName() + "_" + process.hashCode() + ".err"));
+ logWriters.add(lw);
+ lw.start();
+ lw = new LogWriter(process.getInputStream(), new File(logDir, clazz.getSimpleName() + "_" + process.hashCode() + ".out"));
+ logWriters.add(lw);
+ lw.start();
+
+ return process;
+ }
+
+ private void appendProp(FileWriter fileWriter, Property key, String value, Map<String,String> siteConfig) throws IOException {
+ appendProp(fileWriter, key.getKey(), value, siteConfig);
+ }
+
+ private void appendProp(FileWriter fileWriter, String key, String value, Map<String,String> siteConfig) throws IOException {
+ if (!siteConfig.containsKey(key))
+ fileWriter.append("<property><name>" + key + "</name><value>" + value + "</value></property>\n");
+ }
+
+ /**
+ * @param dir
+ * An empty or nonexistant temp directoy that Accumulo and Zookeeper can store data in. Creating the directory is left to the user. Java 7, Guava,
+ * and Junit provide methods for creating temporary directories.
+ * @param rootPassword
+ * Initial root password for instance.
+ * @throws IOException
+ */
+ public MiniAccumuloCluster(File dir, String rootPassword) throws IOException {
+ this(new MiniAccumuloConfig(dir, rootPassword));
+ }
+
+ /**
+ * @param config
+ * initial configuration
+ * @throws IOException
+ */
+
+ public MiniAccumuloCluster(MiniAccumuloConfig config) throws IOException {
+
+ if (config.getDir().exists() && !config.getDir().isDirectory())
+ throw new IllegalArgumentException("Must pass in directory, " + config.getDir() + " is a file");
+
+ if (config.getDir().exists() && config.getDir().list().length != 0)
+ throw new IllegalArgumentException("Directory " + config.getDir() + " is not empty");
+
+ this.config = config;
+
+ libDir = new File(config.getDir(), "lib");
+ confDir = new File(config.getDir(), "conf");
+ accumuloDir = new File(config.getDir(), "accumulo");
+ zooKeeperDir = new File(config.getDir(), "zookeeper");
+ logDir = new File(config.getDir(), "logs");
+ walogDir = new File(config.getDir(), "walogs");
+
+ confDir.mkdirs();
+ accumuloDir.mkdirs();
+ zooKeeperDir.mkdirs();
+ logDir.mkdirs();
+ walogDir.mkdirs();
+ libDir.mkdirs();
+
+ zooKeeperPort = PortUtils.getRandomFreePort();
+
+ File siteFile = new File(confDir, "accumulo-site.xml");
+
+ FileWriter fileWriter = new FileWriter(siteFile);
+ fileWriter.append("<configuration>\n");
+
+ HashMap<String,String> siteConfig = new HashMap<String,String>(config.getSiteConfig());
+
+ appendProp(fileWriter, Property.INSTANCE_DFS_URI, "file:///", siteConfig);
+ appendProp(fileWriter, Property.INSTANCE_DFS_DIR, accumuloDir.getAbsolutePath(), siteConfig);
+ appendProp(fileWriter, Property.INSTANCE_ZK_HOST, "localhost:" + zooKeeperPort, siteConfig);
+ appendProp(fileWriter, Property.INSTANCE_SECRET, INSTANCE_SECRET, siteConfig);
+ appendProp(fileWriter, Property.MASTER_CLIENTPORT, "" + PortUtils.getRandomFreePort(), siteConfig);
+ appendProp(fileWriter, Property.TSERV_CLIENTPORT, "" + PortUtils.getRandomFreePort(), siteConfig);
+ appendProp(fileWriter, Property.TSERV_PORTSEARCH, "true", siteConfig);
+ appendProp(fileWriter, Property.LOGGER_DIR, walogDir.getAbsolutePath(), siteConfig);
+ appendProp(fileWriter, Property.TSERV_DATACACHE_SIZE, "10M", siteConfig);
+ appendProp(fileWriter, Property.TSERV_INDEXCACHE_SIZE, "10M", siteConfig);
+ appendProp(fileWriter, Property.TSERV_MAXMEM, "50M", siteConfig);
+ appendProp(fileWriter, Property.TSERV_WALOG_MAX_SIZE, "100M", siteConfig);
+ appendProp(fileWriter, Property.TSERV_NATIVEMAP_ENABLED, "false", siteConfig);
+ appendProp(fileWriter, Property.TRACE_PORT, "" + PortUtils.getRandomFreePort(), siteConfig);
+ appendProp(fileWriter, Property.LOGGER_SORT_BUFFER_SIZE, "50M", siteConfig);
+ appendProp(fileWriter, Property.LOGGER_PORTSEARCH, "true", siteConfig);
+
+ // since there is a small amount of memory, check more frequently for majc... setting may not be needed in 1.5
+ appendProp(fileWriter, Property.TSERV_MAJC_DELAY, "3", siteConfig);
+ String cp = System.getenv("ACCUMULO_HOME") + "/lib/.*.jar," + "$ZOOKEEPER_HOME/zookeeper[^.].*.jar," + "$HADOOP_HOME/[^.].*.jar,"
+ + "$HADOOP_HOME/lib/[^.].*.jar," + "$HADOOP_PREFIX/share/hadoop/common/.*.jar," + "$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,"
+ + "$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar," + "$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar";
+ appendProp(fileWriter, Property.GENERAL_CLASSPATHS, cp, siteConfig);
+ appendProp(fileWriter, Property.GENERAL_DYNAMIC_CLASSPATHS, libDir.getAbsolutePath(), siteConfig);
+
+ for (Entry<String,String> entry : siteConfig.entrySet())
+ fileWriter.append("<property><name>" + entry.getKey() + "</name><value>" + entry.getValue() + "</value></property>\n");
+ fileWriter.append("</configuration>\n");
+ fileWriter.close();
+
+ zooCfgFile = new File(confDir, "zoo.cfg");
+ fileWriter = new FileWriter(zooCfgFile);
+
+ // zookeeper uses Properties to read its config, so use that to write in order to properly escape things like Windows paths
+ Properties zooCfg = new Properties();
+ zooCfg.setProperty("tickTime", "1000");
+ zooCfg.setProperty("initLimit", "10");
+ zooCfg.setProperty("syncLimit", "5");
+ zooCfg.setProperty("clientPort", zooKeeperPort + "");
+ zooCfg.setProperty("maxClientCnxns", "100");
+ zooCfg.setProperty("dataDir", zooKeeperDir.getAbsolutePath());
+ zooCfg.store(fileWriter, null);
+
+ fileWriter.close();
+ }
+
+ /**
+ * Starts Accumulo and Zookeeper processes. Can only be called once.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws IllegalStateException
+ * if already started
+ */
+ public void start() throws IOException, InterruptedException {
+ if (zooKeeperProcess != null)
+ throw new IllegalStateException("Already started");
+
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ try {
+ MiniAccumuloCluster.this.stop();
+ } catch (IOException e) {
+ e.printStackTrace();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ });
+
+ zooKeeperProcess = exec(Main.class, ZooKeeperServerMain.class.getName(), zooCfgFile.getAbsolutePath());
+
+ // sleep a little bit to let zookeeper come up before calling init, seems to work better
+ UtilWaitThread.sleep(250);
+
+ // TODO initialization could probably be done in process
+ Process initProcess = exec(Initialize.class);
+ initProcess.getOutputStream().write((INSTANCE_NAME + "\n").getBytes());
+ initProcess.getOutputStream().write((config.getRootPassword() + "\n").getBytes());
+ initProcess.getOutputStream().write((config.getRootPassword() + "\n").getBytes());
+ initProcess.getOutputStream().flush();
+ int ret = initProcess.waitFor();
+ if (ret != 0) {
+ throw new RuntimeException("Initialize process returned " + ret);
+ }
+
+ tabletServerProcesses = new Process[config.getNumTservers()];
+ for (int i = 0; i < config.getNumTservers(); i++) {
+ tabletServerProcesses[i] = exec(TabletServer.class);
+ }
+
+ loggerProcess = exec(LogService.class);
+ masterProcess = exec(Master.class);
+ }
+
+ /**
+ * @return Accumulo instance name
+ */
+
+ public String getInstanceName() {
+ return INSTANCE_NAME;
+ }
+
+ /**
+ * @return zookeeper connection string
+ */
+
+ public String getZooKeepers() {
+ return "localhost:" + zooKeeperPort;
+ }
+
+ /**
+ * Stops Accumulo and Zookeeper processes. If stop is not called, there is a shutdown hook that is setup to kill the processes. Howerver its probably best to
+ * call stop in a finally block as soon as possible.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ */
+
+ public void stop() throws IOException, InterruptedException {
+ if (zooKeeperProcess != null)
+ zooKeeperProcess.destroy();
+ if (loggerProcess != null)
+ loggerProcess.destroy();
+ if (masterProcess != null)
+ masterProcess.destroy();
+ if (tabletServerProcesses != null) {
+ for (Process tserver : tabletServerProcesses) {
+ tserver.destroy();
+ }
+ }
+
+ for (LogWriter lw : logWriters)
+ lw.flush();
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9060cb74/src/mini/src/main/java/org/apache/accumulo/mini/MiniAccumuloConfig.java
----------------------------------------------------------------------
diff --git a/src/mini/src/main/java/org/apache/accumulo/mini/MiniAccumuloConfig.java b/src/mini/src/main/java/org/apache/accumulo/mini/MiniAccumuloConfig.java
new file mode 100644
index 0000000..7472b13
--- /dev/null
+++ b/src/mini/src/main/java/org/apache/accumulo/mini/MiniAccumuloConfig.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.mini;
+
+import java.io.File;
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * Holds configuration for {@link MiniAccumuloCluster}. Required configurations must be passed to constructor and all other configurations are optional.
+ *
+ * @since 1.5.0, 1.4.4
+ */
+
+public class MiniAccumuloConfig {
+
+ private File dir = null;
+ private String rootPassword = null;
+ private Map<String,String> siteConfig = Collections.emptyMap();
+ private int numTservers = 2;
+
+ /**
+ * @param dir
+ * An empty or nonexistant temp directoy that Accumulo and Zookeeper can store data in. Creating the directory is left to the user. Java 7, Guava,
+ * and Junit provide methods for creating temporary directories.
+ * @param rootPassword
+ * The initial password for the Accumulo root user
+ */
+
+ public MiniAccumuloConfig(File dir, String rootPassword) {
+ this.dir = dir;
+ this.rootPassword = rootPassword;
+ }
+
+ public File getDir() {
+ return dir;
+ }
+
+ public String getRootPassword() {
+ return rootPassword;
+ }
+
+ public int getNumTservers() {
+ return numTservers;
+ }
+
+ /**
+ * Calling this method is optional. If not set, it defaults to two.
+ *
+ * @param numTservers
+ * the number of tablet servers that mini accumulo cluster should start
+ */
+
+ public MiniAccumuloConfig setNumTservers(int numTservers) {
+ if (numTservers < 1)
+ throw new IllegalArgumentException("Must have at least one tablet server");
+ this.numTservers = numTservers;
+ return this;
+ }
+
+ public Map<String,String> getSiteConfig() {
+ return siteConfig;
+ }
+
+ /**
+ * Calling this method is optional. If not set, it defautls to an empty map.
+ *
+ * @param siteConfig
+ * key/values that you normally put in accumulo-site.xml can be put here
+ */
+ public MiniAccumuloConfig setSiteConfig(Map<String,String> siteConfig) {
+ this.siteConfig = siteConfig;
+ return this;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9060cb74/src/mini/src/test/java/org/apache/accumulo/mini/MiniAccumuloClusterTest.java
----------------------------------------------------------------------
diff --git a/src/mini/src/test/java/org/apache/accumulo/mini/MiniAccumuloClusterTest.java b/src/mini/src/test/java/org/apache/accumulo/mini/MiniAccumuloClusterTest.java
new file mode 100644
index 0000000..041eb2a
--- /dev/null
+++ b/src/mini/src/test/java/org/apache/accumulo/mini/MiniAccumuloClusterTest.java
@@ -0,0 +1 @@
+/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.accumulo.mini;
import java.util.Collections;
import java.util.Map.Entry;
import java.util.UUID;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.ac
cumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.ZooKeeperInstance;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.LongCombiner;
import org.apache.accumulo.core.iterators.user.SummingCombiner;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.accumulo.core.security.TablePermission;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.rules.TemporaryFolde
r;
public class MiniAccumuloClusterTest {
private static final Logger logger = Logger.getLogger(MiniAccumuloClusterTest.class);
private static TemporaryFolder tmpDir = new TemporaryFolder();
private static MiniAccumuloCluster accumulo;
@BeforeClass
public static void setupMiniCluster() throws Exception {
tmpDir.create();
logger.info("MiniCluster started @ " + tmpDir.getRoot());
Logger.getLogger("org.apache.zookeeper").setLevel(Level.ERROR);
accumulo = new MiniAccumuloCluster(tmpDir.getRoot(), "superSecret");
accumulo.start();
}
@Test(timeout = 30000)
public void test() throws Exception {
Connector conn = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers()).getConnector("root", "superSecret".getBytes());
conn.tableOperations().create("table1");
conn.securityOperations().createUser("user1", "pass1".getBytes(), new Authorizations("A", "B"));
conn.securityOperations().grantTablePermission("user1", "table
1", TablePermission.WRITE);
conn.securityOperations().grantTablePermission("user1", "table1", TablePermission.READ);
IteratorSetting is = new IteratorSetting(10, SummingCombiner.class);
SummingCombiner.setEncodingType(is, LongCombiner.Type.STRING);
SummingCombiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("META", "COUNT")));
conn.tableOperations().attachIterator("table1", is);
Connector uconn = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers()).getConnector("user1", "pass1".getBytes());
BatchWriter bw = uconn.createBatchWriter("table1", 10000, 1000000, 2);
UUID uuid = UUID.randomUUID();
Mutation m = new Mutation(uuid.toString());
m.put("META", "SIZE", new ColumnVisibility("A|B"), "8");
m.put("META", "CRC", new ColumnVisibility("A|B"), "456");
m.put("META", "COUNT", new ColumnVisibility("A|B"), "1");
m.put("DATA", "IMG", new ColumnVisibility("A&B"), "ABCDEFGH");
bw.addMut
ation(m);
bw.flush();
m = new Mutation(uuid.toString());
m.put("META", "COUNT", new ColumnVisibility("A|B"), "1");
m.put("META", "CRC", new ColumnVisibility("A|B"), "123");
bw.addMutation(m);
bw.close();
int count = 0;
Scanner scanner = uconn.createScanner("table1", new Authorizations("A"));
for (Entry<Key,Value> entry : scanner) {
if (entry.getKey().getColumnQualifierData().toString().equals("COUNT")) {
Assert.assertEquals("2", entry.getValue().toString());
} else if (entry.getKey().getColumnQualifierData().toString().equals("SIZE")) {
Assert.assertEquals("8", entry.getValue().toString());
} else if (entry.getKey().getColumnQualifierData().toString().equals("CRC")) {
Assert.assertEquals("123", entry.getValue().toString());
} else {
Assert.assertTrue(false);
}
count++;
}
Assert.assertEquals(3, count);
count = 0;
scanner = uconn.createScanner("table1", new Auth
orizations("A", "B"));
for (Entry<Key,Value> entry : scanner) {
if (entry.getKey().getColumnQualifierData().toString().equals("IMG")) {
Assert.assertEquals("ABCDEFGH", entry.getValue().toString());
}
count++;
}
Assert.assertEquals(4, count);
conn.tableOperations().delete("table1");
}
@Test(timeout = 20000)
public void testMultipleTabletServersRunning() throws AccumuloException, AccumuloSecurityException {
Connector conn = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers()).getConnector("root", "superSecret".getBytes());
while (conn.instanceOperations().getTabletServers().size() != 2) {
UtilWaitThread.sleep(500);
}
}
@AfterClass
public static void tearDownMiniCluster() throws Exception {
accumulo.stop();
tmpDir.delete();
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9060cb74/src/mini/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/src/mini/src/test/resources/log4j.properties b/src/mini/src/test/resources/log4j.properties
new file mode 100644
index 0000000..c833d57
--- /dev/null
+++ b/src/mini/src/test/resources/log4j.properties
@@ -0,0 +1 @@
+log4j.rootLogger=INFO, CA
log4j.appender.CA=org.apache.log4j.ConsoleAppender
log4j.appender.CA.layout=org.apache.log4j.PatternLayout
log4j.appender.CA.layout.ConversionPattern=[%t] %-5p %c %x - %m%n
log4j.logger.org.apache.zookeeper=ERROR,CA
log4j.logger.org.apache.accumulo.core.client.impl.ServerClient=ERROR
\ No newline at end of file
[23/50] [abbrv] git commit: ACCUMULO-1473 Fixed multiple bugs in
FindOfflineTablets
Posted by el...@apache.org.
ACCUMULO-1473 Fixed multiple bugs in FindOfflineTablets
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.4@1487877 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e5dad25f
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e5dad25f
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e5dad25f
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: e5dad25fb62544cd271a06535a801288e7482c15
Parents: 730e330
Author: Keith Turner <kt...@apache.org>
Authored: Thu May 30 14:37:13 2013 +0000
Committer: Keith Turner <kt...@apache.org>
Committed: Thu May 30 14:37:13 2013 +0000
----------------------------------------------------------------------
.../server/util/FindOfflineTablets.java | 30 ++++++++++++++------
1 file changed, 22 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e5dad25f/src/server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java b/src/server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
index e346df1..bb65971 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
@@ -16,11 +16,13 @@
*/
package org.apache.accumulo.server.util;
+import java.util.Iterator;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.ZooKeeperInstance;
-import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.master.state.tables.TableState;
import org.apache.accumulo.server.master.LiveTServerSet;
import org.apache.accumulo.server.master.LiveTServerSet.Listener;
@@ -30,6 +32,7 @@ import org.apache.accumulo.server.master.state.TabletLocationState;
import org.apache.accumulo.server.master.state.TabletState;
import org.apache.accumulo.server.master.state.tables.TableManager;
import org.apache.accumulo.server.security.SecurityConstants;
+import org.apache.commons.collections.iterators.IteratorChain;
import org.apache.log4j.Logger;
public class FindOfflineTablets {
@@ -39,28 +42,39 @@ public class FindOfflineTablets {
* @param args
*/
public static void main(String[] args) throws Exception {
- if (args.length != 4) {
- System.err.println("Usage: accumulo.server.util.FindOfflineTablets instance zookeepers");
+ if (args.length != 2) {
+ System.err.println("Usage: " + FindOfflineTablets.class.getName() + " <instance> <zookeepers>");
System.exit(1);
}
String instance = args[0];
String keepers = args[1];
Instance zooInst = new ZooKeeperInstance(instance, keepers);
- MetaDataTableScanner scanner = new MetaDataTableScanner(zooInst, SecurityConstants.getSystemCredentials(), new Range());
+ MetaDataTableScanner rootScanner = new MetaDataTableScanner(zooInst, SecurityConstants.getSystemCredentials(),
+ Constants.ROOT_TABLET_EXTENT.toMetadataRange());
+ MetaDataTableScanner metaScanner = new MetaDataTableScanner(zooInst, SecurityConstants.getSystemCredentials(), Constants.NON_ROOT_METADATA_KEYSPACE);
+ @SuppressWarnings("unchecked")
+ Iterator<TabletLocationState> scanner = (Iterator<TabletLocationState>) new IteratorChain(rootScanner, metaScanner);
+ final AtomicBoolean scanning = new AtomicBoolean(false);
LiveTServerSet tservers = new LiveTServerSet(zooInst, new Listener() {
@Override
public void update(LiveTServerSet current, Set<TServerInstance> deleted, Set<TServerInstance> added) {
- if (!deleted.isEmpty())
+ if (!deleted.isEmpty() && scanning.get())
log.warn("Tablet servers deleted while scanning: " + deleted);
- if (!added.isEmpty())
+ if (!added.isEmpty() && scanning.get())
log.warn("Tablet servers added while scanning: " + added);
}
});
+
+ tservers.startListeningForTabletServerChanges();
+ scanning.set(true);
+
while (scanner.hasNext()) {
TabletLocationState locationState = scanner.next();
TabletState state = locationState.getState(tservers.getCurrentServers());
- if (state != TabletState.HOSTED && TableManager.getInstance().getTableState(locationState.extent.getTableId().toString()) != TableState.OFFLINE)
- System.out.println(locationState + " is " + state);
+ if (state != null && state != TabletState.HOSTED
+ && TableManager.getInstance().getTableState(locationState.extent.getTableId().toString()) != TableState.OFFLINE)
+ if (!locationState.extent.equals(Constants.ROOT_TABLET_EXTENT))
+ System.out.println(locationState + " is " + state + " #walogs:" + locationState.walogs.size());
}
}
[02/50] [abbrv] git commit: ACCUMULO-1365 use the MAC mechanism for
finding a random port
Posted by el...@apache.org.
ACCUMULO-1365 use the MAC mechanism for finding a random port
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.4@1477797 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/73c34230
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/73c34230
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/73c34230
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 73c34230c14c3e8217c014e6600a78692a4faeaa
Parents: bb71618
Author: Eric C. Newton <ec...@apache.org>
Authored: Tue Apr 30 19:48:29 2013 +0000
Committer: Eric C. Newton <ec...@apache.org>
Committed: Tue Apr 30 19:48:29 2013 +0000
----------------------------------------------------------------------
.../src/test/java/org/apache/accumulo/proxy/SimpleTest.java | 5 ++---
.../main/java/org/apache/accumulo/test/MiniAccumuloCluster.java | 2 +-
2 files changed, 3 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/73c34230/src/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
----------------------------------------------------------------------
diff --git a/src/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java b/src/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
index 744eef4..5474947 100644
--- a/src/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
+++ b/src/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
@@ -120,8 +120,7 @@ public class SimpleTest {
protocolFactories.add(org.apache.thrift.protocol.TBinaryProtocol.Factory.class);
protocolFactories.add(org.apache.thrift.protocol.TCompactProtocol.Factory.class);
- Random rand = new Random();
- return protocolFactories.get(rand.nextInt(protocolFactories.size()));
+ return protocolFactories.get(random.nextInt(protocolFactories.size()));
}
@BeforeClass
@@ -137,7 +136,7 @@ public class SimpleTest {
protocolClass = getRandomProtocol();
System.out.println(protocolClass.getName());
- proxyPort = 40000 + random.nextInt(20000);
+ proxyPort = MiniAccumuloCluster.getRandomFreePort();
proxyServer = Proxy.createProxyServer(org.apache.accumulo.proxy.thrift.AccumuloProxy.class, org.apache.accumulo.proxy.ProxyServer.class, proxyPort,
protocolClass, props);
thread = new Thread() {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/73c34230/src/test/src/main/java/org/apache/accumulo/test/MiniAccumuloCluster.java
----------------------------------------------------------------------
diff --git a/src/test/src/main/java/org/apache/accumulo/test/MiniAccumuloCluster.java b/src/test/src/main/java/org/apache/accumulo/test/MiniAccumuloCluster.java
index eb6feb2..e490a3b 100644
--- a/src/test/src/main/java/org/apache/accumulo/test/MiniAccumuloCluster.java
+++ b/src/test/src/main/java/org/apache/accumulo/test/MiniAccumuloCluster.java
@@ -124,7 +124,7 @@ public class MiniAccumuloCluster {
private MiniAccumuloConfig config;
private Process[] tabletServerProcesses;
- private int getRandomFreePort() {
+ static public int getRandomFreePort() {
Random r = new Random();
int count = 0;
[08/50] [abbrv] git commit: ACCUMULO-1227 test was failing
Posted by el...@apache.org.
ACCUMULO-1227 test was failing
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.4@1480121 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/0e463d47
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/0e463d47
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/0e463d47
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 0e463d478b2ce06516153d0bceac5b1d5a2ca6a9
Parents: 82511dc
Author: Eric C. Newton <ec...@apache.org>
Authored: Tue May 7 22:24:53 2013 +0000
Committer: Eric C. Newton <ec...@apache.org>
Committed: Tue May 7 22:24:53 2013 +0000
----------------------------------------------------------------------
.../org/apache/accumulo/server/test/functional/LargeRowTest.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0e463d47/src/server/src/main/java/org/apache/accumulo/server/test/functional/LargeRowTest.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/test/functional/LargeRowTest.java b/src/server/src/main/java/org/apache/accumulo/server/test/functional/LargeRowTest.java
index d387aa4..bc02929 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/test/functional/LargeRowTest.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/test/functional/LargeRowTest.java
@@ -42,8 +42,8 @@ public class LargeRowTest extends FunctionalTest {
private static final String PRE_SPLIT_TABLE_NAME = "lrps";
private static final int NUM_ROWS = 100;
private static final int ROW_SIZE = 1 << 17;
- private static final int SPLIT_THRESH = ROW_SIZE * NUM_ROWS / 5;
private static final int NUM_PRE_SPLITS = 9;
+ private static final int SPLIT_THRESH = ROW_SIZE * NUM_ROWS / NUM_PRE_SPLITS;
@Override
public void cleanup() {}
@@ -95,7 +95,7 @@ public class LargeRowTest extends FunctionalTest {
UtilWaitThread.sleep(10000);
- checkSplits(REG_TABLE_NAME, 1, 9);
+ checkSplits(REG_TABLE_NAME, NUM_PRE_SPLITS / 2, NUM_PRE_SPLITS);
verify(REG_TABLE_NAME);
}
[45/50] [abbrv] git commit: ACCUMULO-1603 Merge branch
'1.4.4-SNAPSHOT' into 1.5.1-SNAPSHOT
Posted by el...@apache.org.
ACCUMULO-1603 Merge branch '1.4.4-SNAPSHOT' into 1.5.1-SNAPSHOT
Conflicts:
.gitignore
src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f5b2efa0
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f5b2efa0
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f5b2efa0
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: f5b2efa0a22250809702e5ed2885dc99f49703a6
Parents: b03b864 108c492
Author: Josh Elser <el...@apache.org>
Authored: Tue Jul 23 20:43:02 2013 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Tue Jul 23 20:43:02 2013 -0400
----------------------------------------------------------------------
.gitignore | 11 +++++++++++
.../org/apache/accumulo/server/trace/TraceServer.java | 2 +-
2 files changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f5b2efa0/.gitignore
----------------------------------------------------------------------
diff --cc .gitignore
index d36204e,5c55d79..7afd1e5
--- a/.gitignore
+++ b/.gitignore
@@@ -74,3 -162,14 +74,14 @@@ target
/test/system/continuous/walkers.txt
/test/system/continuous/ingesters.txt
+ # src/minicluster
-src/minicluster/.classpath
-src/minicluster/.project
-src/minicluster/.settings
-src/minicluster/target
++minicluster/.classpath
++minicluster/.project
++minicluster/.settings
++minicluster/target
+
+ # src/proxy
-src/proxy/.classpath
-src/proxy/.project
-src/proxy/.settings
-src/proxy/target
++proxy/.classpath
++proxy/.project
++proxy/.settings
++proxy/target
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f5b2efa0/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
index b6cb6f5,0000000..77cdb17
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
+++ b/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
@@@ -1,293 -1,0 +1,293 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.trace;
+
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.nio.channels.ServerSocketChannel;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileUtil;
+import org.apache.accumulo.core.iterators.user.AgeOffFilter;
+import org.apache.accumulo.core.security.SecurityUtil;
+import org.apache.accumulo.core.trace.TraceFormatter;
+import org.apache.accumulo.core.util.AddressUtil;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.server.Accumulo;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.util.time.SimpleTimer;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.start.classloader.AccumuloClassLoader;
+import org.apache.accumulo.trace.instrument.Span;
+import org.apache.accumulo.trace.thrift.RemoteSpan;
+import org.apache.accumulo.trace.thrift.SpanReceiver.Iface;
+import org.apache.accumulo.trace.thrift.SpanReceiver.Processor;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+import org.apache.thrift.TByteArrayOutputStream;
+import org.apache.thrift.TException;
+import org.apache.thrift.protocol.TCompactProtocol;
+import org.apache.thrift.server.TServer;
+import org.apache.thrift.server.TThreadPoolServer;
+import org.apache.thrift.transport.TServerSocket;
+import org.apache.thrift.transport.TServerTransport;
+import org.apache.thrift.transport.TTransport;
+import org.apache.thrift.transport.TTransportException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.Watcher.Event.EventType;
+import org.apache.zookeeper.Watcher.Event.KeeperState;
+
+public class TraceServer implements Watcher {
+
+ final private static Logger log = Logger.getLogger(TraceServer.class);
+ final private ServerConfiguration serverConfiguration;
+ final private TServer server;
+ private BatchWriter writer = null;
+ private Connector connector;
+ final String table;
+
+ private static void put(Mutation m, String cf, String cq, byte[] bytes, int len) {
+ m.put(new Text(cf), new Text(cq), new Value(bytes, 0, len));
+ }
+
+ static class ByteArrayTransport extends TTransport {
+ TByteArrayOutputStream out = new TByteArrayOutputStream();
+
+ @Override
+ public boolean isOpen() {
+ return true;
+ }
+
+ @Override
+ public void open() throws TTransportException {}
+
+ @Override
+ public void close() {}
+
+ @Override
+ public int read(byte[] buf, int off, int len) {
+ return 0;
+ }
+
+ @Override
+ public void write(byte[] buf, int off, int len) throws TTransportException {
+ out.write(buf, off, len);
+ }
+
+ public byte[] get() {
+ return out.get();
+ }
+
+ public int len() {
+ return out.len();
+ }
+ }
+
+ class Receiver implements Iface {
+ @Override
+ public void span(RemoteSpan s) throws TException {
+ String idString = Long.toHexString(s.traceId);
+ String startString = Long.toHexString(s.start);
+ Mutation spanMutation = new Mutation(new Text(idString));
+ Mutation indexMutation = new Mutation(new Text("idx:" + s.svc + ":" + startString));
+ long diff = s.stop - s.start;
+ indexMutation.put(new Text(s.description), new Text(s.sender), new Value((idString + ":" + Long.toHexString(diff)).getBytes()));
+ ByteArrayTransport transport = new ByteArrayTransport();
+ TCompactProtocol protocol = new TCompactProtocol(transport);
+ s.write(protocol);
+ String parentString = Long.toHexString(s.parentId);
+ if (s.parentId == Span.ROOT_SPAN_ID)
+ parentString = "";
+ put(spanMutation, "span", parentString + ":" + Long.toHexString(s.spanId), transport.get(), transport.len());
+ // Map the root span to time so we can look up traces by time
+ Mutation timeMutation = null;
+ if (s.parentId == Span.ROOT_SPAN_ID) {
+ timeMutation = new Mutation(new Text("start:" + startString));
+ put(timeMutation, "id", idString, transport.get(), transport.len());
+ }
+ try {
+ if (writer == null)
+ resetWriter();
+ if (writer == null)
+ return;
+ writer.addMutation(spanMutation);
+ writer.addMutation(indexMutation);
+ if (timeMutation != null)
+ writer.addMutation(timeMutation);
+ } catch (Exception ex) {
+ log.error("Unable to write mutation to table: " + spanMutation, ex);
+ }
+ }
+
+ }
+
+ public TraceServer(ServerConfiguration serverConfiguration, String hostname) throws Exception {
+ this.serverConfiguration = serverConfiguration;
+ AccumuloConfiguration conf = serverConfiguration.getConfiguration();
+ table = conf.get(Property.TRACE_TABLE);
+ while (true) {
+ try {
+ String principal = conf.get(Property.TRACE_USER);
+ AuthenticationToken at;
+ Map<String,String> loginMap = conf.getAllPropertiesWithPrefix(Property.TRACE_TOKEN_PROPERTY_PREFIX);
+ if (loginMap.isEmpty()) {
+ Property p = Property.TRACE_PASSWORD;
+ at = new PasswordToken(conf.get(p).getBytes());
+ } else {
+ Properties props = new Properties();
+ AuthenticationToken token = AccumuloClassLoader.getClassLoader().loadClass(conf.get(Property.TRACE_TOKEN_TYPE)).asSubclass(AuthenticationToken.class)
+ .newInstance();
+
+ int prefixLength = Property.TRACE_TOKEN_PROPERTY_PREFIX.getKey().length() + 1;
+ for (Entry<String,String> entry : loginMap.entrySet()) {
+ props.put(entry.getKey().substring(prefixLength), entry.getValue());
+ }
+
+ token.init(props);
+
+ at = token;
+ }
+
+ connector = serverConfiguration.getInstance().getConnector(principal, at);
+ if (!connector.tableOperations().exists(table)) {
+ connector.tableOperations().create(table);
+ IteratorSetting setting = new IteratorSetting(10, "ageoff", AgeOffFilter.class.getName());
+ AgeOffFilter.setTTL(setting, 7 * 24 * 60 * 60 * 1000l);
+ connector.tableOperations().attachIterator(table, setting);
+ }
+ connector.tableOperations().setProperty(table, Property.TABLE_FORMATTER_CLASS.getKey(), TraceFormatter.class.getName());
+ break;
+ } catch (Exception ex) {
- log.info("waiting to checking/create the trace table: " + ex);
++ log.info("Waiting to checking/create the trace table.", ex);
+ UtilWaitThread.sleep(1000);
+ }
+ }
+
+ int port = conf.getPort(Property.TRACE_PORT);
+ final ServerSocket sock = ServerSocketChannel.open().socket();
+ sock.setReuseAddress(true);
+ sock.bind(new InetSocketAddress(port));
+ final TServerTransport transport = new TServerSocket(sock);
+ TThreadPoolServer.Args options = new TThreadPoolServer.Args(transport);
+ options.processor(new Processor<Iface>(new Receiver()));
+ server = new TThreadPoolServer(options);
+ final InetSocketAddress address = new InetSocketAddress(hostname, sock.getLocalPort());
+ registerInZooKeeper(AddressUtil.toString(address));
+
+ writer = connector.createBatchWriter(table, new BatchWriterConfig().setMaxLatency(5, TimeUnit.SECONDS));
+ }
+
+ public void run() throws Exception {
+ SimpleTimer.getInstance().schedule(new Runnable() {
+ @Override
+ public void run() {
+ flush();
+ }
+ }, 1000, 1000);
+ server.serve();
+ }
+
+ private void flush() {
+ try {
+ writer.flush();
+ } catch (MutationsRejectedException e) {
+ log.error("Error flushing traces", e);
+ resetWriter();
+ }
+ }
+
+ synchronized private void resetWriter() {
+ try {
+ if (writer != null)
+ writer.close();
+ } catch (Exception ex) {
+ log.error("Error closing batch writer", ex);
+ } finally {
+ writer = null;
+ try {
+ writer = connector.createBatchWriter(table, new BatchWriterConfig());
+ } catch (Exception ex) {
+ log.error("Unable to create a batch writer: " + ex);
+ }
+ }
+ }
+
+ private void registerInZooKeeper(String name) throws Exception {
+ String root = ZooUtil.getRoot(serverConfiguration.getInstance()) + Constants.ZTRACERS;
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ String path = zoo.putEphemeralSequential(root + "/trace-", name.getBytes());
+ zoo.exists(path, this);
+ }
+
+ public static void main(String[] args) throws Exception {
+ SecurityUtil.serverLogin();
+ Instance instance = HdfsZooInstance.getInstance();
+ ServerConfiguration conf = new ServerConfiguration(instance);
+ FileSystem fs = FileUtil.getFileSystem(CachedConfiguration.getInstance(), conf.getConfiguration());
+ Accumulo.init(fs, conf, "tracer");
+ String hostname = Accumulo.getLocalAddress(args);
+ TraceServer server = new TraceServer(conf, hostname);
+ Accumulo.enableTracing(hostname, "tserver");
+ server.run();
+ log.info("tracer stopping");
+ }
+
+ @Override
+ public void process(WatchedEvent event) {
+ log.debug("event " + event.getPath() + " " + event.getType() + " " + event.getState());
+ if (event.getState() == KeeperState.Expired) {
+ log.warn("Trace server lost zookeeper registration at " + event.getPath());
+ server.stop();
+ } else if (event.getType() == EventType.NodeDeleted) {
+ log.warn("Trace server zookeeper entry lost " + event.getPath());
+ server.stop();
+ }
+ if (event.getPath() != null) {
+ try {
+ if (ZooReaderWriter.getInstance().exists(event.getPath(), this))
+ return;
+ } catch (Exception ex) {
+ log.error(ex, ex);
+ }
+ log.warn("Trace server unable to reset watch on zookeeper registration");
+ server.stop();
+ }
+ }
+
+}
[42/50] [abbrv] git commit: ACCUMULO-1603 Update .gitignore to
account for eclipse and maven build dir for minicluster and proxy
Posted by el...@apache.org.
ACCUMULO-1603 Update .gitignore to account for eclipse and maven build dir for
minicluster and proxy
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/513364f5
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/513364f5
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/513364f5
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 513364f575255800824f4065602a1fcaca793a78
Parents: 333062d
Author: Josh Elser <el...@apache.org>
Authored: Tue Jul 23 19:36:55 2013 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Tue Jul 23 19:36:55 2013 -0400
----------------------------------------------------------------------
.gitignore | 12 ++++++++++++
1 file changed, 12 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/513364f5/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index e613a79..5c55d79 100644
--- a/.gitignore
+++ b/.gitignore
@@ -161,3 +161,15 @@
/test/system/continuous/continuous-env.sh
/test/system/continuous/walkers.txt
/test/system/continuous/ingesters.txt
+
+# src/minicluster
+src/minicluster/.classpath
+src/minicluster/.project
+src/minicluster/.settings
+src/minicluster/target
+
+# src/proxy
+src/proxy/.classpath
+src/proxy/.project
+src/proxy/.settings
+src/proxy/target
[31/50] [abbrv] git commit: ACCUMULO-1467 keep proxy in 1.4
consistent w/ 1.5
Posted by el...@apache.org.
ACCUMULO-1467 keep proxy in 1.4 consistent w/ 1.5
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.4@1498901 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/4fd10667
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/4fd10667
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/4fd10667
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 4fd10667d07e360cb4bc4d156d99683b24f0fac7
Parents: a329a7a
Author: Keith Turner <kt...@apache.org>
Authored: Tue Jul 2 12:25:03 2013 +0000
Committer: Keith Turner <kt...@apache.org>
Committed: Tue Jul 2 12:25:03 2013 +0000
----------------------------------------------------------------------
src/proxy/proxy.properties | 2 +-
.../src/main/java/org/apache/accumulo/proxy/Proxy.java | 9 +++++++--
2 files changed, 8 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4fd10667/src/proxy/proxy.properties
----------------------------------------------------------------------
diff --git a/src/proxy/proxy.properties b/src/proxy/proxy.properties
index 8044e09..a57c259 100644
--- a/src/proxy/proxy.properties
+++ b/src/proxy/proxy.properties
@@ -17,7 +17,7 @@ useMockInstance=false
useMiniAccumulo=false
protocolFactory=org.apache.thrift.protocol.TCompactProtocol$Factory
port=42424
+maxFrameSize=16M
instance=test
zookeepers=localhost:2181
-
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4fd10667/src/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java
----------------------------------------------------------------------
diff --git a/src/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java b/src/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java
index a65e5b6..7883d83 100644
--- a/src/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java
+++ b/src/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java
@@ -22,8 +22,9 @@ import java.io.IOException;
import java.lang.reflect.Constructor;
import java.util.Properties;
-import org.apache.accumulo.proxy.thrift.AccumuloProxy;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.minicluster.MiniAccumuloCluster;
+import org.apache.accumulo.proxy.thrift.AccumuloProxy;
import org.apache.log4j.Logger;
import org.apache.thrift.TProcessor;
import org.apache.thrift.protocol.TCompactProtocol;
@@ -114,6 +115,7 @@ public class Proxy {
Class<?> proxyProcClass = Class.forName(api.getName() + "$Processor");
Class<?> proxyIfaceClass = Class.forName(api.getName() + "$Iface");
+
@SuppressWarnings("unchecked")
Constructor<? extends TProcessor> proxyProcConstructor = (Constructor<? extends TProcessor>) proxyProcClass.getConstructor(proxyIfaceClass);
@@ -121,7 +123,10 @@ public class Proxy {
THsHaServer.Args args = new THsHaServer.Args(socket);
args.processor(processor);
- args.transportFactory(new TFramedTransport.Factory());
+ final long maxFrameSize = AccumuloConfiguration.getMemoryInBytes(properties.getProperty("maxFrameSize", "16M"));
+ if (maxFrameSize > Integer.MAX_VALUE)
+ throw new RuntimeException(maxFrameSize + " is larger than MAX_INT");
+ args.transportFactory(new TFramedTransport.Factory((int)maxFrameSize));
args.protocolFactory(protoClass.newInstance());
return new THsHaServer(args);
}
[37/50] [abbrv] git commit: ACCUMULO-1561 updating rpm-maven-plugin
version
Posted by el...@apache.org.
ACCUMULO-1561 updating rpm-maven-plugin version
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.4@1501024 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e4f34c4d
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e4f34c4d
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e4f34c4d
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: e4f34c4decc0c0f5aedd65d4dfca3a47570b1f93
Parents: 5440254
Author: Mike Drob <md...@apache.org>
Authored: Mon Jul 8 23:50:17 2013 +0000
Committer: Mike Drob <md...@apache.org>
Committed: Mon Jul 8 23:50:17 2013 +0000
----------------------------------------------------------------------
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e4f34c4d/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index d1877b1..9ed57b9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -189,7 +189,7 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>rpm-maven-plugin</artifactId>
- <version>2.0-beta-2</version>
+ <version>2.1-alpha-3</version>
<inherited>false</inherited>
<configuration>
<name>accumulo</name>
[36/50] [abbrv] git commit: ACCUMULO-1557 applying Jonathan Hsieh's
patch
Posted by el...@apache.org.
ACCUMULO-1557 applying Jonathan Hsieh's patch
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.4@1500876 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/54402545
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/54402545
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/54402545
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 54402545e8cb3b61222b369bf5fbdbe950387524
Parents: e3d247d
Author: Eric C. Newton <ec...@apache.org>
Authored: Mon Jul 8 18:47:01 2013 +0000
Committer: Eric C. Newton <ec...@apache.org>
Committed: Mon Jul 8 18:47:01 2013 +0000
----------------------------------------------------------------------
test/system/auto/simple/zooCacheTest.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/54402545/test/system/auto/simple/zooCacheTest.py
----------------------------------------------------------------------
diff --git a/test/system/auto/simple/zooCacheTest.py b/test/system/auto/simple/zooCacheTest.py
index e11dec8..040c413 100755
--- a/test/system/auto/simple/zooCacheTest.py
+++ b/test/system/auto/simple/zooCacheTest.py
@@ -14,7 +14,7 @@
# limitations under the License.
import os
-
+import shutil
import unittest
import time
@@ -30,6 +30,7 @@ class ZooCacheTest(TestUtilsMixin, unittest.TestCase):
self.create_config_file(self.settings.copy())
def runTest(self):
+ shutil.rmtree('/tmp/zcTest-42')
handleCC = self.runClassOn('localhost', 'org.apache.accumulo.server.test.functional.CacheTestClean', ['/zcTest-42','/tmp/zcTest-42'])
self.waitForStop(handleCC, 10)
handleR1 = self.runClassOn('localhost', 'org.apache.accumulo.server.test.functional.CacheTestReader', ['/zcTest-42','/tmp/zcTest-42', ZOOKEEPERS])
[29/50] [abbrv] git commit: ACCUMULO-1504 Pulling back Miguel's
changes for IteratorSetting from trunk to 1.4
Posted by el...@apache.org.
ACCUMULO-1504 Pulling back Miguel's changes for IteratorSetting from trunk to 1.4
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.4@1494816 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a91173bc
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a91173bc
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a91173bc
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: a91173bcbb71052f730961e842cbc5b713d58104
Parents: 9b2a56b
Author: Josh Elser <el...@apache.org>
Authored: Thu Jun 20 00:16:14 2013 +0000
Committer: Josh Elser <el...@apache.org>
Committed: Thu Jun 20 00:16:14 2013 +0000
----------------------------------------------------------------------
.../accumulo/core/client/IteratorSetting.java | 46 ++++++++++++
.../core/client/IteratorSettingTest.java | 76 ++++++++++++++++++++
2 files changed, 122 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a91173bc/src/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java b/src/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
index a3ea003..e815072 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
@@ -251,6 +251,52 @@ public class IteratorSetting {
}
/**
+ * @see java.lang.Object#hashCode()
+ */
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((iteratorClass == null) ? 0 : iteratorClass.hashCode());
+ result = prime * result + ((name == null) ? 0 : name.hashCode());
+ result = prime * result + priority;
+ result = prime * result + ((properties == null) ? 0 : properties.hashCode());
+ return result;
+ }
+
+ /**
+ * @see java.lang.Object#equals()
+ */
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (!(obj instanceof IteratorSetting))
+ return false;
+ IteratorSetting other = (IteratorSetting) obj;
+ if (iteratorClass == null) {
+ if (other.iteratorClass != null)
+ return false;
+ } else if (!iteratorClass.equals(other.iteratorClass))
+ return false;
+ if (name == null) {
+ if (other.name != null)
+ return false;
+ } else if (!name.equals(other.name))
+ return false;
+ if (priority != other.priority)
+ return false;
+ if (properties == null) {
+ if (other.properties != null)
+ return false;
+ } else if (!properties.equals(other.properties))
+ return false;
+ return true;
+ }
+
+ /**
* @see java.lang.Object#toString()
*/
@Override
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a91173bc/src/core/src/test/java/org/apache/accumulo/core/client/IteratorSettingTest.java
----------------------------------------------------------------------
diff --git a/src/core/src/test/java/org/apache/accumulo/core/client/IteratorSettingTest.java b/src/core/src/test/java/org/apache/accumulo/core/client/IteratorSettingTest.java
new file mode 100644
index 0000000..69bf1b5
--- /dev/null
+++ b/src/core/src/test/java/org/apache/accumulo/core/client/IteratorSettingTest.java
@@ -0,0 +1,76 @@
+package org.apache.accumulo.core.client;
+
+import static org.junit.Assert.*;
+
+import org.apache.accumulo.core.iterators.Combiner;
+import org.apache.accumulo.core.iterators.DevNull;
+import org.apache.hadoop.io.Writable;
+import org.junit.Test;
+
+/**
+ * Test cases for the IteratorSetting class
+ */
+public class IteratorSettingTest {
+
+ IteratorSetting setting1 = new IteratorSetting(500, "combiner", Combiner.class.getName());
+ IteratorSetting setting2 = new IteratorSetting(500, "combiner", Combiner.class.getName());
+ IteratorSetting setting3 = new IteratorSetting(500, "combiner", Combiner.class.getName());
+ IteratorSetting devnull = new IteratorSetting(500, "devNull", DevNull.class.getName());
+ IteratorSetting nullsetting = null;
+ IteratorSetting setting4 = new IteratorSetting(300, "combiner", Combiner.class.getName());
+ IteratorSetting setting5 = new IteratorSetting(500, "foocombiner", Combiner.class.getName());
+ IteratorSetting setting6 = new IteratorSetting(500, "combiner", "MySuperCombiner");
+
+ @Test
+ public final void testHashCodeSameObject() {
+ assertEquals(setting1.hashCode(), setting1.hashCode());
+ }
+
+ @Test
+ public final void testHashCodeEqualObjects() {
+ assertEquals(setting1.hashCode(), setting2.hashCode());
+ }
+
+ @Test
+ public final void testEqualsObjectReflexive() {
+ assertEquals(setting1, setting1);
+ }
+
+ @Test
+ public final void testEqualsObjectSymmetric() {
+ assertEquals(setting1, setting2);
+ assertEquals(setting2, setting1);
+ }
+
+ @Test
+ public final void testEqualsObjectTransitive() {
+ assertEquals(setting1, setting2);
+ assertEquals(setting2, setting3);
+ assertEquals(setting1, setting3);
+ }
+
+ @Test
+ public final void testEqualsNullSetting() {
+ assertNotEquals(setting1, nullsetting);
+ }
+
+ @Test
+ public final void testEqualsObjectNotEqual() {
+ assertNotEquals(setting1, devnull);
+ }
+
+ @Test
+ public final void testEqualsObjectProperties() {
+ IteratorSetting mysettings = new IteratorSetting(500, "combiner", Combiner.class.getName());
+ assertEquals(setting1, mysettings);
+ mysettings.addOption("myoption1", "myvalue1");
+ assertNotEquals(setting1, mysettings);
+ }
+
+ @Test
+ public final void testEqualsDifferentMembers() {
+ assertNotEquals(setting1, setting4);
+ assertNotEquals(setting1, setting5);
+ assertNotEquals(setting1, setting6);
+ }
+}
[27/50] [abbrv] git commit: ACCUMULO-1505 - fixing up some warnings
in the patch
Posted by el...@apache.org.
ACCUMULO-1505 - fixing up some warnings in the patch
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.4@1491910 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/6b11a329
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/6b11a329
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/6b11a329
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 6b11a329c1de23045cfa040e80fe2503160da88e
Parents: 225d728
Author: John Vines <vi...@apache.org>
Authored: Tue Jun 11 19:13:33 2013 +0000
Committer: John Vines <vi...@apache.org>
Committed: Tue Jun 11 19:13:33 2013 +0000
----------------------------------------------------------------------
.../accumulo/core/client/mock/MockConnectorTest.java | 11 +++++------
1 file changed, 5 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6b11a329/src/core/src/test/java/org/apache/accumulo/core/client/mock/MockConnectorTest.java
----------------------------------------------------------------------
diff --git a/src/core/src/test/java/org/apache/accumulo/core/client/mock/MockConnectorTest.java b/src/core/src/test/java/org/apache/accumulo/core/client/mock/MockConnectorTest.java
index 6fa9dff..1e36502 100644
--- a/src/core/src/test/java/org/apache/accumulo/core/client/mock/MockConnectorTest.java
+++ b/src/core/src/test/java/org/apache/accumulo/core/client/mock/MockConnectorTest.java
@@ -16,9 +16,9 @@
*/
package org.apache.accumulo.core.client.mock;
-import static junit.framework.Assert.assertEquals;
-import static junit.framework.Assert.assertFalse;
-import static junit.framework.Assert.assertTrue;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import java.util.ArrayList;
import java.util.Collections;
@@ -27,8 +27,6 @@ import java.util.List;
import java.util.Map.Entry;
import java.util.Random;
-import junit.framework.Assert;
-
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
@@ -48,6 +46,7 @@ import org.apache.accumulo.core.iterators.Combiner;
import org.apache.accumulo.core.iterators.user.SummingCombiner;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.hadoop.io.Text;
+import org.junit.Assert;
import org.junit.Test;
public class MockConnectorTest {
@@ -108,7 +107,7 @@ public class MockConnectorTest {
Assert.fail("addMutations should throw IAE for null iterable");
} catch (IllegalArgumentException iae) {}
- bw.addMutations(Collections.EMPTY_LIST);
+ bw.addMutations(Collections.<Mutation>emptyList());
Mutation bad = new Mutation("bad");
try {
[03/50] [abbrv] git commit: ACCUMULO-1365 move getRandomFreePort()
out of MAC API
Posted by el...@apache.org.
ACCUMULO-1365 move getRandomFreePort() out of MAC API
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.4@1478018 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7f5d0179
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7f5d0179
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7f5d0179
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 7f5d017940dbd7bd73d278d766549b0f729b9d79
Parents: 73c3423
Author: Keith Turner <kt...@apache.org>
Authored: Wed May 1 14:25:11 2013 +0000
Committer: Keith Turner <kt...@apache.org>
Committed: Wed May 1 14:25:11 2013 +0000
----------------------------------------------------------------------
.../org/apache/accumulo/proxy/SimpleTest.java | 3 +-
.../apache/accumulo/server/util/PortUtils.java | 51 ++++++++++++++++++++
.../accumulo/test/MiniAccumuloCluster.java | 37 ++------------
3 files changed, 58 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7f5d0179/src/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
----------------------------------------------------------------------
diff --git a/src/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java b/src/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
index 5474947..a841bd2 100644
--- a/src/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
+++ b/src/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
@@ -75,6 +75,7 @@ import org.apache.accumulo.proxy.thrift.UnknownScanner;
import org.apache.accumulo.proxy.thrift.UnknownWriter;
import org.apache.accumulo.proxy.thrift.WriterOptions;
import org.apache.accumulo.server.test.functional.SlowIterator;
+import org.apache.accumulo.server.util.PortUtils;
import org.apache.accumulo.test.MiniAccumuloCluster;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -136,7 +137,7 @@ public class SimpleTest {
protocolClass = getRandomProtocol();
System.out.println(protocolClass.getName());
- proxyPort = MiniAccumuloCluster.getRandomFreePort();
+ proxyPort = PortUtils.getRandomFreePort();
proxyServer = Proxy.createProxyServer(org.apache.accumulo.proxy.thrift.AccumuloProxy.class, org.apache.accumulo.proxy.ProxyServer.class, proxyPort,
protocolClass, props);
thread = new Thread() {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7f5d0179/src/server/src/main/java/org/apache/accumulo/server/util/PortUtils.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/util/PortUtils.java b/src/server/src/main/java/org/apache/accumulo/server/util/PortUtils.java
new file mode 100644
index 0000000..f710b0f
--- /dev/null
+++ b/src/server/src/main/java/org/apache/accumulo/server/util/PortUtils.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.io.IOException;
+import java.net.ServerSocket;
+import java.util.Random;
+
+public class PortUtils {
+
+ public static int getRandomFreePort() {
+ Random r = new Random();
+ int count = 0;
+
+ while (count < 13) {
+ int port = r.nextInt((1 << 16) - 1024) + 1024;
+
+ ServerSocket so = null;
+ try {
+ so = new ServerSocket(port);
+ so.setReuseAddress(true);
+ return port;
+ } catch (IOException ioe) {
+
+ } finally {
+ if (so != null)
+ try {
+ so.close();
+ } catch (IOException e) {}
+ }
+
+ }
+
+ throw new RuntimeException("Unable to find port");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7f5d0179/src/test/src/main/java/org/apache/accumulo/test/MiniAccumuloCluster.java
----------------------------------------------------------------------
diff --git a/src/test/src/main/java/org/apache/accumulo/test/MiniAccumuloCluster.java b/src/test/src/main/java/org/apache/accumulo/test/MiniAccumuloCluster.java
index e490a3b..e0e49d4 100644
--- a/src/test/src/main/java/org/apache/accumulo/test/MiniAccumuloCluster.java
+++ b/src/test/src/main/java/org/apache/accumulo/test/MiniAccumuloCluster.java
@@ -23,7 +23,6 @@ import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
-import java.net.ServerSocket;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -31,7 +30,6 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
-import java.util.Random;
import java.util.TimerTask;
import org.apache.accumulo.core.conf.Property;
@@ -40,6 +38,7 @@ import org.apache.accumulo.server.logger.LogService;
import org.apache.accumulo.server.master.Master;
import org.apache.accumulo.server.tabletserver.TabletServer;
import org.apache.accumulo.server.util.Initialize;
+import org.apache.accumulo.server.util.PortUtils;
import org.apache.accumulo.server.util.time.SimpleTimer;
import org.apache.accumulo.start.Main;
import org.apache.zookeeper.server.ZooKeeperServerMain;
@@ -124,32 +123,6 @@ public class MiniAccumuloCluster {
private MiniAccumuloConfig config;
private Process[] tabletServerProcesses;
- static public int getRandomFreePort() {
- Random r = new Random();
- int count = 0;
-
- while (count < 13) {
- int port = r.nextInt((1 << 16) - 1024) + 1024;
-
- ServerSocket so = null;
- try {
- so = new ServerSocket(port);
- so.setReuseAddress(true);
- return port;
- } catch (IOException ioe) {
-
- } finally {
- if (so != null)
- try {
- so.close();
- } catch (IOException e) {}
- }
-
- }
-
- throw new RuntimeException("Unable to find port");
- }
-
Process exec(Class<? extends Object> clazz, String... args) throws IOException {
String javaHome = System.getProperty("java.home");
String javaBin = javaHome + File.separator + "bin" + File.separator + "java";
@@ -243,7 +216,7 @@ public class MiniAccumuloCluster {
walogDir.mkdirs();
libDir.mkdirs();
- zooKeeperPort = getRandomFreePort();
+ zooKeeperPort = PortUtils.getRandomFreePort();
File siteFile = new File(confDir, "accumulo-site.xml");
@@ -256,8 +229,8 @@ public class MiniAccumuloCluster {
appendProp(fileWriter, Property.INSTANCE_DFS_DIR, accumuloDir.getAbsolutePath(), siteConfig);
appendProp(fileWriter, Property.INSTANCE_ZK_HOST, "localhost:" + zooKeeperPort, siteConfig);
appendProp(fileWriter, Property.INSTANCE_SECRET, INSTANCE_SECRET, siteConfig);
- appendProp(fileWriter, Property.MASTER_CLIENTPORT, "" + getRandomFreePort(), siteConfig);
- appendProp(fileWriter, Property.TSERV_CLIENTPORT, "" + getRandomFreePort(), siteConfig);
+ appendProp(fileWriter, Property.MASTER_CLIENTPORT, "" + PortUtils.getRandomFreePort(), siteConfig);
+ appendProp(fileWriter, Property.TSERV_CLIENTPORT, "" + PortUtils.getRandomFreePort(), siteConfig);
appendProp(fileWriter, Property.TSERV_PORTSEARCH, "true", siteConfig);
appendProp(fileWriter, Property.LOGGER_DIR, walogDir.getAbsolutePath(), siteConfig);
appendProp(fileWriter, Property.TSERV_DATACACHE_SIZE, "10M", siteConfig);
@@ -265,7 +238,7 @@ public class MiniAccumuloCluster {
appendProp(fileWriter, Property.TSERV_MAXMEM, "50M", siteConfig);
appendProp(fileWriter, Property.TSERV_WALOG_MAX_SIZE, "100M", siteConfig);
appendProp(fileWriter, Property.TSERV_NATIVEMAP_ENABLED, "false", siteConfig);
- appendProp(fileWriter, Property.TRACE_PORT, "" + getRandomFreePort(), siteConfig);
+ appendProp(fileWriter, Property.TRACE_PORT, "" + PortUtils.getRandomFreePort(), siteConfig);
appendProp(fileWriter, Property.LOGGER_SORT_BUFFER_SIZE, "50M", siteConfig);
appendProp(fileWriter, Property.LOGGER_PORTSEARCH, "true", siteConfig);
[28/50] [abbrv] git commit: ACCUMULO-1518 Patch from David M. Lyle
that allows opening rfiles w/ multiple dots in name.
Posted by el...@apache.org.
ACCUMULO-1518 Patch from David M. Lyle that allows opening rfiles w/ multiple dots in name.
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.4@1493899 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9b2a56b2
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9b2a56b2
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9b2a56b2
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 9b2a56b2f0d538f00e733399396c17fe55322dc8
Parents: 6b11a32
Author: Keith Turner <kt...@apache.org>
Authored: Mon Jun 17 19:36:52 2013 +0000
Committer: Keith Turner <kt...@apache.org>
Committed: Mon Jun 17 19:36:52 2013 +0000
----------------------------------------------------------------------
.../accumulo/core/file/FileOperations.java | 5 +-
.../accumulo/core/file/FileOperationsTest.java | 67 ++++++++++++++++++++
2 files changed, 69 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b2a56b2/src/core/src/main/java/org/apache/accumulo/core/file/FileOperations.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/file/FileOperations.java b/src/core/src/main/java/org/apache/accumulo/core/file/FileOperations.java
index 9f60725..17e540b 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/file/FileOperations.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/file/FileOperations.java
@@ -44,14 +44,13 @@ class DispatchingFileFactory extends FileOperations {
if (name.startsWith(Constants.MAPFILE_EXTENSION + "_")) {
return new MapFileOperations();
}
-
String[] sp = name.split("\\.");
- if (sp.length != 2) {
+ if (sp.length < 2) {
throw new IllegalArgumentException("File name " + name + " has no extension");
}
- String extension = sp[1];
+ String extension = sp[sp.length - 1];
if (extension.equals(Constants.MAPFILE_EXTENSION) || extension.equals(Constants.MAPFILE_EXTENSION + "_tmp")) {
return new MapFileOperations();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b2a56b2/src/core/src/test/java/org/apache/accumulo/core/file/FileOperationsTest.java
----------------------------------------------------------------------
diff --git a/src/core/src/test/java/org/apache/accumulo/core/file/FileOperationsTest.java b/src/core/src/test/java/org/apache/accumulo/core/file/FileOperationsTest.java
new file mode 100644
index 0000000..30c667d
--- /dev/null
+++ b/src/core/src/test/java/org/apache/accumulo/core/file/FileOperationsTest.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.core.file;
+
+import static org.junit.Assert.assertFalse;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.file.rfile.RFile;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.junit.Test;
+
+public class FileOperationsTest {
+
+ public FileOperationsTest() {}
+
+ /**
+ * Test for filenames with +1 dot
+ */
+ @Test
+ public void handlesFilenamesWithMoreThanOneDot() throws IOException {
+
+ Boolean caughtException = false;
+ FileSKVWriter writer = null;
+ String filename = "target/test.file." + RFile.EXTENSION;
+ File testFile = new File(filename);
+ if (testFile.exists()) {
+ FileUtils.forceDelete(testFile);
+ }
+ try {
+ FileOperations fileOperations = FileOperations.getInstance();
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.getLocal(conf);
+ AccumuloConfiguration acuconf = AccumuloConfiguration.getDefaultConfiguration();
+ writer = fileOperations.openWriter(filename, fs, conf, acuconf);
+ writer.close();
+ } catch (Exception ex) {
+ caughtException = true;
+ } finally {
+ if (writer != null) {
+ writer.close();
+ }
+ FileUtils.forceDelete(testFile);
+ }
+
+ assertFalse("Should not throw with more than 1 dot in filename.", caughtException);
+ }
+}
[06/50] [abbrv] git commit: ACCUMULO-14 move MAC to mini package
Posted by el...@apache.org.
ACCUMULO-14 move MAC to mini package
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.4@1479114 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/87cd6236
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/87cd6236
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/87cd6236
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 87cd6236822b37a760bb0c4fb23dc89633943017
Parents: 9637704
Author: Keith Turner <kt...@apache.org>
Authored: Sat May 4 14:20:20 2013 +0000
Committer: Keith Turner <kt...@apache.org>
Committed: Sat May 4 14:20:20 2013 +0000
----------------------------------------------------------------------
.../java/org/apache/accumulo/proxy/Proxy.java | 2 +-
.../org/apache/accumulo/proxy/SimpleTest.java | 2 +-
.../server/mini/MiniAccumuloCluster.java | 364 +++++++++++++++++++
.../server/mini/MiniAccumuloConfig.java | 89 +++++
.../server/test/MiniAccumuloCluster.java | 364 -------------------
.../server/test/MiniAccumuloConfig.java | 89 -----
.../server/mini/MiniAccumuloClusterTest.java | 150 ++++++++
.../server/test/MiniAccumuloClusterTest.java | 150 --------
8 files changed, 605 insertions(+), 605 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/87cd6236/src/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java
----------------------------------------------------------------------
diff --git a/src/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java b/src/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java
index 8fcd4a8..7b106dd 100644
--- a/src/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java
+++ b/src/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java
@@ -23,7 +23,7 @@ import java.lang.reflect.Constructor;
import java.util.Properties;
import org.apache.accumulo.proxy.thrift.AccumuloProxy;
-import org.apache.accumulo.server.test.MiniAccumuloCluster;
+import org.apache.accumulo.server.mini.MiniAccumuloCluster;
import org.apache.log4j.Logger;
import org.apache.thrift.TProcessor;
import org.apache.thrift.protocol.TCompactProtocol;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/87cd6236/src/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
----------------------------------------------------------------------
diff --git a/src/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java b/src/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
index 20b6279..8a898f4 100644
--- a/src/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
+++ b/src/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
@@ -74,7 +74,7 @@ import org.apache.accumulo.proxy.thrift.TimeType;
import org.apache.accumulo.proxy.thrift.UnknownScanner;
import org.apache.accumulo.proxy.thrift.UnknownWriter;
import org.apache.accumulo.proxy.thrift.WriterOptions;
-import org.apache.accumulo.server.test.MiniAccumuloCluster;
+import org.apache.accumulo.server.mini.MiniAccumuloCluster;
import org.apache.accumulo.server.test.functional.SlowIterator;
import org.apache.accumulo.server.util.PortUtils;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/87cd6236/src/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloCluster.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloCluster.java b/src/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloCluster.java
new file mode 100644
index 0000000..2cb3ae2
--- /dev/null
+++ b/src/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloCluster.java
@@ -0,0 +1,364 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.mini;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.TimerTask;
+
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.server.logger.LogService;
+import org.apache.accumulo.server.master.Master;
+import org.apache.accumulo.server.tabletserver.TabletServer;
+import org.apache.accumulo.server.util.Initialize;
+import org.apache.accumulo.server.util.PortUtils;
+import org.apache.accumulo.server.util.time.SimpleTimer;
+import org.apache.accumulo.start.Main;
+import org.apache.zookeeper.server.ZooKeeperServerMain;
+
+/**
+ * A utility class that will create Zookeeper and Accumulo processes that write all of their data to a single local directory. This class makes it easy to test
+ * code against a real Accumulo instance. Its much more accurate for testing than MockAccumulo, but much slower than MockAccumulo.
+ *
+ * @since 1.5.0, 1.4.4
+ */
+public class MiniAccumuloCluster {
+
+ private static final String INSTANCE_SECRET = "DONTTELL";
+ private static final String INSTANCE_NAME = "miniInstance";
+
+ private static class LogWriter extends Thread {
+ private BufferedReader in;
+ private BufferedWriter out;
+
+ /**
+ * @throws java.io.IOException
+ */
+ public LogWriter(InputStream stream, File logFile) throws IOException {
+ this.setDaemon(true);
+ this.in = new BufferedReader(new InputStreamReader(stream));
+ out = new BufferedWriter(new FileWriter(logFile));
+
+ SimpleTimer.getInstance().schedule(new TimerTask() {
+ @Override
+ public void run() {
+ try {
+ flush();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }, 1000, 1000);
+ }
+
+ public synchronized void flush() throws IOException {
+ if (out != null)
+ out.flush();
+ }
+
+ @Override
+ public void run() {
+ String line;
+
+ try {
+ while ((line = in.readLine()) != null) {
+ out.append(line);
+ out.append("\n");
+ }
+
+ synchronized (this) {
+ out.close();
+ out = null;
+ in.close();
+ }
+
+ } catch (IOException e) {
+ }
+ }
+ }
+
+ private File libDir;
+ private File confDir;
+ private File zooKeeperDir;
+ private File accumuloDir;
+ private File zooCfgFile;
+ private File logDir;
+ private File walogDir;
+
+ private Process zooKeeperProcess;
+ private Process masterProcess;
+ private Process loggerProcess;
+
+ private int zooKeeperPort;
+
+ private List<LogWriter> logWriters = new ArrayList<LogWriter>();
+
+ private MiniAccumuloConfig config;
+ private Process[] tabletServerProcesses;
+
+ Process exec(Class<? extends Object> clazz, String... args) throws IOException {
+ String javaHome = System.getProperty("java.home");
+ String javaBin = javaHome + File.separator + "bin" + File.separator + "java";
+ String classpath = System.getProperty("java.class.path");
+
+ classpath = confDir.getAbsolutePath() + File.pathSeparator + classpath;
+
+ String className = clazz.getCanonicalName();
+
+ ArrayList<String> argList = new ArrayList<String>();
+
+ argList.addAll(Arrays.asList(javaBin, "-cp", classpath, "-Xmx128m", "-XX:+UseConcMarkSweepGC", "-XX:CMSInitiatingOccupancyFraction=75",
+ Main.class.getName(), className));
+
+ argList.addAll(Arrays.asList(args));
+
+ ProcessBuilder builder = new ProcessBuilder(argList);
+
+ builder.environment().put("ACCUMULO_HOME", config.getDir().getAbsolutePath());
+ builder.environment().put("ACCUMULO_LOG_DIR", logDir.getAbsolutePath());
+
+ // if we're running under accumulo.start, we forward these env vars
+ String env = System.getenv("HADOOP_PREFIX");
+ if (env != null)
+ builder.environment().put("HADOOP_PREFIX", env);
+ env = System.getenv("ZOOKEEPER_HOME");
+ if (env != null)
+ builder.environment().put("ZOOKEEPER_HOME", env);
+
+ Process process = builder.start();
+
+ LogWriter lw;
+ lw = new LogWriter(process.getErrorStream(), new File(logDir, clazz.getSimpleName() + "_" + process.hashCode() + ".err"));
+ logWriters.add(lw);
+ lw.start();
+ lw = new LogWriter(process.getInputStream(), new File(logDir, clazz.getSimpleName() + "_" + process.hashCode() + ".out"));
+ logWriters.add(lw);
+ lw.start();
+
+ return process;
+ }
+
+ private void appendProp(FileWriter fileWriter, Property key, String value, Map<String,String> siteConfig) throws IOException {
+ appendProp(fileWriter, key.getKey(), value, siteConfig);
+ }
+
+ private void appendProp(FileWriter fileWriter, String key, String value, Map<String,String> siteConfig) throws IOException {
+ if (!siteConfig.containsKey(key))
+ fileWriter.append("<property><name>" + key + "</name><value>" + value + "</value></property>\n");
+ }
+
+ /**
+ * @param dir
+ * An empty or nonexistant temp directoy that Accumulo and Zookeeper can store data in. Creating the directory is left to the user. Java 7, Guava,
+ * and Junit provide methods for creating temporary directories.
+ * @param rootPassword
+ * Initial root password for instance.
+ * @throws IOException
+ */
+ public MiniAccumuloCluster(File dir, String rootPassword) throws IOException {
+ this(new MiniAccumuloConfig(dir, rootPassword));
+ }
+
+ /**
+ * @param config
+ * initial configuration
+ * @throws IOException
+ */
+
+ public MiniAccumuloCluster(MiniAccumuloConfig config) throws IOException {
+
+ if (config.getDir().exists() && !config.getDir().isDirectory())
+ throw new IllegalArgumentException("Must pass in directory, " + config.getDir() + " is a file");
+
+ if (config.getDir().exists() && config.getDir().list().length != 0)
+ throw new IllegalArgumentException("Directory " + config.getDir() + " is not empty");
+
+ this.config = config;
+
+ libDir = new File(config.getDir(), "lib");
+ confDir = new File(config.getDir(), "conf");
+ accumuloDir = new File(config.getDir(), "accumulo");
+ zooKeeperDir = new File(config.getDir(), "zookeeper");
+ logDir = new File(config.getDir(), "logs");
+ walogDir = new File(config.getDir(), "walogs");
+
+ confDir.mkdirs();
+ accumuloDir.mkdirs();
+ zooKeeperDir.mkdirs();
+ logDir.mkdirs();
+ walogDir.mkdirs();
+ libDir.mkdirs();
+
+ zooKeeperPort = PortUtils.getRandomFreePort();
+
+ File siteFile = new File(confDir, "accumulo-site.xml");
+
+ FileWriter fileWriter = new FileWriter(siteFile);
+ fileWriter.append("<configuration>\n");
+
+ HashMap<String,String> siteConfig = new HashMap<String,String>(config.getSiteConfig());
+
+ appendProp(fileWriter, Property.INSTANCE_DFS_URI, "file:///", siteConfig);
+ appendProp(fileWriter, Property.INSTANCE_DFS_DIR, accumuloDir.getAbsolutePath(), siteConfig);
+ appendProp(fileWriter, Property.INSTANCE_ZK_HOST, "localhost:" + zooKeeperPort, siteConfig);
+ appendProp(fileWriter, Property.INSTANCE_SECRET, INSTANCE_SECRET, siteConfig);
+ appendProp(fileWriter, Property.MASTER_CLIENTPORT, "" + PortUtils.getRandomFreePort(), siteConfig);
+ appendProp(fileWriter, Property.TSERV_CLIENTPORT, "" + PortUtils.getRandomFreePort(), siteConfig);
+ appendProp(fileWriter, Property.TSERV_PORTSEARCH, "true", siteConfig);
+ appendProp(fileWriter, Property.LOGGER_DIR, walogDir.getAbsolutePath(), siteConfig);
+ appendProp(fileWriter, Property.TSERV_DATACACHE_SIZE, "10M", siteConfig);
+ appendProp(fileWriter, Property.TSERV_INDEXCACHE_SIZE, "10M", siteConfig);
+ appendProp(fileWriter, Property.TSERV_MAXMEM, "50M", siteConfig);
+ appendProp(fileWriter, Property.TSERV_WALOG_MAX_SIZE, "100M", siteConfig);
+ appendProp(fileWriter, Property.TSERV_NATIVEMAP_ENABLED, "false", siteConfig);
+ appendProp(fileWriter, Property.TRACE_PORT, "" + PortUtils.getRandomFreePort(), siteConfig);
+ appendProp(fileWriter, Property.LOGGER_SORT_BUFFER_SIZE, "50M", siteConfig);
+ appendProp(fileWriter, Property.LOGGER_PORTSEARCH, "true", siteConfig);
+
+ // since there is a small amount of memory, check more frequently for majc... setting may not be needed in 1.5
+ appendProp(fileWriter, Property.TSERV_MAJC_DELAY, "3", siteConfig);
+ String cp = System.getenv("ACCUMULO_HOME") + "/lib/.*.jar," + "$ZOOKEEPER_HOME/zookeeper[^.].*.jar," + "$HADOOP_HOME/[^.].*.jar,"
+ + "$HADOOP_HOME/lib/[^.].*.jar," + "$HADOOP_PREFIX/share/hadoop/common/.*.jar," + "$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,"
+ + "$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar," + "$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar";
+ appendProp(fileWriter, Property.GENERAL_CLASSPATHS, cp, siteConfig);
+ appendProp(fileWriter, Property.GENERAL_DYNAMIC_CLASSPATHS, libDir.getAbsolutePath(), siteConfig);
+
+ for (Entry<String,String> entry : siteConfig.entrySet())
+ fileWriter.append("<property><name>" + entry.getKey() + "</name><value>" + entry.getValue() + "</value></property>\n");
+ fileWriter.append("</configuration>\n");
+ fileWriter.close();
+
+ zooCfgFile = new File(confDir, "zoo.cfg");
+ fileWriter = new FileWriter(zooCfgFile);
+
+ // zookeeper uses Properties to read its config, so use that to write in order to properly escape things like Windows paths
+ Properties zooCfg = new Properties();
+ zooCfg.setProperty("tickTime", "1000");
+ zooCfg.setProperty("initLimit", "10");
+ zooCfg.setProperty("syncLimit", "5");
+ zooCfg.setProperty("clientPort", zooKeeperPort + "");
+ zooCfg.setProperty("maxClientCnxns", "100");
+ zooCfg.setProperty("dataDir", zooKeeperDir.getAbsolutePath());
+ zooCfg.store(fileWriter, null);
+
+ fileWriter.close();
+ }
+
+ /**
+ * Starts Accumulo and Zookeeper processes. Can only be called once.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws IllegalStateException
+ * if already started
+ */
+ public void start() throws IOException, InterruptedException {
+ if (zooKeeperProcess != null)
+ throw new IllegalStateException("Already started");
+
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ try {
+ MiniAccumuloCluster.this.stop();
+ } catch (IOException e) {
+ e.printStackTrace();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ });
+
+ zooKeeperProcess = exec(Main.class, ZooKeeperServerMain.class.getName(), zooCfgFile.getAbsolutePath());
+
+ // sleep a little bit to let zookeeper come up before calling init, seems to work better
+ UtilWaitThread.sleep(250);
+
+ // TODO initialization could probably be done in process
+ Process initProcess = exec(Initialize.class);
+ initProcess.getOutputStream().write((INSTANCE_NAME + "\n").getBytes());
+ initProcess.getOutputStream().write((config.getRootPassword() + "\n").getBytes());
+ initProcess.getOutputStream().write((config.getRootPassword() + "\n").getBytes());
+ initProcess.getOutputStream().flush();
+ int ret = initProcess.waitFor();
+ if (ret != 0) {
+ throw new RuntimeException("Initialize process returned " + ret);
+ }
+
+ tabletServerProcesses = new Process[config.getNumTservers()];
+ for (int i = 0; i < config.getNumTservers(); i++) {
+ tabletServerProcesses[i] = exec(TabletServer.class);
+ }
+
+ loggerProcess = exec(LogService.class);
+ masterProcess = exec(Master.class);
+ }
+
+ /**
+ * @return Accumulo instance name
+ */
+
+ public String getInstanceName() {
+ return INSTANCE_NAME;
+ }
+
+ /**
+ * @return zookeeper connection string
+ */
+
+ public String getZooKeepers() {
+ return "localhost:" + zooKeeperPort;
+ }
+
+ /**
+ * Stops Accumulo and Zookeeper processes. If stop is not called, there is a shutdown hook that is setup to kill the processes. Howerver its probably best to
+ * call stop in a finally block as soon as possible.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ */
+
+ public void stop() throws IOException, InterruptedException {
+ if (zooKeeperProcess != null)
+ zooKeeperProcess.destroy();
+ if (loggerProcess != null)
+ loggerProcess.destroy();
+ if (masterProcess != null)
+ masterProcess.destroy();
+ if (tabletServerProcesses != null) {
+ for (Process tserver : tabletServerProcesses) {
+ tserver.destroy();
+ }
+ }
+
+ for (LogWriter lw : logWriters)
+ lw.flush();
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/87cd6236/src/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloConfig.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloConfig.java b/src/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloConfig.java
new file mode 100644
index 0000000..a82f277
--- /dev/null
+++ b/src/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloConfig.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.mini;
+
+import java.io.File;
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * Holds configuration for {@link MiniAccumuloCluster}. Required configurations must be passed to constructor and all other configurations are optional.
+ *
+ * @since 1.5.0, 1.4.4
+ */
+
+public class MiniAccumuloConfig {
+
+ private File dir = null;
+ private String rootPassword = null;
+ private Map<String,String> siteConfig = Collections.emptyMap();
+ private int numTservers = 2;
+
+ /**
+ * @param dir
+ * An empty or nonexistant temp directoy that Accumulo and Zookeeper can store data in. Creating the directory is left to the user. Java 7, Guava,
+ * and Junit provide methods for creating temporary directories.
+ * @param rootPassword
+ * The initial password for the Accumulo root user
+ */
+
+ public MiniAccumuloConfig(File dir, String rootPassword) {
+ this.dir = dir;
+ this.rootPassword = rootPassword;
+ }
+
+ public File getDir() {
+ return dir;
+ }
+
+ public String getRootPassword() {
+ return rootPassword;
+ }
+
+ public int getNumTservers() {
+ return numTservers;
+ }
+
+ /**
+ * Calling this method is optional. If not set, it defaults to two.
+ *
+ * @param numTservers
+ * the number of tablet servers that mini accumulo cluster should start
+ */
+
+ public MiniAccumuloConfig setNumTservers(int numTservers) {
+ if (numTservers < 1)
+ throw new IllegalArgumentException("Must have at least one tablet server");
+ this.numTservers = numTservers;
+ return this;
+ }
+
+ public Map<String,String> getSiteConfig() {
+ return siteConfig;
+ }
+
+ /**
+ * Calling this method is optional. If not set, it defautls to an empty map.
+ *
+ * @param siteConfig
+ * key/values that you normally put in accumulo-site.xml can be put here
+ */
+ public MiniAccumuloConfig setSiteConfig(Map<String,String> siteConfig) {
+ this.siteConfig = siteConfig;
+ return this;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/87cd6236/src/server/src/main/java/org/apache/accumulo/server/test/MiniAccumuloCluster.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/test/MiniAccumuloCluster.java b/src/server/src/main/java/org/apache/accumulo/server/test/MiniAccumuloCluster.java
deleted file mode 100644
index 4f74cb2..0000000
--- a/src/server/src/main/java/org/apache/accumulo/server/test/MiniAccumuloCluster.java
+++ /dev/null
@@ -1,364 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.server.test;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Properties;
-import java.util.TimerTask;
-
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.util.UtilWaitThread;
-import org.apache.accumulo.server.logger.LogService;
-import org.apache.accumulo.server.master.Master;
-import org.apache.accumulo.server.tabletserver.TabletServer;
-import org.apache.accumulo.server.util.Initialize;
-import org.apache.accumulo.server.util.PortUtils;
-import org.apache.accumulo.server.util.time.SimpleTimer;
-import org.apache.accumulo.start.Main;
-import org.apache.zookeeper.server.ZooKeeperServerMain;
-
-/**
- * A utility class that will create Zookeeper and Accumulo processes that write all of their data to a single local directory. This class makes it easy to test
- * code against a real Accumulo instance. Its much more accurate for testing than MockAccumulo, but much slower than MockAccumulo.
- *
- * @since 1.5.0, 1.4.4
- */
-public class MiniAccumuloCluster {
-
- private static final String INSTANCE_SECRET = "DONTTELL";
- private static final String INSTANCE_NAME = "miniInstance";
-
- private static class LogWriter extends Thread {
- private BufferedReader in;
- private BufferedWriter out;
-
- /**
- * @throws java.io.IOException
- */
- public LogWriter(InputStream stream, File logFile) throws IOException {
- this.setDaemon(true);
- this.in = new BufferedReader(new InputStreamReader(stream));
- out = new BufferedWriter(new FileWriter(logFile));
-
- SimpleTimer.getInstance().schedule(new TimerTask() {
- @Override
- public void run() {
- try {
- flush();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- }, 1000, 1000);
- }
-
- public synchronized void flush() throws IOException {
- if (out != null)
- out.flush();
- }
-
- @Override
- public void run() {
- String line;
-
- try {
- while ((line = in.readLine()) != null) {
- out.append(line);
- out.append("\n");
- }
-
- synchronized (this) {
- out.close();
- out = null;
- in.close();
- }
-
- } catch (IOException e) {
- }
- }
- }
-
- private File libDir;
- private File confDir;
- private File zooKeeperDir;
- private File accumuloDir;
- private File zooCfgFile;
- private File logDir;
- private File walogDir;
-
- private Process zooKeeperProcess;
- private Process masterProcess;
- private Process loggerProcess;
-
- private int zooKeeperPort;
-
- private List<LogWriter> logWriters = new ArrayList<LogWriter>();
-
- private MiniAccumuloConfig config;
- private Process[] tabletServerProcesses;
-
- Process exec(Class<? extends Object> clazz, String... args) throws IOException {
- String javaHome = System.getProperty("java.home");
- String javaBin = javaHome + File.separator + "bin" + File.separator + "java";
- String classpath = System.getProperty("java.class.path");
-
- classpath = confDir.getAbsolutePath() + File.pathSeparator + classpath;
-
- String className = clazz.getCanonicalName();
-
- ArrayList<String> argList = new ArrayList<String>();
-
- argList.addAll(Arrays.asList(javaBin, "-cp", classpath, "-Xmx128m", "-XX:+UseConcMarkSweepGC", "-XX:CMSInitiatingOccupancyFraction=75",
- Main.class.getName(), className));
-
- argList.addAll(Arrays.asList(args));
-
- ProcessBuilder builder = new ProcessBuilder(argList);
-
- builder.environment().put("ACCUMULO_HOME", config.getDir().getAbsolutePath());
- builder.environment().put("ACCUMULO_LOG_DIR", logDir.getAbsolutePath());
-
- // if we're running under accumulo.start, we forward these env vars
- String env = System.getenv("HADOOP_PREFIX");
- if (env != null)
- builder.environment().put("HADOOP_PREFIX", env);
- env = System.getenv("ZOOKEEPER_HOME");
- if (env != null)
- builder.environment().put("ZOOKEEPER_HOME", env);
-
- Process process = builder.start();
-
- LogWriter lw;
- lw = new LogWriter(process.getErrorStream(), new File(logDir, clazz.getSimpleName() + "_" + process.hashCode() + ".err"));
- logWriters.add(lw);
- lw.start();
- lw = new LogWriter(process.getInputStream(), new File(logDir, clazz.getSimpleName() + "_" + process.hashCode() + ".out"));
- logWriters.add(lw);
- lw.start();
-
- return process;
- }
-
- private void appendProp(FileWriter fileWriter, Property key, String value, Map<String,String> siteConfig) throws IOException {
- appendProp(fileWriter, key.getKey(), value, siteConfig);
- }
-
- private void appendProp(FileWriter fileWriter, String key, String value, Map<String,String> siteConfig) throws IOException {
- if (!siteConfig.containsKey(key))
- fileWriter.append("<property><name>" + key + "</name><value>" + value + "</value></property>\n");
- }
-
- /**
- * @param dir
- * An empty or nonexistant temp directoy that Accumulo and Zookeeper can store data in. Creating the directory is left to the user. Java 7, Guava,
- * and Junit provide methods for creating temporary directories.
- * @param rootPassword
- * Initial root password for instance.
- * @throws IOException
- */
- public MiniAccumuloCluster(File dir, String rootPassword) throws IOException {
- this(new MiniAccumuloConfig(dir, rootPassword));
- }
-
- /**
- * @param config
- * initial configuration
- * @throws IOException
- */
-
- public MiniAccumuloCluster(MiniAccumuloConfig config) throws IOException {
-
- if (config.getDir().exists() && !config.getDir().isDirectory())
- throw new IllegalArgumentException("Must pass in directory, " + config.getDir() + " is a file");
-
- if (config.getDir().exists() && config.getDir().list().length != 0)
- throw new IllegalArgumentException("Directory " + config.getDir() + " is not empty");
-
- this.config = config;
-
- libDir = new File(config.getDir(), "lib");
- confDir = new File(config.getDir(), "conf");
- accumuloDir = new File(config.getDir(), "accumulo");
- zooKeeperDir = new File(config.getDir(), "zookeeper");
- logDir = new File(config.getDir(), "logs");
- walogDir = new File(config.getDir(), "walogs");
-
- confDir.mkdirs();
- accumuloDir.mkdirs();
- zooKeeperDir.mkdirs();
- logDir.mkdirs();
- walogDir.mkdirs();
- libDir.mkdirs();
-
- zooKeeperPort = PortUtils.getRandomFreePort();
-
- File siteFile = new File(confDir, "accumulo-site.xml");
-
- FileWriter fileWriter = new FileWriter(siteFile);
- fileWriter.append("<configuration>\n");
-
- HashMap<String,String> siteConfig = new HashMap<String,String>(config.getSiteConfig());
-
- appendProp(fileWriter, Property.INSTANCE_DFS_URI, "file:///", siteConfig);
- appendProp(fileWriter, Property.INSTANCE_DFS_DIR, accumuloDir.getAbsolutePath(), siteConfig);
- appendProp(fileWriter, Property.INSTANCE_ZK_HOST, "localhost:" + zooKeeperPort, siteConfig);
- appendProp(fileWriter, Property.INSTANCE_SECRET, INSTANCE_SECRET, siteConfig);
- appendProp(fileWriter, Property.MASTER_CLIENTPORT, "" + PortUtils.getRandomFreePort(), siteConfig);
- appendProp(fileWriter, Property.TSERV_CLIENTPORT, "" + PortUtils.getRandomFreePort(), siteConfig);
- appendProp(fileWriter, Property.TSERV_PORTSEARCH, "true", siteConfig);
- appendProp(fileWriter, Property.LOGGER_DIR, walogDir.getAbsolutePath(), siteConfig);
- appendProp(fileWriter, Property.TSERV_DATACACHE_SIZE, "10M", siteConfig);
- appendProp(fileWriter, Property.TSERV_INDEXCACHE_SIZE, "10M", siteConfig);
- appendProp(fileWriter, Property.TSERV_MAXMEM, "50M", siteConfig);
- appendProp(fileWriter, Property.TSERV_WALOG_MAX_SIZE, "100M", siteConfig);
- appendProp(fileWriter, Property.TSERV_NATIVEMAP_ENABLED, "false", siteConfig);
- appendProp(fileWriter, Property.TRACE_PORT, "" + PortUtils.getRandomFreePort(), siteConfig);
- appendProp(fileWriter, Property.LOGGER_SORT_BUFFER_SIZE, "50M", siteConfig);
- appendProp(fileWriter, Property.LOGGER_PORTSEARCH, "true", siteConfig);
-
- // since there is a small amount of memory, check more frequently for majc... setting may not be needed in 1.5
- appendProp(fileWriter, Property.TSERV_MAJC_DELAY, "3", siteConfig);
- String cp = System.getenv("ACCUMULO_HOME") + "/lib/.*.jar," + "$ZOOKEEPER_HOME/zookeeper[^.].*.jar," + "$HADOOP_HOME/[^.].*.jar,"
- + "$HADOOP_HOME/lib/[^.].*.jar," + "$HADOOP_PREFIX/share/hadoop/common/.*.jar," + "$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,"
- + "$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar," + "$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar";
- appendProp(fileWriter, Property.GENERAL_CLASSPATHS, cp, siteConfig);
- appendProp(fileWriter, Property.GENERAL_DYNAMIC_CLASSPATHS, libDir.getAbsolutePath(), siteConfig);
-
- for (Entry<String,String> entry : siteConfig.entrySet())
- fileWriter.append("<property><name>" + entry.getKey() + "</name><value>" + entry.getValue() + "</value></property>\n");
- fileWriter.append("</configuration>\n");
- fileWriter.close();
-
- zooCfgFile = new File(confDir, "zoo.cfg");
- fileWriter = new FileWriter(zooCfgFile);
-
- // zookeeper uses Properties to read its config, so use that to write in order to properly escape things like Windows paths
- Properties zooCfg = new Properties();
- zooCfg.setProperty("tickTime", "1000");
- zooCfg.setProperty("initLimit", "10");
- zooCfg.setProperty("syncLimit", "5");
- zooCfg.setProperty("clientPort", zooKeeperPort + "");
- zooCfg.setProperty("maxClientCnxns", "100");
- zooCfg.setProperty("dataDir", zooKeeperDir.getAbsolutePath());
- zooCfg.store(fileWriter, null);
-
- fileWriter.close();
- }
-
- /**
- * Starts Accumulo and Zookeeper processes. Can only be called once.
- *
- * @throws IOException
- * @throws InterruptedException
- * @throws IllegalStateException
- * if already started
- */
- public void start() throws IOException, InterruptedException {
- if (zooKeeperProcess != null)
- throw new IllegalStateException("Already started");
-
- Runtime.getRuntime().addShutdownHook(new Thread() {
- @Override
- public void run() {
- try {
- MiniAccumuloCluster.this.stop();
- } catch (IOException e) {
- e.printStackTrace();
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- });
-
- zooKeeperProcess = exec(Main.class, ZooKeeperServerMain.class.getName(), zooCfgFile.getAbsolutePath());
-
- // sleep a little bit to let zookeeper come up before calling init, seems to work better
- UtilWaitThread.sleep(250);
-
- // TODO initialization could probably be done in process
- Process initProcess = exec(Initialize.class);
- initProcess.getOutputStream().write((INSTANCE_NAME + "\n").getBytes());
- initProcess.getOutputStream().write((config.getRootPassword() + "\n").getBytes());
- initProcess.getOutputStream().write((config.getRootPassword() + "\n").getBytes());
- initProcess.getOutputStream().flush();
- int ret = initProcess.waitFor();
- if (ret != 0) {
- throw new RuntimeException("Initialize process returned " + ret);
- }
-
- tabletServerProcesses = new Process[config.getNumTservers()];
- for (int i = 0; i < config.getNumTservers(); i++) {
- tabletServerProcesses[i] = exec(TabletServer.class);
- }
-
- loggerProcess = exec(LogService.class);
- masterProcess = exec(Master.class);
- }
-
- /**
- * @return Accumulo instance name
- */
-
- public String getInstanceName() {
- return INSTANCE_NAME;
- }
-
- /**
- * @return zookeeper connection string
- */
-
- public String getZooKeepers() {
- return "localhost:" + zooKeeperPort;
- }
-
- /**
- * Stops Accumulo and Zookeeper processes. If stop is not called, there is a shutdown hook that is setup to kill the processes. Howerver its probably best to
- * call stop in a finally block as soon as possible.
- *
- * @throws IOException
- * @throws InterruptedException
- */
-
- public void stop() throws IOException, InterruptedException {
- if (zooKeeperProcess != null)
- zooKeeperProcess.destroy();
- if (loggerProcess != null)
- loggerProcess.destroy();
- if (masterProcess != null)
- masterProcess.destroy();
- if (tabletServerProcesses != null) {
- for (Process tserver : tabletServerProcesses) {
- tserver.destroy();
- }
- }
-
- for (LogWriter lw : logWriters)
- lw.flush();
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/87cd6236/src/server/src/main/java/org/apache/accumulo/server/test/MiniAccumuloConfig.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/test/MiniAccumuloConfig.java b/src/server/src/main/java/org/apache/accumulo/server/test/MiniAccumuloConfig.java
deleted file mode 100644
index 6b3b211..0000000
--- a/src/server/src/main/java/org/apache/accumulo/server/test/MiniAccumuloConfig.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.server.test;
-
-import java.io.File;
-import java.util.Collections;
-import java.util.Map;
-
-/**
- * Holds configuration for {@link MiniAccumuloCluster}. Required configurations must be passed to constructor and all other configurations are optional.
- *
- * @since 1.5.0, 1.4.4
- */
-
-public class MiniAccumuloConfig {
-
- private File dir = null;
- private String rootPassword = null;
- private Map<String,String> siteConfig = Collections.emptyMap();
- private int numTservers = 2;
-
- /**
- * @param dir
- * An empty or nonexistant temp directoy that Accumulo and Zookeeper can store data in. Creating the directory is left to the user. Java 7, Guava,
- * and Junit provide methods for creating temporary directories.
- * @param rootPassword
- * The initial password for the Accumulo root user
- */
-
- public MiniAccumuloConfig(File dir, String rootPassword) {
- this.dir = dir;
- this.rootPassword = rootPassword;
- }
-
- public File getDir() {
- return dir;
- }
-
- public String getRootPassword() {
- return rootPassword;
- }
-
- public int getNumTservers() {
- return numTservers;
- }
-
- /**
- * Calling this method is optional. If not set, it defaults to two.
- *
- * @param numTservers
- * the number of tablet servers that mini accumulo cluster should start
- */
-
- public MiniAccumuloConfig setNumTservers(int numTservers) {
- if (numTservers < 1)
- throw new IllegalArgumentException("Must have at least one tablet server");
- this.numTservers = numTservers;
- return this;
- }
-
- public Map<String,String> getSiteConfig() {
- return siteConfig;
- }
-
- /**
- * Calling this method is optional. If not set, it defautls to an empty map.
- *
- * @param siteConfig
- * key/values that you normally put in accumulo-site.xml can be put here
- */
- public MiniAccumuloConfig setSiteConfig(Map<String,String> siteConfig) {
- this.siteConfig = siteConfig;
- return this;
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/87cd6236/src/server/src/test/java/org/apache/accumulo/server/mini/MiniAccumuloClusterTest.java
----------------------------------------------------------------------
diff --git a/src/server/src/test/java/org/apache/accumulo/server/mini/MiniAccumuloClusterTest.java b/src/server/src/test/java/org/apache/accumulo/server/mini/MiniAccumuloClusterTest.java
new file mode 100644
index 0000000..083d4b8
--- /dev/null
+++ b/src/server/src/test/java/org/apache/accumulo/server/mini/MiniAccumuloClusterTest.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.mini;
+
+import java.util.Collections;
+import java.util.Map.Entry;
+import java.util.UUID;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.LongCombiner;
+import org.apache.accumulo.core.iterators.user.SummingCombiner;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.server.mini.MiniAccumuloCluster;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+public class MiniAccumuloClusterTest {
+
+ private static final Logger logger = Logger.getLogger(MiniAccumuloClusterTest.class);
+
+ private static TemporaryFolder tmpDir = new TemporaryFolder();
+ private static MiniAccumuloCluster accumulo;
+
+ @BeforeClass
+ public static void setupMiniCluster() throws Exception {
+
+ tmpDir.create();
+ logger.info("MiniCluster started @ " + tmpDir.getRoot());
+ Logger.getLogger("org.apache.zookeeper").setLevel(Level.ERROR);
+
+ accumulo = new MiniAccumuloCluster(tmpDir.getRoot(), "superSecret");
+ accumulo.start();
+ }
+
+ @Test(timeout = 30000)
+ public void test() throws Exception {
+ Connector conn = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers()).getConnector("root", "superSecret".getBytes());
+
+ conn.tableOperations().create("table1");
+
+ conn.securityOperations().createUser("user1", "pass1".getBytes(), new Authorizations("A", "B"));
+ conn.securityOperations().grantTablePermission("user1", "table1", TablePermission.WRITE);
+ conn.securityOperations().grantTablePermission("user1", "table1", TablePermission.READ);
+
+ IteratorSetting is = new IteratorSetting(10, SummingCombiner.class);
+ SummingCombiner.setEncodingType(is, LongCombiner.Type.STRING);
+ SummingCombiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("META", "COUNT")));
+
+ conn.tableOperations().attachIterator("table1", is);
+
+ Connector uconn = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers()).getConnector("user1", "pass1".getBytes());
+
+ BatchWriter bw = uconn.createBatchWriter("table1", 10000, 1000000, 2);
+
+ UUID uuid = UUID.randomUUID();
+
+ Mutation m = new Mutation(uuid.toString());
+ m.put("META", "SIZE", new ColumnVisibility("A|B"), "8");
+ m.put("META", "CRC", new ColumnVisibility("A|B"), "456");
+ m.put("META", "COUNT", new ColumnVisibility("A|B"), "1");
+ m.put("DATA", "IMG", new ColumnVisibility("A&B"), "ABCDEFGH");
+
+ bw.addMutation(m);
+ bw.flush();
+
+ m = new Mutation(uuid.toString());
+ m.put("META", "COUNT", new ColumnVisibility("A|B"), "1");
+ m.put("META", "CRC", new ColumnVisibility("A|B"), "123");
+ bw.addMutation(m);
+
+ bw.close();
+
+ int count = 0;
+ Scanner scanner = uconn.createScanner("table1", new Authorizations("A"));
+ for (Entry<Key,Value> entry : scanner) {
+ if (entry.getKey().getColumnQualifierData().toString().equals("COUNT")) {
+ Assert.assertEquals("2", entry.getValue().toString());
+ } else if (entry.getKey().getColumnQualifierData().toString().equals("SIZE")) {
+ Assert.assertEquals("8", entry.getValue().toString());
+ } else if (entry.getKey().getColumnQualifierData().toString().equals("CRC")) {
+ Assert.assertEquals("123", entry.getValue().toString());
+ } else {
+ Assert.assertTrue(false);
+ }
+ count++;
+ }
+
+ Assert.assertEquals(3, count);
+
+ count = 0;
+ scanner = uconn.createScanner("table1", new Authorizations("A", "B"));
+ for (Entry<Key,Value> entry : scanner) {
+ if (entry.getKey().getColumnQualifierData().toString().equals("IMG")) {
+ Assert.assertEquals("ABCDEFGH", entry.getValue().toString());
+ }
+ count++;
+ }
+
+ Assert.assertEquals(4, count);
+
+ conn.tableOperations().delete("table1");
+ }
+
+ @Test(timeout = 20000)
+ public void testMultipleTabletServersRunning() throws AccumuloException, AccumuloSecurityException {
+
+ Connector conn = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers()).getConnector("root", "superSecret".getBytes());
+
+ while (conn.instanceOperations().getTabletServers().size() != 2) {
+ UtilWaitThread.sleep(500);
+ }
+ }
+
+ @AfterClass
+ public static void tearDownMiniCluster() throws Exception {
+ accumulo.stop();
+ tmpDir.delete();
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/87cd6236/src/server/src/test/java/org/apache/accumulo/server/test/MiniAccumuloClusterTest.java
----------------------------------------------------------------------
diff --git a/src/server/src/test/java/org/apache/accumulo/server/test/MiniAccumuloClusterTest.java b/src/server/src/test/java/org/apache/accumulo/server/test/MiniAccumuloClusterTest.java
deleted file mode 100644
index a469fc5..0000000
--- a/src/server/src/test/java/org/apache/accumulo/server/test/MiniAccumuloClusterTest.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.server.test;
-
-import java.util.Collections;
-import java.util.Map.Entry;
-import java.util.UUID;
-
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.IteratorSetting;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.client.ZooKeeperInstance;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.iterators.LongCombiner;
-import org.apache.accumulo.core.iterators.user.SummingCombiner;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.security.ColumnVisibility;
-import org.apache.accumulo.core.security.TablePermission;
-import org.apache.accumulo.core.util.UtilWaitThread;
-import org.apache.accumulo.server.test.MiniAccumuloCluster;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-public class MiniAccumuloClusterTest {
-
- private static final Logger logger = Logger.getLogger(MiniAccumuloClusterTest.class);
-
- private static TemporaryFolder tmpDir = new TemporaryFolder();
- private static MiniAccumuloCluster accumulo;
-
- @BeforeClass
- public static void setupMiniCluster() throws Exception {
-
- tmpDir.create();
- logger.info("MiniCluster started @ " + tmpDir.getRoot());
- Logger.getLogger("org.apache.zookeeper").setLevel(Level.ERROR);
-
- accumulo = new MiniAccumuloCluster(tmpDir.getRoot(), "superSecret");
- accumulo.start();
- }
-
- @Test(timeout = 30000)
- public void test() throws Exception {
- Connector conn = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers()).getConnector("root", "superSecret".getBytes());
-
- conn.tableOperations().create("table1");
-
- conn.securityOperations().createUser("user1", "pass1".getBytes(), new Authorizations("A", "B"));
- conn.securityOperations().grantTablePermission("user1", "table1", TablePermission.WRITE);
- conn.securityOperations().grantTablePermission("user1", "table1", TablePermission.READ);
-
- IteratorSetting is = new IteratorSetting(10, SummingCombiner.class);
- SummingCombiner.setEncodingType(is, LongCombiner.Type.STRING);
- SummingCombiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("META", "COUNT")));
-
- conn.tableOperations().attachIterator("table1", is);
-
- Connector uconn = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers()).getConnector("user1", "pass1".getBytes());
-
- BatchWriter bw = uconn.createBatchWriter("table1", 10000, 1000000, 2);
-
- UUID uuid = UUID.randomUUID();
-
- Mutation m = new Mutation(uuid.toString());
- m.put("META", "SIZE", new ColumnVisibility("A|B"), "8");
- m.put("META", "CRC", new ColumnVisibility("A|B"), "456");
- m.put("META", "COUNT", new ColumnVisibility("A|B"), "1");
- m.put("DATA", "IMG", new ColumnVisibility("A&B"), "ABCDEFGH");
-
- bw.addMutation(m);
- bw.flush();
-
- m = new Mutation(uuid.toString());
- m.put("META", "COUNT", new ColumnVisibility("A|B"), "1");
- m.put("META", "CRC", new ColumnVisibility("A|B"), "123");
- bw.addMutation(m);
-
- bw.close();
-
- int count = 0;
- Scanner scanner = uconn.createScanner("table1", new Authorizations("A"));
- for (Entry<Key,Value> entry : scanner) {
- if (entry.getKey().getColumnQualifierData().toString().equals("COUNT")) {
- Assert.assertEquals("2", entry.getValue().toString());
- } else if (entry.getKey().getColumnQualifierData().toString().equals("SIZE")) {
- Assert.assertEquals("8", entry.getValue().toString());
- } else if (entry.getKey().getColumnQualifierData().toString().equals("CRC")) {
- Assert.assertEquals("123", entry.getValue().toString());
- } else {
- Assert.assertTrue(false);
- }
- count++;
- }
-
- Assert.assertEquals(3, count);
-
- count = 0;
- scanner = uconn.createScanner("table1", new Authorizations("A", "B"));
- for (Entry<Key,Value> entry : scanner) {
- if (entry.getKey().getColumnQualifierData().toString().equals("IMG")) {
- Assert.assertEquals("ABCDEFGH", entry.getValue().toString());
- }
- count++;
- }
-
- Assert.assertEquals(4, count);
-
- conn.tableOperations().delete("table1");
- }
-
- @Test(timeout = 20000)
- public void testMultipleTabletServersRunning() throws AccumuloException, AccumuloSecurityException {
-
- Connector conn = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers()).getConnector("root", "superSecret".getBytes());
-
- while (conn.instanceOperations().getTabletServers().size() != 2) {
- UtilWaitThread.sleep(500);
- }
- }
-
- @AfterClass
- public static void tearDownMiniCluster() throws Exception {
- accumulo.stop();
- tmpDir.delete();
- }
-}
[12/50] [abbrv] git commit: ACCUMULO-1418 use ZooReaderWriter,
and not raw ZooKeeper
Posted by el...@apache.org.
ACCUMULO-1418 use ZooReaderWriter, and not raw ZooKeeper
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.4@1482989 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/3d886c25
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/3d886c25
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/3d886c25
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 3d886c256e3ef4e299d792b10f099d43a496d55f
Parents: 854b91f
Author: Eric C. Newton <ec...@apache.org>
Authored: Wed May 15 18:27:12 2013 +0000
Committer: Eric C. Newton <ec...@apache.org>
Committed: Wed May 15 18:27:12 2013 +0000
----------------------------------------------------------------------
.../accumulo/server/util/RestoreZookeeper.java | 21 ++++++--------------
1 file changed, 6 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3d886c25/src/server/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java b/src/server/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
index 6b0c7dc..b5b330e 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
@@ -23,15 +23,13 @@ import java.util.Stack;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
-import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.core.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.server.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
import org.apache.commons.codec.binary.Base64;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.WatchedEvent;
-import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.ZooKeeper;
import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;
@@ -39,11 +37,11 @@ import org.xml.sax.helpers.DefaultHandler;
public class RestoreZookeeper {
private static class Restore extends DefaultHandler {
- ZooKeeper zk = null;
+ IZooReaderWriter zk = null;
Stack<String> cwd = new Stack<String>();
boolean overwrite = false;
- Restore(ZooKeeper zk, boolean overwrite) {
+ Restore(IZooReaderWriter zk, boolean overwrite) {
this.zk = zk;
this.overwrite = overwrite;
}
@@ -79,7 +77,7 @@ public class RestoreZookeeper {
data = Base64.decodeBase64(value.getBytes());
try {
try {
- ZooUtil.putPersistentData(zk, path, data, overwrite ? NodeExistsPolicy.OVERWRITE : NodeExistsPolicy.FAIL);
+ zk.putPersistentData(path, data, overwrite ? NodeExistsPolicy.OVERWRITE : NodeExistsPolicy.FAIL);
} catch (KeeperException e) {
if (e.code().equals(KeeperException.Code.NODEEXISTS))
throw new RuntimeException(path + " exists. Remove it first.");
@@ -98,8 +96,6 @@ public class RestoreZookeeper {
public static void main(String[] args) throws Exception {
Logger.getRootLogger().setLevel(Level.WARN);
- String server = args[0];
- int timeout = 30 * 1000;
InputStream in = System.in;
boolean overwrite = false;
if (args.length > 1) {
@@ -109,14 +105,9 @@ public class RestoreZookeeper {
if (arg.equals("--overwrite"))
overwrite = true;
- ZooKeeper zk = new ZooKeeper(server, timeout, new Watcher() {
- @Override
- public void process(WatchedEvent event) {}
- });
-
SAXParserFactory factory = SAXParserFactory.newInstance();
SAXParser parser = factory.newSAXParser();
- parser.parse(in, new Restore(zk, overwrite));
+ parser.parse(in, new Restore(ZooReaderWriter.getInstance(), overwrite));
in.close();
}
}
[47/50] [abbrv] git commit: Merge remote-tracking branch
'origin/1.5.1-SNAPSHOT' into 1.5.1-SNAPSHOT-1.4.4-SNAPSHOT-merged
Posted by el...@apache.org.
Merge remote-tracking branch 'origin/1.5.1-SNAPSHOT' into 1.5.1-SNAPSHOT-1.4.4-SNAPSHOT-merged
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/cb5a5431
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/cb5a5431
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/cb5a5431
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: cb5a5431581fcce7433c1a4205e25e0273595c7c
Parents: 726a5a8 a7bc937
Author: Josh Elser <el...@apache.org>
Authored: Tue Jul 23 22:28:27 2013 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Tue Jul 23 22:28:27 2013 -0400
----------------------------------------------------------------------
pom.xml | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
[05/50] [abbrv] git commit: ACCUMULO-1375 Pull back the final
proxy/README from trunk to 1.4
Posted by el...@apache.org.
ACCUMULO-1375 Pull back the final proxy/README from trunk to 1.4
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.4@1479113 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9637704e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9637704e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9637704e
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 9637704e94464eca571bd947f6a2ad4b300e2d4c
Parents: d52636c
Author: Josh Elser <el...@apache.org>
Authored: Sat May 4 14:18:52 2013 +0000
Committer: Josh Elser <el...@apache.org>
Committed: Sat May 4 14:18:52 2013 +0000
----------------------------------------------------------------------
src/proxy/README | 50 ++++++++++++++++++++++++++++++++------------------
1 file changed, 32 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9637704e/src/proxy/README
----------------------------------------------------------------------
diff --git a/src/proxy/README b/src/proxy/README
index 03cda49..97125f1 100644
--- a/src/proxy/README
+++ b/src/proxy/README
@@ -16,33 +16,47 @@ Notice: Licensed to the Apache Software Foundation (ASF) under one
specific language governing permissions and limitations
under the License.
-This is a thrift proxy server to interact with the Accumulo database.
+This module provides proxy server for Apache Accumulo. It enables using languages
+other than Java to interact with the database.
1. Building
-To compile, you must first install Thrift 0.6.1.
+The proxy server is built by the Accumulo build process. Read ../../README
+for more information.
-After thrift is installed, compile the project with:
-mvn clean install
-This will compile the thrift and java files, and will provide C++, python and ruby bindings.
+2. Installation
-2. Proxy Server Execution
+The proxy server is installed during the Accumulo installation process. Read ../../README
+for more information.
-Please note that the proxy can only function correctly when connected to an Accumulo 1.4 instance, or when run standalone in the Mock configuration.
-To run the server, edit the configuration file with parameters suited for your Accumulo installation. If you wish to instead use an in-memory Mock instance, set:
- useMockInstance=true
-This option will override any "real" Accumulo instance information.
+3. Configuration
-Once the properties file is configured correctly, run this command:
-./bin/accumulo proxy -p proxy/proxy.properties
+Please note the proxy server only functions correctly when connected to an
+Accumulo 1.4 instance, or when run standalone in the Mock configuration.
-3. Clients
+ - Edit the proxy.properties file.
+ - Change the useMockInstance value to 'true' if you wish to use an in-memory Mock instance.
+ - Change the useMiniAccumulo value to 'true' if you wish to use a Mini Accumulo Cluster.
+ - When using a "real" Accumulo instance:
+ - Ensure useMockInstance and useMiniAccumulo are both set to 'false'
+ - Set the instance name
+ - Set the list of ZooKeepers
-An example java client is incuded with this distribution in the class TestProxyClient. Also the unit tests included show how to use the proxy. Normal Accumulo APIs are emulated whereever possible.
+4. Execution
-Additional client examples can be found in the examples directory. These clients are tested and function, however, the setup for each language is beyond the scope of this document currently.
+Run the following command.
-In general, to create a client for a new language one must compile the proxy.thrift API (along with the accumulo core data.thrift API) to the language of choice.
-Then one includes the resulting generated code along with the thrift core library for that language to use the proxy.
+ ${ACCUMULO_HOME}/bin/accumulo proxy -p ${ACCUMULO_HOME}/proxy/proxy.properties
-As a note, many of the clients here assume "secret" is your root password. Protip: to make the examples work, change this code to your password, do not change your root password to "secret".
+5. Clients
+
+You need the language-specific library for Thrift installed to be able to use said Thrift client
+code in that language. In other words, you need to install the Python Thrift library to use the Python
+example. Typically, your operating system's package manager will be able to automatically install
+these for you in an expected location such as /usr/lib/python/site-packages/thrift.
+
+An example Java client is incuded with this distribution in the class TestProxyClient. Also the
+unit tests included show how to use the proxy. Normal Accumulo APIs are emulated wherever possible.
+
+Additional client examples can be found in the examples directory. These clients are tested and
+functional; however, the setup for each language is beyond the scope of this document currently.
[14/50] [abbrv] git commit: ACCUMULO-1429 use the specified tablename
Posted by el...@apache.org.
ACCUMULO-1429 use the specified tablename
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.4@1483954 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a3d5dc62
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a3d5dc62
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a3d5dc62
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: a3d5dc629347799f177ed578c85318e2c765257f
Parents: 55a97ce
Author: Eric C. Newton <ec...@apache.org>
Authored: Fri May 17 19:16:01 2013 +0000
Committer: Eric C. Newton <ec...@apache.org>
Committed: Fri May 17 19:16:01 2013 +0000
----------------------------------------------------------------------
.../org/apache/accumulo/core/util/shell/commands/ScanCommand.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3d5dc62/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ScanCommand.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ScanCommand.java b/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ScanCommand.java
index ca1ea83..275a07f 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ScanCommand.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ScanCommand.java
@@ -108,7 +108,7 @@ public class ScanCommand extends Command {
}
protected void addScanIterators(Shell shellState, Scanner scanner, String tableName) {
- List<IteratorSetting> tableScanIterators = shellState.scanIteratorOptions.get(shellState.getTableName());
+ List<IteratorSetting> tableScanIterators = shellState.scanIteratorOptions.get(tableName);
if (tableScanIterators == null) {
Shell.log.debug("Found no scan iterators to set");
return;
[18/50] [abbrv] Reversing ACCUMULO-1438.
Posted by el...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e46360cf/src/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloCluster.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloCluster.java b/src/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloCluster.java
new file mode 100644
index 0000000..2cb3ae2
--- /dev/null
+++ b/src/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloCluster.java
@@ -0,0 +1,364 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.mini;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.TimerTask;
+
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.server.logger.LogService;
+import org.apache.accumulo.server.master.Master;
+import org.apache.accumulo.server.tabletserver.TabletServer;
+import org.apache.accumulo.server.util.Initialize;
+import org.apache.accumulo.server.util.PortUtils;
+import org.apache.accumulo.server.util.time.SimpleTimer;
+import org.apache.accumulo.start.Main;
+import org.apache.zookeeper.server.ZooKeeperServerMain;
+
+/**
+ * A utility class that will create Zookeeper and Accumulo processes that write all of their data to a single local directory. This class makes it easy to test
+ * code against a real Accumulo instance. Its much more accurate for testing than MockAccumulo, but much slower than MockAccumulo.
+ *
+ * @since 1.5.0, 1.4.4
+ */
+public class MiniAccumuloCluster {
+
+ private static final String INSTANCE_SECRET = "DONTTELL";
+ private static final String INSTANCE_NAME = "miniInstance";
+
+ private static class LogWriter extends Thread {
+ private BufferedReader in;
+ private BufferedWriter out;
+
+ /**
+ * @throws java.io.IOException
+ */
+ public LogWriter(InputStream stream, File logFile) throws IOException {
+ this.setDaemon(true);
+ this.in = new BufferedReader(new InputStreamReader(stream));
+ out = new BufferedWriter(new FileWriter(logFile));
+
+ SimpleTimer.getInstance().schedule(new TimerTask() {
+ @Override
+ public void run() {
+ try {
+ flush();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }, 1000, 1000);
+ }
+
+ public synchronized void flush() throws IOException {
+ if (out != null)
+ out.flush();
+ }
+
+ @Override
+ public void run() {
+ String line;
+
+ try {
+ while ((line = in.readLine()) != null) {
+ out.append(line);
+ out.append("\n");
+ }
+
+ synchronized (this) {
+ out.close();
+ out = null;
+ in.close();
+ }
+
+ } catch (IOException e) {
+ }
+ }
+ }
+
+ private File libDir;
+ private File confDir;
+ private File zooKeeperDir;
+ private File accumuloDir;
+ private File zooCfgFile;
+ private File logDir;
+ private File walogDir;
+
+ private Process zooKeeperProcess;
+ private Process masterProcess;
+ private Process loggerProcess;
+
+ private int zooKeeperPort;
+
+ private List<LogWriter> logWriters = new ArrayList<LogWriter>();
+
+ private MiniAccumuloConfig config;
+ private Process[] tabletServerProcesses;
+
+ Process exec(Class<? extends Object> clazz, String... args) throws IOException {
+ String javaHome = System.getProperty("java.home");
+ String javaBin = javaHome + File.separator + "bin" + File.separator + "java";
+ String classpath = System.getProperty("java.class.path");
+
+ classpath = confDir.getAbsolutePath() + File.pathSeparator + classpath;
+
+ String className = clazz.getCanonicalName();
+
+ ArrayList<String> argList = new ArrayList<String>();
+
+ argList.addAll(Arrays.asList(javaBin, "-cp", classpath, "-Xmx128m", "-XX:+UseConcMarkSweepGC", "-XX:CMSInitiatingOccupancyFraction=75",
+ Main.class.getName(), className));
+
+ argList.addAll(Arrays.asList(args));
+
+ ProcessBuilder builder = new ProcessBuilder(argList);
+
+ builder.environment().put("ACCUMULO_HOME", config.getDir().getAbsolutePath());
+ builder.environment().put("ACCUMULO_LOG_DIR", logDir.getAbsolutePath());
+
+ // if we're running under accumulo.start, we forward these env vars
+ String env = System.getenv("HADOOP_PREFIX");
+ if (env != null)
+ builder.environment().put("HADOOP_PREFIX", env);
+ env = System.getenv("ZOOKEEPER_HOME");
+ if (env != null)
+ builder.environment().put("ZOOKEEPER_HOME", env);
+
+ Process process = builder.start();
+
+ LogWriter lw;
+ lw = new LogWriter(process.getErrorStream(), new File(logDir, clazz.getSimpleName() + "_" + process.hashCode() + ".err"));
+ logWriters.add(lw);
+ lw.start();
+ lw = new LogWriter(process.getInputStream(), new File(logDir, clazz.getSimpleName() + "_" + process.hashCode() + ".out"));
+ logWriters.add(lw);
+ lw.start();
+
+ return process;
+ }
+
+ private void appendProp(FileWriter fileWriter, Property key, String value, Map<String,String> siteConfig) throws IOException {
+ appendProp(fileWriter, key.getKey(), value, siteConfig);
+ }
+
+ private void appendProp(FileWriter fileWriter, String key, String value, Map<String,String> siteConfig) throws IOException {
+ if (!siteConfig.containsKey(key))
+ fileWriter.append("<property><name>" + key + "</name><value>" + value + "</value></property>\n");
+ }
+
+ /**
+ * @param dir
+ * An empty or nonexistant temp directoy that Accumulo and Zookeeper can store data in. Creating the directory is left to the user. Java 7, Guava,
+ * and Junit provide methods for creating temporary directories.
+ * @param rootPassword
+ * Initial root password for instance.
+ * @throws IOException
+ */
+ public MiniAccumuloCluster(File dir, String rootPassword) throws IOException {
+ this(new MiniAccumuloConfig(dir, rootPassword));
+ }
+
+ /**
+ * @param config
+ * initial configuration
+ * @throws IOException
+ */
+
+ public MiniAccumuloCluster(MiniAccumuloConfig config) throws IOException {
+
+ if (config.getDir().exists() && !config.getDir().isDirectory())
+ throw new IllegalArgumentException("Must pass in directory, " + config.getDir() + " is a file");
+
+ if (config.getDir().exists() && config.getDir().list().length != 0)
+ throw new IllegalArgumentException("Directory " + config.getDir() + " is not empty");
+
+ this.config = config;
+
+ libDir = new File(config.getDir(), "lib");
+ confDir = new File(config.getDir(), "conf");
+ accumuloDir = new File(config.getDir(), "accumulo");
+ zooKeeperDir = new File(config.getDir(), "zookeeper");
+ logDir = new File(config.getDir(), "logs");
+ walogDir = new File(config.getDir(), "walogs");
+
+ confDir.mkdirs();
+ accumuloDir.mkdirs();
+ zooKeeperDir.mkdirs();
+ logDir.mkdirs();
+ walogDir.mkdirs();
+ libDir.mkdirs();
+
+ zooKeeperPort = PortUtils.getRandomFreePort();
+
+ File siteFile = new File(confDir, "accumulo-site.xml");
+
+ FileWriter fileWriter = new FileWriter(siteFile);
+ fileWriter.append("<configuration>\n");
+
+ HashMap<String,String> siteConfig = new HashMap<String,String>(config.getSiteConfig());
+
+ appendProp(fileWriter, Property.INSTANCE_DFS_URI, "file:///", siteConfig);
+ appendProp(fileWriter, Property.INSTANCE_DFS_DIR, accumuloDir.getAbsolutePath(), siteConfig);
+ appendProp(fileWriter, Property.INSTANCE_ZK_HOST, "localhost:" + zooKeeperPort, siteConfig);
+ appendProp(fileWriter, Property.INSTANCE_SECRET, INSTANCE_SECRET, siteConfig);
+ appendProp(fileWriter, Property.MASTER_CLIENTPORT, "" + PortUtils.getRandomFreePort(), siteConfig);
+ appendProp(fileWriter, Property.TSERV_CLIENTPORT, "" + PortUtils.getRandomFreePort(), siteConfig);
+ appendProp(fileWriter, Property.TSERV_PORTSEARCH, "true", siteConfig);
+ appendProp(fileWriter, Property.LOGGER_DIR, walogDir.getAbsolutePath(), siteConfig);
+ appendProp(fileWriter, Property.TSERV_DATACACHE_SIZE, "10M", siteConfig);
+ appendProp(fileWriter, Property.TSERV_INDEXCACHE_SIZE, "10M", siteConfig);
+ appendProp(fileWriter, Property.TSERV_MAXMEM, "50M", siteConfig);
+ appendProp(fileWriter, Property.TSERV_WALOG_MAX_SIZE, "100M", siteConfig);
+ appendProp(fileWriter, Property.TSERV_NATIVEMAP_ENABLED, "false", siteConfig);
+ appendProp(fileWriter, Property.TRACE_PORT, "" + PortUtils.getRandomFreePort(), siteConfig);
+ appendProp(fileWriter, Property.LOGGER_SORT_BUFFER_SIZE, "50M", siteConfig);
+ appendProp(fileWriter, Property.LOGGER_PORTSEARCH, "true", siteConfig);
+
+ // since there is a small amount of memory, check more frequently for majc... setting may not be needed in 1.5
+ appendProp(fileWriter, Property.TSERV_MAJC_DELAY, "3", siteConfig);
+ String cp = System.getenv("ACCUMULO_HOME") + "/lib/.*.jar," + "$ZOOKEEPER_HOME/zookeeper[^.].*.jar," + "$HADOOP_HOME/[^.].*.jar,"
+ + "$HADOOP_HOME/lib/[^.].*.jar," + "$HADOOP_PREFIX/share/hadoop/common/.*.jar," + "$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,"
+ + "$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar," + "$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar";
+ appendProp(fileWriter, Property.GENERAL_CLASSPATHS, cp, siteConfig);
+ appendProp(fileWriter, Property.GENERAL_DYNAMIC_CLASSPATHS, libDir.getAbsolutePath(), siteConfig);
+
+ for (Entry<String,String> entry : siteConfig.entrySet())
+ fileWriter.append("<property><name>" + entry.getKey() + "</name><value>" + entry.getValue() + "</value></property>\n");
+ fileWriter.append("</configuration>\n");
+ fileWriter.close();
+
+ zooCfgFile = new File(confDir, "zoo.cfg");
+ fileWriter = new FileWriter(zooCfgFile);
+
+ // zookeeper uses Properties to read its config, so use that to write in order to properly escape things like Windows paths
+ Properties zooCfg = new Properties();
+ zooCfg.setProperty("tickTime", "1000");
+ zooCfg.setProperty("initLimit", "10");
+ zooCfg.setProperty("syncLimit", "5");
+ zooCfg.setProperty("clientPort", zooKeeperPort + "");
+ zooCfg.setProperty("maxClientCnxns", "100");
+ zooCfg.setProperty("dataDir", zooKeeperDir.getAbsolutePath());
+ zooCfg.store(fileWriter, null);
+
+ fileWriter.close();
+ }
+
+ /**
+ * Starts Accumulo and Zookeeper processes. Can only be called once.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws IllegalStateException
+ * if already started
+ */
+ public void start() throws IOException, InterruptedException {
+ if (zooKeeperProcess != null)
+ throw new IllegalStateException("Already started");
+
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ try {
+ MiniAccumuloCluster.this.stop();
+ } catch (IOException e) {
+ e.printStackTrace();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ });
+
+ zooKeeperProcess = exec(Main.class, ZooKeeperServerMain.class.getName(), zooCfgFile.getAbsolutePath());
+
+ // sleep a little bit to let zookeeper come up before calling init, seems to work better
+ UtilWaitThread.sleep(250);
+
+ // TODO initialization could probably be done in process
+ Process initProcess = exec(Initialize.class);
+ initProcess.getOutputStream().write((INSTANCE_NAME + "\n").getBytes());
+ initProcess.getOutputStream().write((config.getRootPassword() + "\n").getBytes());
+ initProcess.getOutputStream().write((config.getRootPassword() + "\n").getBytes());
+ initProcess.getOutputStream().flush();
+ int ret = initProcess.waitFor();
+ if (ret != 0) {
+ throw new RuntimeException("Initialize process returned " + ret);
+ }
+
+ tabletServerProcesses = new Process[config.getNumTservers()];
+ for (int i = 0; i < config.getNumTservers(); i++) {
+ tabletServerProcesses[i] = exec(TabletServer.class);
+ }
+
+ loggerProcess = exec(LogService.class);
+ masterProcess = exec(Master.class);
+ }
+
+ /**
+ * @return Accumulo instance name
+ */
+
+ public String getInstanceName() {
+ return INSTANCE_NAME;
+ }
+
+ /**
+ * @return zookeeper connection string
+ */
+
+ public String getZooKeepers() {
+ return "localhost:" + zooKeeperPort;
+ }
+
+ /**
+ * Stops Accumulo and Zookeeper processes. If stop is not called, there is a shutdown hook that is setup to kill the processes. Howerver its probably best to
+ * call stop in a finally block as soon as possible.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ */
+
+ public void stop() throws IOException, InterruptedException {
+ if (zooKeeperProcess != null)
+ zooKeeperProcess.destroy();
+ if (loggerProcess != null)
+ loggerProcess.destroy();
+ if (masterProcess != null)
+ masterProcess.destroy();
+ if (tabletServerProcesses != null) {
+ for (Process tserver : tabletServerProcesses) {
+ tserver.destroy();
+ }
+ }
+
+ for (LogWriter lw : logWriters)
+ lw.flush();
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e46360cf/src/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloConfig.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloConfig.java b/src/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloConfig.java
new file mode 100644
index 0000000..a82f277
--- /dev/null
+++ b/src/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloConfig.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.mini;
+
+import java.io.File;
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * Holds configuration for {@link MiniAccumuloCluster}. Required configurations must be passed to constructor and all other configurations are optional.
+ *
+ * @since 1.5.0, 1.4.4
+ */
+
+public class MiniAccumuloConfig {
+
+ private File dir = null;
+ private String rootPassword = null;
+ private Map<String,String> siteConfig = Collections.emptyMap();
+ private int numTservers = 2;
+
+ /**
+ * @param dir
+ * An empty or nonexistant temp directoy that Accumulo and Zookeeper can store data in. Creating the directory is left to the user. Java 7, Guava,
+ * and Junit provide methods for creating temporary directories.
+ * @param rootPassword
+ * The initial password for the Accumulo root user
+ */
+
+ public MiniAccumuloConfig(File dir, String rootPassword) {
+ this.dir = dir;
+ this.rootPassword = rootPassword;
+ }
+
+ public File getDir() {
+ return dir;
+ }
+
+ public String getRootPassword() {
+ return rootPassword;
+ }
+
+ public int getNumTservers() {
+ return numTservers;
+ }
+
+ /**
+ * Calling this method is optional. If not set, it defaults to two.
+ *
+ * @param numTservers
+ * the number of tablet servers that mini accumulo cluster should start
+ */
+
+ public MiniAccumuloConfig setNumTservers(int numTservers) {
+ if (numTservers < 1)
+ throw new IllegalArgumentException("Must have at least one tablet server");
+ this.numTservers = numTservers;
+ return this;
+ }
+
+ public Map<String,String> getSiteConfig() {
+ return siteConfig;
+ }
+
+ /**
+ * Calling this method is optional. If not set, it defautls to an empty map.
+ *
+ * @param siteConfig
+ * key/values that you normally put in accumulo-site.xml can be put here
+ */
+ public MiniAccumuloConfig setSiteConfig(Map<String,String> siteConfig) {
+ this.siteConfig = siteConfig;
+ return this;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e46360cf/src/server/src/test/java/org/apache/accumulo/server/mini/MiniAccumuloClusterTest.java
----------------------------------------------------------------------
diff --git a/src/server/src/test/java/org/apache/accumulo/server/mini/MiniAccumuloClusterTest.java b/src/server/src/test/java/org/apache/accumulo/server/mini/MiniAccumuloClusterTest.java
new file mode 100644
index 0000000..083d4b8
--- /dev/null
+++ b/src/server/src/test/java/org/apache/accumulo/server/mini/MiniAccumuloClusterTest.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.mini;
+
+import java.util.Collections;
+import java.util.Map.Entry;
+import java.util.UUID;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.LongCombiner;
+import org.apache.accumulo.core.iterators.user.SummingCombiner;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.server.mini.MiniAccumuloCluster;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+public class MiniAccumuloClusterTest {
+
+ private static final Logger logger = Logger.getLogger(MiniAccumuloClusterTest.class);
+
+ private static TemporaryFolder tmpDir = new TemporaryFolder();
+ private static MiniAccumuloCluster accumulo;
+
+ @BeforeClass
+ public static void setupMiniCluster() throws Exception {
+
+ tmpDir.create();
+ logger.info("MiniCluster started @ " + tmpDir.getRoot());
+ Logger.getLogger("org.apache.zookeeper").setLevel(Level.ERROR);
+
+ accumulo = new MiniAccumuloCluster(tmpDir.getRoot(), "superSecret");
+ accumulo.start();
+ }
+
+ @Test(timeout = 30000)
+ public void test() throws Exception {
+ Connector conn = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers()).getConnector("root", "superSecret".getBytes());
+
+ conn.tableOperations().create("table1");
+
+ conn.securityOperations().createUser("user1", "pass1".getBytes(), new Authorizations("A", "B"));
+ conn.securityOperations().grantTablePermission("user1", "table1", TablePermission.WRITE);
+ conn.securityOperations().grantTablePermission("user1", "table1", TablePermission.READ);
+
+ IteratorSetting is = new IteratorSetting(10, SummingCombiner.class);
+ SummingCombiner.setEncodingType(is, LongCombiner.Type.STRING);
+ SummingCombiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("META", "COUNT")));
+
+ conn.tableOperations().attachIterator("table1", is);
+
+ Connector uconn = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers()).getConnector("user1", "pass1".getBytes());
+
+ BatchWriter bw = uconn.createBatchWriter("table1", 10000, 1000000, 2);
+
+ UUID uuid = UUID.randomUUID();
+
+ Mutation m = new Mutation(uuid.toString());
+ m.put("META", "SIZE", new ColumnVisibility("A|B"), "8");
+ m.put("META", "CRC", new ColumnVisibility("A|B"), "456");
+ m.put("META", "COUNT", new ColumnVisibility("A|B"), "1");
+ m.put("DATA", "IMG", new ColumnVisibility("A&B"), "ABCDEFGH");
+
+ bw.addMutation(m);
+ bw.flush();
+
+ m = new Mutation(uuid.toString());
+ m.put("META", "COUNT", new ColumnVisibility("A|B"), "1");
+ m.put("META", "CRC", new ColumnVisibility("A|B"), "123");
+ bw.addMutation(m);
+
+ bw.close();
+
+ int count = 0;
+ Scanner scanner = uconn.createScanner("table1", new Authorizations("A"));
+ for (Entry<Key,Value> entry : scanner) {
+ if (entry.getKey().getColumnQualifierData().toString().equals("COUNT")) {
+ Assert.assertEquals("2", entry.getValue().toString());
+ } else if (entry.getKey().getColumnQualifierData().toString().equals("SIZE")) {
+ Assert.assertEquals("8", entry.getValue().toString());
+ } else if (entry.getKey().getColumnQualifierData().toString().equals("CRC")) {
+ Assert.assertEquals("123", entry.getValue().toString());
+ } else {
+ Assert.assertTrue(false);
+ }
+ count++;
+ }
+
+ Assert.assertEquals(3, count);
+
+ count = 0;
+ scanner = uconn.createScanner("table1", new Authorizations("A", "B"));
+ for (Entry<Key,Value> entry : scanner) {
+ if (entry.getKey().getColumnQualifierData().toString().equals("IMG")) {
+ Assert.assertEquals("ABCDEFGH", entry.getValue().toString());
+ }
+ count++;
+ }
+
+ Assert.assertEquals(4, count);
+
+ conn.tableOperations().delete("table1");
+ }
+
+ @Test(timeout = 20000)
+ public void testMultipleTabletServersRunning() throws AccumuloException, AccumuloSecurityException {
+
+ Connector conn = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers()).getConnector("root", "superSecret".getBytes());
+
+ while (conn.instanceOperations().getTabletServers().size() != 2) {
+ UtilWaitThread.sleep(500);
+ }
+ }
+
+ @AfterClass
+ public static void tearDownMiniCluster() throws Exception {
+ accumulo.stop();
+ tmpDir.delete();
+ }
+}
[17/50] [abbrv] git commit: Adding ignore for src/mini/target snd
eclipse project files
Posted by el...@apache.org.
Adding ignore for src/mini/target snd eclipse project files
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.4@1484641 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7bc56865
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7bc56865
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7bc56865
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 7bc56865ffb773a2c7134a0b004c16379e3a40fc
Parents: 9060cb7
Author: Corey J. Nolet <cj...@apache.org>
Authored: Tue May 21 01:57:50 2013 +0000
Committer: Corey J. Nolet <cj...@apache.org>
Committed: Tue May 21 01:57:50 2013 +0000
----------------------------------------------------------------------
----------------------------------------------------------------------
[25/50] [abbrv] git commit: ACCUMULO-1498 ensure 1.3.6 tag merges
cleanly into 1.4 branch by updating svn metadata
Posted by el...@apache.org.
ACCUMULO-1498 ensure 1.3.6 tag merges cleanly into 1.4 branch by updating svn metadata
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.4@1490749 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/8fd1c3b0
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/8fd1c3b0
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/8fd1c3b0
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 8fd1c3b0af87c42a69555a1945c89e222f37a45c
Parents: 07b18f2
Author: Christopher Tubbs <ct...@apache.org>
Authored: Fri Jun 7 17:41:20 2013 +0000
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Fri Jun 7 17:41:20 2013 +0000
----------------------------------------------------------------------
----------------------------------------------------------------------
[34/50] [abbrv] git commit: ACCUMULO-1519 removing force flag from
deletetable commands in cloudstone tests
Posted by el...@apache.org.
ACCUMULO-1519 removing force flag from deletetable commands in cloudstone tests
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.4@1500455 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/732e881e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/732e881e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/732e881e
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 732e881e57aace131e0dd45043dbd1f53c4c5e1f
Parents: d5da59a
Author: Mike Drob <md...@apache.org>
Authored: Sun Jul 7 15:37:10 2013 +0000
Committer: Mike Drob <md...@apache.org>
Committed: Sun Jul 7 15:37:10 2013 +0000
----------------------------------------------------------------------
test/system/bench/lib/CreateTablesBenchmark.py | 4 ++--
test/system/bench/lib/IngestBenchmark.py | 4 ++--
test/system/bench/lib/RowHashBenchmark.py | 8 ++++----
test/system/bench/lib/TableSplitsBenchmark.py | 4 ++--
test/system/bench/lib/TeraSortBenchmark.py | 4 ++--
5 files changed, 12 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/732e881e/test/system/bench/lib/CreateTablesBenchmark.py
----------------------------------------------------------------------
diff --git a/test/system/bench/lib/CreateTablesBenchmark.py b/test/system/bench/lib/CreateTablesBenchmark.py
index 8e129b7..f900108 100755
--- a/test/system/bench/lib/CreateTablesBenchmark.py
+++ b/test/system/bench/lib/CreateTablesBenchmark.py
@@ -35,7 +35,7 @@ class CreateTablesBenchmark(Benchmark):
log.debug("Checking for table existence: %s" % currentTable)
code, out, err = cloudshell.run(self.username, self.password, 'table %s\n' % currentTable)
if out.find('does not exist') == -1:
- command = 'deletetable -f %s\n' % (currentTable)
+ command = 'deletetable %s\n' % (currentTable)
log.debug("Running Command %r", command)
code, out, err = cloudshell.run(self.username, self.password, command)
self.assertEqual(code, 0, 'Did not successfully delete table: %s' % currentTable)
@@ -51,7 +51,7 @@ class CreateTablesBenchmark(Benchmark):
# print err
for x in range(1, self.tables):
currentTable = 'test_ingest%d' % (x)
- command = 'deletetable -f %s\n' % (currentTable)
+ command = 'deletetable %s\n' % (currentTable)
log.debug("Running Command %r", command)
code, out, err = cloudshell.run(self.username, self.password, command)
self.assertEqual(code, 0, 'Did not successfully delete table: %s' % currentTable)
http://git-wip-us.apache.org/repos/asf/accumulo/blob/732e881e/test/system/bench/lib/IngestBenchmark.py
----------------------------------------------------------------------
diff --git a/test/system/bench/lib/IngestBenchmark.py b/test/system/bench/lib/IngestBenchmark.py
index 32023f4..9370810 100755
--- a/test/system/bench/lib/IngestBenchmark.py
+++ b/test/system/bench/lib/IngestBenchmark.py
@@ -32,14 +32,14 @@ class IngestBenchmark(Benchmark):
code, out, err = cloudshell.run(self.username, self.password, 'table test_ingest\n')
if out.find('does not exist') == -1:
log.debug("Deleting table test_ingest")
- code, out, err = cloudshell.run(self.username, self.password, 'deletetable -f test_ingest\n')
+ code, out, err = cloudshell.run(self.username, self.password, 'deletetable test_ingest\n')
self.assertEquals(code, 0, "Could not delete the table 'test_ingest'")
code, out, err = cloudshell.run(self.username, self.password, 'createtable test_ingest\n')
self.assertEqual(code, 0, "Could not create the table 'test_ingest'")
Benchmark.setUp(self)
def tearDown(self):
- command = 'deletetable -f test_ingest\n'
+ command = 'deletetable test_ingest\n'
log.debug("Running Command %r", command)
code, out, err = cloudshell.run(self.username, self.password, command)
self.assertEqual(code, 0, "Could not delete the table 'test_ingest'")
http://git-wip-us.apache.org/repos/asf/accumulo/blob/732e881e/test/system/bench/lib/RowHashBenchmark.py
----------------------------------------------------------------------
diff --git a/test/system/bench/lib/RowHashBenchmark.py b/test/system/bench/lib/RowHashBenchmark.py
index 1f678bd..95c91f7 100755
--- a/test/system/bench/lib/RowHashBenchmark.py
+++ b/test/system/bench/lib/RowHashBenchmark.py
@@ -46,12 +46,12 @@ class RowHashBenchmark(Benchmark):
file = os.path.join( dir, 'splits' )
code, out, err = cloudshell.run(self.username, self.password, 'table %s\n' % self.input_table)
if out.find('does not exist') == -1:
- code, out, err = cloudshell.run(self.username, self.password, 'deletetable -f %s\n' % self.input_table)
+ code, out, err = cloudshell.run(self.username, self.password, 'deletetable %s\n' % self.input_table)
self.sleep(15)
code, out, err = cloudshell.run(self.username, self.password, "createtable %s -sf %s\n" % (self.input_table, file))
code, out, err = cloudshell.run(self.username, self.password, 'table %s\n' % self.output_table)
if out.find('does not exist') == -1:
- code, out, err = cloudshell.run(self.username, self.password, 'deletetable -f %s\n' %
+ code, out, err = cloudshell.run(self.username, self.password, 'deletetable %s\n' %
self.output_table)
self.sleep(15)
code, out, err = cloudshell.run(self.username, self.password, "createtable %s -sf %s\n" % (self.output_table, file))
@@ -73,9 +73,9 @@ class RowHashBenchmark(Benchmark):
Benchmark.setUp(self)
def tearDown(self):
- code, out, err = cloudshell.run(self.username, self.password, "deletetable -f %s\n" % self.input_table)
+ code, out, err = cloudshell.run(self.username, self.password, "deletetable %s\n" % self.input_table)
self.assertEqual(code, 0, 'Could not delete %s, %s' % (self.input_table, out))
- code, out, err = cloudshell.run(self.username, self.password, "deletetable -f %s\n" % self.output_table)
+ code, out, err = cloudshell.run(self.username, self.password, "deletetable %s\n" % self.output_table)
self.assertEqual(code, 0, 'Could not delete %s, %s' % (self.output_table, out))
Benchmark.tearDown(self)
http://git-wip-us.apache.org/repos/asf/accumulo/blob/732e881e/test/system/bench/lib/TableSplitsBenchmark.py
----------------------------------------------------------------------
diff --git a/test/system/bench/lib/TableSplitsBenchmark.py b/test/system/bench/lib/TableSplitsBenchmark.py
index 67414ea..ea509b8 100755
--- a/test/system/bench/lib/TableSplitsBenchmark.py
+++ b/test/system/bench/lib/TableSplitsBenchmark.py
@@ -39,7 +39,7 @@ class TableSplitsBenchmark(Benchmark):
code, out, err = cloudshell.run(self.username, self.password, 'table %s\n' % self.tablename)
if out.find('does not exist') == -1:
log.debug('Deleting table %s' % self.tablename)
- code, out, err = cloudshell.run(self.username, self.password, 'deletetable -f %s\n' % self.tablename)
+ code, out, err = cloudshell.run(self.username, self.password, 'deletetable %s\n' % self.tablename)
self.assertEqual(code, 0, "Could not delete table")
Benchmark.setUp(self)
@@ -54,7 +54,7 @@ class TableSplitsBenchmark(Benchmark):
return 'Creates a table with splits. Lower score is better.'
def tearDown(self):
- command = 'deletetable -f %s\n' % self.tablename
+ command = 'deletetable %s\n' % self.tablename
log.debug("Running Command %r", command)
code, out, err = cloudshell.run(self.username, self.password, command)
self.assertEqual(code, 0, "Could not delete table")
http://git-wip-us.apache.org/repos/asf/accumulo/blob/732e881e/test/system/bench/lib/TeraSortBenchmark.py
----------------------------------------------------------------------
diff --git a/test/system/bench/lib/TeraSortBenchmark.py b/test/system/bench/lib/TeraSortBenchmark.py
index 5f3e1b7..602e07a 100755
--- a/test/system/bench/lib/TeraSortBenchmark.py
+++ b/test/system/bench/lib/TeraSortBenchmark.py
@@ -45,11 +45,11 @@ class TeraSortBenchmark(Benchmark):
code, out, err = cloudshell.run(self.username, self.password, 'table %s\n' % self.tablename)
if out.find('does not exist') == -1:
log.debug('Deleting table %s' % self.tablename)
- code, out, err = cloudshell.run(self.username, self.password, 'deletetable -f %s\n' % self.tablename)
+ code, out, err = cloudshell.run(self.username, self.password, 'deletetable %s\n' % self.tablename)
Benchmark.setUp(self)
def tearDown(self):
- code, out, err = cloudshell.run(self.username, self.password, "deletetable -f %s\n" % self.tablename)
+ code, out, err = cloudshell.run(self.username, self.password, "deletetable %s\n" % self.tablename)
self.assertEqual(code, 0, 'Could not delete %s, %s' % (self.tablename, out))
Benchmark.tearDown(self)