You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2014/03/12 06:12:15 UTC
[01/14] git commit: ACCUMULO-2438 Timeout is a little too tight on a
busy system.
Repository: accumulo
Updated Branches:
refs/heads/ACCUMULO-2061 243aafa49 -> db76dd751
ACCUMULO-2438 Timeout is a little too tight on a busy system.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/6c050913
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/6c050913
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/6c050913
Branch: refs/heads/ACCUMULO-2061
Commit: 6c050913cdf8c0fa0743f6a4408f4808026f11c8
Parents: 4cc25ca
Author: Josh Elser <el...@apache.org>
Authored: Tue Mar 11 00:25:34 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Tue Mar 11 00:25:34 2014 -0400
----------------------------------------------------------------------
test/src/test/java/org/apache/accumulo/test/DeleteRowsTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6c050913/test/src/test/java/org/apache/accumulo/test/DeleteRowsTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/DeleteRowsTest.java b/test/src/test/java/org/apache/accumulo/test/DeleteRowsTest.java
index a627dc1..9e5921d 100644
--- a/test/src/test/java/org/apache/accumulo/test/DeleteRowsTest.java
+++ b/test/src/test/java/org/apache/accumulo/test/DeleteRowsTest.java
@@ -60,7 +60,7 @@ public class DeleteRowsTest {
folder.delete();
}
- @Test(timeout = 120 * 1000)
+ @Test(timeout = 5 * 60 * 1000)
public void test() throws Exception {
ZooKeeperInstance zk = new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers());
Connector c = zk.getConnector("root", new PasswordToken(secret));
[08/14] git commit: ACCUMULO-2267 added a basic config rpm,
initialize hdfs for accumulo, deal with versions on the accumulo jars,
fix the start-up scripts, tested on Centos6.5
Posted by el...@apache.org.
ACCUMULO-2267 added a basic config rpm, initialize hdfs for accumulo, deal with versions on the accumulo jars, fix the start-up scripts, tested on Centos6.5
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/fc4de902
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/fc4de902
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/fc4de902
Branch: refs/heads/ACCUMULO-2061
Commit: fc4de9022e3b82e71c4fca51e4f80b2cacb3ea90
Parents: 6b2704c
Author: Eric Newton <er...@gmail.com>
Authored: Tue Mar 11 14:56:13 2014 -0400
Committer: Eric Newton <er...@gmail.com>
Committed: Tue Mar 11 14:56:25 2014 -0400
----------------------------------------------------------------------
assemble/pom.xml | 7 ++
bin/accumulo | 5 +
core/pom.xml | 2 +-
init/pom.xml | 98 ++++++++++++++++++++
.../main/packaging/etc/accumulo/accumulo-env.sh | 65 +++++++++++++
.../packaging/etc/accumulo/accumulo-site.xml | 87 +++++++++++++++++
.../main/packaging/etc/accumulo/auditLog.xml | 41 ++++++++
init/src/main/packaging/etc/accumulo/gc | 1 +
.../packaging/etc/accumulo/generic_logger.xml | 83 +++++++++++++++++
.../packaging/etc/accumulo/log4j.properties | 41 ++++++++
init/src/main/packaging/etc/accumulo/masters | 1 +
.../packaging/etc/accumulo/monitor_logger.xml | 64 +++++++++++++
init/src/main/packaging/etc/accumulo/monitors | 1 +
init/src/main/packaging/etc/accumulo/slaves | 1 +
init/src/main/packaging/etc/accumulo/tracers | 1 +
pom.xml | 13 +++
server/base/pom.xml | 7 +-
.../src/main/packaging/etc/init.d/accumulo-gc | 6 +-
.../main/packaging/etc/init.d/accumulo-master | 6 +-
.../main/packaging/etc/init.d/accumulo-monitor | 6 +-
.../main/packaging/etc/init.d/accumulo-tracer | 6 +-
.../main/packaging/etc/init.d/accumulo-tserver | 9 +-
22 files changed, 532 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/fc4de902/assemble/pom.xml
----------------------------------------------------------------------
diff --git a/assemble/pom.xml b/assemble/pom.xml
index c0f0720..23a2187 100644
--- a/assemble/pom.xml
+++ b/assemble/pom.xml
@@ -496,6 +496,12 @@
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-init</artifactId>
+ <classifier>el6</classifier>
+ <type>rpm</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-master</artifactId>
<classifier>el6</classifier>
<type>rpm</type>
@@ -573,6 +579,7 @@
<require>accumulo-native = %{version}-%{release}</require>
<require>accumulo-tracer = %{version}-%{release}</require>
<require>accumulo-tserver = %{version}-%{release}</require>
+ <require>accumulo-init = %{version}-%{release}</require>
</requires>
<mappings>
<mapping>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/fc4de902/bin/accumulo
----------------------------------------------------------------------
diff --git a/bin/accumulo b/bin/accumulo
index 75df322..256598c 100755
--- a/bin/accumulo
+++ b/bin/accumulo
@@ -29,6 +29,11 @@ script=$( basename "${SOURCE}" )
. "${bin}"/config.sh
START_JAR="${ACCUMULO_HOME}/lib/accumulo-start.jar"
+if [ ! -f "$START_JAR" ]; then
+ if [ -x /bin/rpm ]; then
+ START_JAR=$(echo $(/bin/rpm -E "%{_javadir}/accumulo/"accumulo-start-*.jar))
+ fi
+fi
#
# Resolve a program to its installation directory
http://git-wip-us.apache.org/repos/asf/accumulo/blob/fc4de902/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index 10c9bdd..d56ca2c 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -257,7 +257,7 @@
<require>accumulo-start = %{version}-%{release}</require>
</requires>
<preinstallScriptlet>
- <script>getent group accumulo >/dev/null || groupadd -r accumulo; getent passwd accumulo >/dev/null || useradd -r -g accumulo -d %{_datadir}/accumulo -s /sbin/nologin -c "Apache Accumulo" accumulo; exit 0</script>
+ <script>getent group accumulo >/dev/null || groupadd -r accumulo; getent passwd accumulo >/dev/null || useradd -r -g accumulo -d %{_datadir}/accumulo -s /bin/bash -c "Apache Accumulo" accumulo; exit 0</script>
</preinstallScriptlet>
<postinstallScriptlet>
<script>sysctl -q -p %{_sysconfdir}/sysctl.d/accumulo; ln -s %{_sysconfdir}/accumulo %{_datadir}/accumulo/conf</script>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/fc4de902/init/pom.xml
----------------------------------------------------------------------
diff --git a/init/pom.xml b/init/pom.xml
new file mode 100644
index 0000000..dfd1ff8
--- /dev/null
+++ b/init/pom.xml
@@ -0,0 +1,98 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-project</artifactId>
+ <version>1.6.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+ <artifactId>accumulo-init</artifactId>
+ <name>Init</name>
+ <description>Initialize and start a single-node accumulo instance.</description>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-start</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.thrift</groupId>
+ <artifactId>libthrift</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ </dependency>
+ </dependencies>
+ <profiles>
+ <profile>
+ <id>rpm</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>rpm-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>build-rpm</id>
+ <goals>
+ <goal>attached-rpm</goal>
+ </goals>
+ <phase>package</phase>
+ <configuration>
+ <requires>
+ <require>java-sdk >= 1.6.0</require>
+ <require>accumulo-core = %{version}-%{release}</require>
+ </requires>
+ <mappings>
+ <mapping>
+ <configuration>true</configuration>
+ <directory>%{_sysconfdir}/accumulo</directory>
+ <filemode>700</filemode>
+ <username>accumulo</username>
+ <sources>
+ <source>
+ <location>src/main/packaging/etc/accumulo</location>
+ <includes>
+ <include>*/**</include>
+ </includes>
+ </source>
+ </sources>
+ </mapping>
+ </mappings>
+ <postinstallScriptlet>
+ <script>su hdfs sh -c 'hadoop fs -mkdir /accumulo ; hadoop fs -chown accumulo /accumulo' ; su accumulo sh -c '%{_datadir}/accumulo/bin/accumulo init --instance-name instance --password secret'</script>
+ </postinstallScriptlet>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+</project>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/fc4de902/init/src/main/packaging/etc/accumulo/accumulo-env.sh
----------------------------------------------------------------------
diff --git a/init/src/main/packaging/etc/accumulo/accumulo-env.sh b/init/src/main/packaging/etc/accumulo/accumulo-env.sh
new file mode 100755
index 0000000..0ac9374
--- /dev/null
+++ b/init/src/main/packaging/etc/accumulo/accumulo-env.sh
@@ -0,0 +1,65 @@
+#! /usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+###
+### Configure these environment variables to point to your local installations.
+###
+### The functional tests require conditional values, so keep this style:
+###
+### test -z "$JAVA_HOME" && export JAVA_HOME=/usr/local/lib/jdk-1.6.0
+###
+###
+### Note that the -Xmx -Xms settings below require substantial free memory:
+### you may want to use smaller values, especially when running everything
+### on a single machine.
+###
+if [ -z "$HADOOP_HOME" ]
+then
+ test -z "$HADOOP_PREFIX" && export HADOOP_PREFIX=/usr/lib/hadoop
+else
+ HADOOP_PREFIX="$HADOOP_HOME"
+ unset HADOOP_HOME
+fi
+
+# hadoop-1.2:
+# test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="$HADOOP_PREFIX/conf"
+test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="/etc/hadoop/conf"
+
+test -z "$JAVA_HOME" && export JAVA_HOME=/usr/lib/jvm/jre
+test -z "$ZOOKEEPER_HOME" && export ZOOKEEPER_HOME=/usr/lib/zookeeper
+test -z "$ACCUMULO_LOG_DIR" && export ACCUMULO_LOG_DIR=/var/log/accumulo
+if [ -f ${ACCUMULO_CONF_DIR}/accumulo.policy ]
+then
+ POLICY="-Djava.security.manager -Djava.security.policy=${ACCUMULO_CONF_DIR}/accumulo.policy"
+fi
+test -z "$ACCUMULO_TSERVER_OPTS" && export ACCUMULO_TSERVER_OPTS="${POLICY} -Xmx1g -Xms1g -XX:NewSize=500m -XX:MaxNewSize=500m "
+test -z "$ACCUMULO_MASTER_OPTS" && export ACCUMULO_MASTER_OPTS="${POLICY} -Xmx1g -Xms1g"
+test -z "$ACCUMULO_MONITOR_OPTS" && export ACCUMULO_MONITOR_OPTS="${POLICY} -Xmx1g -Xms256m"
+test -z "$ACCUMULO_GC_OPTS" && export ACCUMULO_GC_OPTS="-Xmx256m -Xms256m"
+test -z "$ACCUMULO_GENERAL_OPTS" && export ACCUMULO_GENERAL_OPTS="-XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -Djava.net.preferIPv4Stack=true"
+test -z "$ACCUMULO_OTHER_OPTS" && export ACCUMULO_OTHER_OPTS="-Xmx1g -Xms256m"
+# what do when the JVM runs out of heap memory
+export ACCUMULO_KILL_CMD='kill -9 %p'
+
+### Optionally look for hadoop and accumulo native libraries for your
+### platform in additional directories. (Use DYLD_LIBRARY_PATH on Mac OS X.)
+### May not be necessary for Hadoop 2.x or using an RPM that installs to
+### the correct system library directory.
+# export LD_LIBRARY_PATH=${HADOOP_PREFIX}/lib/native/${PLATFORM}:${LD_LIBRARY_PATH}
+
+# Should the monitor bind to all network interfaces -- default: false
+# export ACCUMULO_MONITOR_BIND_ALL="true"
http://git-wip-us.apache.org/repos/asf/accumulo/blob/fc4de902/init/src/main/packaging/etc/accumulo/accumulo-site.xml
----------------------------------------------------------------------
diff --git a/init/src/main/packaging/etc/accumulo/accumulo-site.xml b/init/src/main/packaging/etc/accumulo/accumulo-site.xml
new file mode 100644
index 0000000..12cb1b9
--- /dev/null
+++ b/init/src/main/packaging/etc/accumulo/accumulo-site.xml
@@ -0,0 +1,87 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+ <!-- Put your site-specific accumulo configurations here. The available configuration values along with their defaults are documented in docs/config.html Unless
+ you are simply testing at your workstation, you will most definitely need to change the three entries below. -->
+
+ <property>
+ <name>instance.zookeeper.host</name>
+ <value>localhost:2181</value>
+ <description>comma separated list of zookeeper servers</description>
+ </property>
+
+ <property>
+ <name>instance.secret</name>
+ <value>CHANGEME</value>
+ <description>A secret unique to a given instance that all servers must know in order to communicate with one another.
+ Change it before initialization. To
+ change it later use ./bin/accumulo org.apache.accumulo.server.util.ChangeSecret --old [oldpasswd] --new [newpasswd],
+ and then update this file.
+ </description>
+ </property>
+
+ <property>
+ <name>tserver.memory.maps.max</name>
+ <value>1G</value>
+ </property>
+
+ <property>
+ <name>tserver.cache.data.size</name>
+ <value>128M</value>
+ </property>
+
+ <property>
+ <name>tserver.cache.index.size</name>
+ <value>128M</value>
+ </property>
+
+ <property>
+ <name>trace.token.property.password</name>
+ <!-- change this to the root user's password, and/or change the user below -->
+ <value>secret</value>
+ </property>
+
+ <property>
+ <name>trace.user</name>
+ <value>root</value>
+ </property>
+
+ <property>
+ <name>general.classpaths</name>
+ <value>
+ /usr/share/java/accumulo/.*.jar,
+ /usr/share/accumulo/lib/.*.jar,
+ /usr/lib/zookeeper.*.jar,
+ /etc/hadoop/conf,
+ /usr/lib/hadoop/.*.jar,
+ /usr/lib/hadoop/lib/.*.jar,
+ /usr/lib/hadoop-hdfs/.*.jar,
+ /usr/lib/hadoop-mapreduce/.*.jar,
+ /usr/lib/hadoop-yarn/.*.jar,
+ </value>
+ <description>Classpaths that accumulo checks for updates and class files.</description>
+ </property>
+
+ <property>
+ <name>general.dynamic.classpaths</name>
+ <value></value>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/fc4de902/init/src/main/packaging/etc/accumulo/auditLog.xml
----------------------------------------------------------------------
diff --git a/init/src/main/packaging/etc/accumulo/auditLog.xml b/init/src/main/packaging/etc/accumulo/auditLog.xml
new file mode 100644
index 0000000..9b7987e
--- /dev/null
+++ b/init/src/main/packaging/etc/accumulo/auditLog.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+
+
+
+ <!-- Write out Audit info to an Audit file -->
+ <appender name="Audit" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.ip.localhost.hostname}.audit"/>
+ <param name="MaxBackupIndex" value="10"/>
+ <param name="DatePattern" value="'.'yyyy-MM-dd"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss,SSS/Z} [%c{2}] %-5p: %m%n"/>
+ </layout>
+ </appender>
+ <logger name="Audit" additivity="false">
+ <appender-ref ref="Audit" />
+ <level value="OFF"/>
+ </logger>
+
+
+
+
+
+</log4j:configuration>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/fc4de902/init/src/main/packaging/etc/accumulo/gc
----------------------------------------------------------------------
diff --git a/init/src/main/packaging/etc/accumulo/gc b/init/src/main/packaging/etc/accumulo/gc
new file mode 100644
index 0000000..2fbb50c
--- /dev/null
+++ b/init/src/main/packaging/etc/accumulo/gc
@@ -0,0 +1 @@
+localhost
http://git-wip-us.apache.org/repos/asf/accumulo/blob/fc4de902/init/src/main/packaging/etc/accumulo/generic_logger.xml
----------------------------------------------------------------------
diff --git a/init/src/main/packaging/etc/accumulo/generic_logger.xml b/init/src/main/packaging/etc/accumulo/generic_logger.xml
new file mode 100644
index 0000000..db79efe
--- /dev/null
+++ b/init/src/main/packaging/etc/accumulo/generic_logger.xml
@@ -0,0 +1,83 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+
+ <!-- Write out everything at the DEBUG level to the debug log -->
+ <appender name="A2" class="org.apache.log4j.RollingFileAppender">
+ <param name="File" value="${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.application}_${org.apache.accumulo.core.ip.localhost.hostname}.debug.log"/>
+ <param name="MaxFileSize" value="1000MB"/>
+ <param name="MaxBackupIndex" value="10"/>
+ <param name="Threshold" value="DEBUG"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{ISO8601} [%-8c{2}] %-5p: %m%n"/>
+ </layout>
+ </appender>
+
+ <!-- Write out INFO and higher to the regular log -->
+ <appender name="A3" class="org.apache.log4j.RollingFileAppender">
+ <param name="File" value="${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.application}_${org.apache.accumulo.core.ip.localhost.hostname}.log"/>
+ <param name="MaxFileSize" value="1000MB"/>
+ <param name="MaxBackupIndex" value="10"/>
+ <param name="Threshold" value="INFO"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{ISO8601} [%-8c{2}] %-5p: %m%n"/>
+ </layout>
+ </appender>
+
+ <!-- Send all logging data to a centralized logger -->
+ <appender name="N1" class="org.apache.log4j.net.SocketAppender">
+ <param name="remoteHost" value="${org.apache.accumulo.core.host.log}"/>
+ <param name="port" value="${org.apache.accumulo.core.host.log.port}"/>
+ <param name="application" value="${org.apache.accumulo.core.application}:${org.apache.accumulo.core.ip.localhost.hostname}"/>
+ <param name="Threshold" value="WARN"/>
+ </appender>
+
+ <!-- If the centralized logger is down, buffer the log events, but drop them if it stays down -->
+ <appender name="ASYNC" class="org.apache.log4j.AsyncAppender">
+ <appender-ref ref="N1" />
+ </appender>
+
+ <!-- Log accumulo events to the debug, normal and remote logs. -->
+ <logger name="org.apache.accumulo" additivity="false">
+ <level value="DEBUG"/>
+ <appender-ref ref="A2" />
+ <appender-ref ref="A3" />
+ <appender-ref ref="ASYNC" />
+ </logger>
+
+ <logger name="org.apache.accumulo.core.file.rfile.bcfile">
+ <level value="INFO"/>
+ </logger>
+
+ <logger name="org.mortbay.log">
+ <level value="WARN"/>
+ </logger>
+
+ <logger name="org.apache.zookeeper">
+ <level value="ERROR"/>
+ </logger>
+
+ <!-- Log non-accumulo events to the debug and normal logs. -->
+ <root>
+ <level value="INFO"/>
+ <appender-ref ref="A2" />
+ <appender-ref ref="A3" />
+ </root>
+
+</log4j:configuration>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/fc4de902/init/src/main/packaging/etc/accumulo/log4j.properties
----------------------------------------------------------------------
diff --git a/init/src/main/packaging/etc/accumulo/log4j.properties b/init/src/main/packaging/etc/accumulo/log4j.properties
new file mode 100644
index 0000000..a4bcb2e
--- /dev/null
+++ b/init/src/main/packaging/etc/accumulo/log4j.properties
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# default logging properties:
+# by default, log everything at INFO or higher to the console
+log4j.rootLogger=INFO,A1
+
+# hide Jetty junk
+log4j.logger.org.mortbay.log=WARN,A1
+
+# hide "Got brand-new compresssor" messages
+log4j.logger.org.apache.hadoop.io.compress=WARN,A1
+
+# hide junk from TestRandomDeletes
+log4j.logger.org.apache.accumulo.test.TestRandomDeletes=WARN,A1
+
+# hide junk from VFS
+log4j.logger.org.apache.commons.vfs2.impl.DefaultFileSystemManager=WARN,A1
+
+# hide almost everything from zookeeper
+log4j.logger.org.apache.zookeeper=ERROR,A1
+
+# hide AUDIT messages in the shell, alternatively you could send them to a different logger
+log4j.logger.org.apache.accumulo.core.util.shell.Shell.audit=WARN,A1
+
+# Send most things to the console
+log4j.appender.A1=org.apache.log4j.ConsoleAppender
+log4j.appender.A1.layout.ConversionPattern=%d{ISO8601} [%-8c{2}] %-5p: %m%n
+log4j.appender.A1.layout=org.apache.log4j.PatternLayout
http://git-wip-us.apache.org/repos/asf/accumulo/blob/fc4de902/init/src/main/packaging/etc/accumulo/masters
----------------------------------------------------------------------
diff --git a/init/src/main/packaging/etc/accumulo/masters b/init/src/main/packaging/etc/accumulo/masters
new file mode 100644
index 0000000..2fbb50c
--- /dev/null
+++ b/init/src/main/packaging/etc/accumulo/masters
@@ -0,0 +1 @@
+localhost
http://git-wip-us.apache.org/repos/asf/accumulo/blob/fc4de902/init/src/main/packaging/etc/accumulo/monitor_logger.xml
----------------------------------------------------------------------
diff --git a/init/src/main/packaging/etc/accumulo/monitor_logger.xml b/init/src/main/packaging/etc/accumulo/monitor_logger.xml
new file mode 100644
index 0000000..91a7671
--- /dev/null
+++ b/init/src/main/packaging/etc/accumulo/monitor_logger.xml
@@ -0,0 +1,64 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+
+ <!-- Write out everything at the DEBUG level to the debug log -->
+ <appender name="A2" class="org.apache.log4j.RollingFileAppender">
+ <param name="File" value="${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.application}_${org.apache.accumulo.core.ip.localhost.hostname}.debug.log"/>
+ <param name="MaxFileSize" value="100MB"/>
+ <param name="MaxBackupIndex" value="10"/>
+ <param name="Threshold" value="DEBUG"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{ISO8601} [%-8c{2}] %-5p: %X{application} %m%n"/>
+ </layout>
+ </appender>
+
+ <!-- Write out INFO and higher to the regular log -->
+ <appender name="A3" class="org.apache.log4j.RollingFileAppender">
+ <param name="File" value="${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.application}_${org.apache.accumulo.core.ip.localhost.hostname}.log"/>
+ <param name="MaxFileSize" value="100MB"/>
+ <param name="MaxBackupIndex" value="10"/>
+ <param name="Threshold" value="INFO"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{ISO8601} [%-8c{2}] %-5p: %X{application} %m%n"/>
+ </layout>
+ </appender>
+
+ <!-- Keep the last few log messages for display to the user -->
+ <appender name="GUI" class="org.apache.accumulo.server.monitor.LogService">
+ <param name="keep" value="40"/>
+ <param name="Threshold" value="WARN"/>
+ </appender>
+
+ <!-- Log accumulo messages to debug, normal and GUI -->
+ <logger name="org.apache.accumulo" additivity="false">
+ <level value="DEBUG"/>
+ <appender-ref ref="A2" />
+ <appender-ref ref="A3" />
+ <appender-ref ref="GUI" />
+ </logger>
+
+ <!-- Log non-accumulo messages to debug, normal logs. -->
+ <root>
+ <level value="INFO"/>
+ <appender-ref ref="A2" />
+ <appender-ref ref="A3" />
+ </root>
+
+</log4j:configuration>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/fc4de902/init/src/main/packaging/etc/accumulo/monitors
----------------------------------------------------------------------
diff --git a/init/src/main/packaging/etc/accumulo/monitors b/init/src/main/packaging/etc/accumulo/monitors
new file mode 100644
index 0000000..2fbb50c
--- /dev/null
+++ b/init/src/main/packaging/etc/accumulo/monitors
@@ -0,0 +1 @@
+localhost
http://git-wip-us.apache.org/repos/asf/accumulo/blob/fc4de902/init/src/main/packaging/etc/accumulo/slaves
----------------------------------------------------------------------
diff --git a/init/src/main/packaging/etc/accumulo/slaves b/init/src/main/packaging/etc/accumulo/slaves
new file mode 100644
index 0000000..2fbb50c
--- /dev/null
+++ b/init/src/main/packaging/etc/accumulo/slaves
@@ -0,0 +1 @@
+localhost
http://git-wip-us.apache.org/repos/asf/accumulo/blob/fc4de902/init/src/main/packaging/etc/accumulo/tracers
----------------------------------------------------------------------
diff --git a/init/src/main/packaging/etc/accumulo/tracers b/init/src/main/packaging/etc/accumulo/tracers
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/init/src/main/packaging/etc/accumulo/tracers
@@ -0,0 +1 @@
+
http://git-wip-us.apache.org/repos/asf/accumulo/blob/fc4de902/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index dd912ca..eb163a1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -90,6 +90,7 @@
<module>server/native</module>
<module>server/tracer</module>
<module>server/tserver</module>
+ <module>init</module>
</modules>
<scm>
<connection>scm:git:git://git.apache.org/accumulo.git</connection>
@@ -277,6 +278,18 @@
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-init</artifactId>
+ <version>${project.version}</version>
+ <classifier>el6</classifier>
+ <type>rpm</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-init</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-master</artifactId>
<version>${project.version}</version>
<classifier>el6</classifier>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/fc4de902/server/base/pom.xml
----------------------------------------------------------------------
diff --git a/server/base/pom.xml b/server/base/pom.xml
index 3e59fb8..5d31daa 100644
--- a/server/base/pom.xml
+++ b/server/base/pom.xml
@@ -127,7 +127,7 @@
<phase>package</phase>
<configuration>
<requires>
- <require>jre >= 1.6.0</require>
+ <require>java-sdk >= 1.6.0</require>
<require>accumulo-core = %{version}-%{release}</require>
</requires>
<mappings>
@@ -136,6 +136,11 @@
<artifact />
</mapping>
<mapping>
+ <directory>%{_var}/log/accumulo</directory>
+ <username>accumulo</username>
+ <filemode>770</filemode>
+ </mapping>
+ <mapping>
<directory>%{_datadir}/accumulo/lib</directory>
<dependency>
<includes>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/fc4de902/server/gc/src/main/packaging/etc/init.d/accumulo-gc
----------------------------------------------------------------------
diff --git a/server/gc/src/main/packaging/etc/init.d/accumulo-gc b/server/gc/src/main/packaging/etc/init.d/accumulo-gc
index 8ddd524..a6574ae 100755
--- a/server/gc/src/main/packaging/etc/init.d/accumulo-gc
+++ b/server/gc/src/main/packaging/etc/init.d/accumulo-gc
@@ -59,8 +59,7 @@ fi
# Function that starts the daemon/service
#
do_start() {
- TARGET_USER_NAME="ACCUMULO_USER"
- TARGET_USER=$(eval "echo \$$TARGET_USER_NAME")
+ TARGET_USER="${ACCUMULO_USER-accumulo}"
# Return
# 0 if daemon has been started
@@ -116,7 +115,8 @@ case "$1" in
esac
;;
status)
- status_of_proc "$DAEMON" "$NAME" && exit 0 || exit $?
+ echo -n "$NAME is "
+ jps -m | grep -q "$ACCUMULO_PROC" && (echo running.; exit 0) || (echo not running.; exit 1)
;;
#reload|force-reload)
#
http://git-wip-us.apache.org/repos/asf/accumulo/blob/fc4de902/server/master/src/main/packaging/etc/init.d/accumulo-master
----------------------------------------------------------------------
diff --git a/server/master/src/main/packaging/etc/init.d/accumulo-master b/server/master/src/main/packaging/etc/init.d/accumulo-master
index bbc3081..545db58 100755
--- a/server/master/src/main/packaging/etc/init.d/accumulo-master
+++ b/server/master/src/main/packaging/etc/init.d/accumulo-master
@@ -60,8 +60,7 @@ fi
#
do_start()
{
- TARGET_USER_NAME="ACCUMULO_USER"
- TARGET_USER=$(eval "echo \$$TARGET_USER_NAME")
+ TARGET_USER="${ACCUMULO_USER-accumulo}"
# Return
# 0 if daemon has been started
# 1 if daemon was already running
@@ -117,7 +116,8 @@ case "$1" in
esac
;;
status)
- status_of_proc "$DAEMON" "$NAME" && exit 0 || exit $?
+ echo -n "$NAME is "
+ jps -m | grep -q "$ACCUMULO_PROC" && (echo running.; exit 0) || (echo not running.; exit 1)
;;
#reload|force-reload)
#
http://git-wip-us.apache.org/repos/asf/accumulo/blob/fc4de902/server/monitor/src/main/packaging/etc/init.d/accumulo-monitor
----------------------------------------------------------------------
diff --git a/server/monitor/src/main/packaging/etc/init.d/accumulo-monitor b/server/monitor/src/main/packaging/etc/init.d/accumulo-monitor
index 9c45339..147bb4b 100755
--- a/server/monitor/src/main/packaging/etc/init.d/accumulo-monitor
+++ b/server/monitor/src/main/packaging/etc/init.d/accumulo-monitor
@@ -60,8 +60,7 @@ fi
#
do_start()
{
- TARGET_USER_NAME="ACCUMULO_MONITOR_USER"
- TARGET_USER=$(eval "echo \$$TARGET_USER_NAME")
+ TARGET_USER="${ACCUMULO_MONITOR_USER-accumulo}"
# Return
# 0 if daemon has been started
# 1 if daemon was already running
@@ -116,7 +115,8 @@ case "$1" in
esac
;;
status)
- status_of_proc "$DAEMON" "$NAME" && exit 0 || exit $?
+ echo -n "$NAME is "
+ jps -m | grep -q "$ACCUMULO_PROC" && (echo running.; exit 0) || (echo not running.; exit 1)
;;
#reload|force-reload)
#
http://git-wip-us.apache.org/repos/asf/accumulo/blob/fc4de902/server/tracer/src/main/packaging/etc/init.d/accumulo-tracer
----------------------------------------------------------------------
diff --git a/server/tracer/src/main/packaging/etc/init.d/accumulo-tracer b/server/tracer/src/main/packaging/etc/init.d/accumulo-tracer
index dd74391..b454769 100755
--- a/server/tracer/src/main/packaging/etc/init.d/accumulo-tracer
+++ b/server/tracer/src/main/packaging/etc/init.d/accumulo-tracer
@@ -60,8 +60,7 @@ fi
#
do_start()
{
- TARGET_USER_NAME="ACCUMULO_TRACER_USER"
- TARGET_USER=$(eval "echo \$$TARGET_USER_NAME")
+ TARGET_USER="${ACCUMULO_TRACER_USER-accumulo}"
# Return
# 0 if daemon has been started
# 1 if daemon was already running
@@ -116,7 +115,8 @@ case "$1" in
esac
;;
status)
- status_of_proc "$DAEMON" "$NAME" && exit 0 || exit $?
+ echo -n "$NAME is "
+ jps -m | grep -q "$ACCUMULO_PROC" && (echo running.; exit 0) || (echo not running.; exit 1)
;;
#reload|force-reload)
#
http://git-wip-us.apache.org/repos/asf/accumulo/blob/fc4de902/server/tserver/src/main/packaging/etc/init.d/accumulo-tserver
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/packaging/etc/init.d/accumulo-tserver b/server/tserver/src/main/packaging/etc/init.d/accumulo-tserver
index 902368b..2bbd138 100755
--- a/server/tserver/src/main/packaging/etc/init.d/accumulo-tserver
+++ b/server/tserver/src/main/packaging/etc/init.d/accumulo-tserver
@@ -60,8 +60,7 @@ fi
#
do_start()
{
- TARGET_USER_NAME="ACCUMULO_USER"
- TARGET_USER=$(eval "echo \$$TARGET_USER_NAME")
+ TARGET_USER="${ACCUMULO_USER-accumulo}"
# Return
# 0 if daemon has been started
# 1 if daemon was already running
@@ -83,8 +82,7 @@ do_start()
#
do_stop()
{
- TARGET_USER_NAME="ACCUMULO_USER"
- TARGET_USER=$(eval "echo \$$TARGET_USER_NAME")
+ TARGET_USER="${ACCUMULO_USER-accumulo}"
# Return
# 0 if daemon has been stopped
# 1 if daemon was already stopped
@@ -119,7 +117,8 @@ case "$1" in
esac
;;
status)
- status_of_proc "$DAEMON" "$NAME" && exit 0 || exit $?
+ echo -n "$NAME is "
+ jps -m | grep -q "$ACCUMULO_PROC" && (echo running.; exit 0) || (echo not running.; exit 1)
;;
#reload|force-reload)
#
[14/14] git commit: Merge remote-tracking branch
'origin/1.6.0-SNAPSHOT' into ACCUMULO-2061
Posted by el...@apache.org.
Merge remote-tracking branch 'origin/1.6.0-SNAPSHOT' into ACCUMULO-2061
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/db76dd75
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/db76dd75
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/db76dd75
Branch: refs/heads/ACCUMULO-2061
Commit: db76dd75161c9a6ad94a6a4cc5e1f5656d80e21c
Parents: 85b5caf 0930732
Author: Josh Elser <el...@apache.org>
Authored: Wed Mar 12 01:06:44 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Wed Mar 12 01:06:44 2014 -0400
----------------------------------------------------------------------
assemble/pom.xml | 7 ++
bin/accumulo | 5 +
core/pom.xml | 2 +-
init/pom.xml | 98 ++++++++++++++++++++
.../main/packaging/etc/accumulo/accumulo-env.sh | 65 +++++++++++++
.../packaging/etc/accumulo/accumulo-site.xml | 87 +++++++++++++++++
.../main/packaging/etc/accumulo/auditLog.xml | 41 ++++++++
init/src/main/packaging/etc/accumulo/gc | 16 ++++
.../packaging/etc/accumulo/generic_logger.xml | 83 +++++++++++++++++
.../packaging/etc/accumulo/log4j.properties | 41 ++++++++
init/src/main/packaging/etc/accumulo/masters | 16 ++++
.../packaging/etc/accumulo/monitor_logger.xml | 64 +++++++++++++
init/src/main/packaging/etc/accumulo/monitors | 16 ++++
init/src/main/packaging/etc/accumulo/slaves | 16 ++++
init/src/main/packaging/etc/accumulo/tracers | 15 +++
pom.xml | 13 +++
.../apache/accumulo/proxy/SimpleProxyIT.java | 13 ++-
server/base/pom.xml | 7 +-
.../src/main/packaging/etc/init.d/accumulo-gc | 6 +-
.../main/packaging/etc/init.d/accumulo-master | 6 +-
.../main/packaging/etc/init.d/accumulo-monitor | 6 +-
.../main/packaging/etc/init.d/accumulo-tracer | 6 +-
.../main/packaging/etc/init.d/accumulo-tserver | 9 +-
.../org/apache/accumulo/test/DeleteRowsIT.java | 2 +-
.../org/apache/accumulo/test/NamespacesIT.java | 8 +-
.../org/apache/accumulo/test/ShellServerIT.java | 6 +-
.../accumulo/test/functional/BloomFilterIT.java | 30 +++---
27 files changed, 640 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
[10/14] git commit: ACCUMULO-2061 Add another test which doesn't
start with /accumulo
Posted by el...@apache.org.
ACCUMULO-2061 Add another test which doesn't start with /accumulo
Make sure that the garbage collector is expecting anything special like
/accumulo in the path.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/bf5da442
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/bf5da442
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/bf5da442
Branch: refs/heads/ACCUMULO-2061
Commit: bf5da4423bd8f54491e5b67176cd639ff8273903
Parents: 243aafa
Author: Josh Elser <el...@apache.org>
Authored: Tue Mar 11 15:13:37 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Tue Mar 11 15:13:37 2014 -0400
----------------------------------------------------------------------
.../accumulo/gc/GarbageCollectionTest.java | 65 ++++++++++++++++++++
1 file changed, 65 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/bf5da442/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java
----------------------------------------------------------------------
diff --git a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java
index 4e2a878..1e9c1dd 100644
--- a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java
+++ b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java
@@ -335,6 +335,71 @@ public class GarbageCollectionTest {
assertRemoved(gce);
}
+
+ @Test
+ public void testCustomDirectories() throws Exception {
+ TestGCE gce = new TestGCE();
+
+ gce.candidates.add("/4/t-0");
+ gce.candidates.add("/4/t-0/F002.rf");
+ gce.candidates.add("hdfs://foo.com:6000/user/foo/tables/5/t-0");
+ gce.candidates.add("/6/t-0");
+ gce.candidates.add("hdfs://foo:6000/user/foo/tables/7/t-0/");
+ gce.candidates.add("/8/t-0");
+ gce.candidates.add("hdfs://foo:6000/user/foo/tables/9/t-0");
+ gce.candidates.add("/a/t-0");
+ gce.candidates.add("hdfs://foo:6000/user/foo/tables/b/t-0");
+ gce.candidates.add("/c/t-0");
+ gce.candidates.add("hdfs://foo:6000/user/foo/tables/d/t-0");
+
+ gce.addDirReference("4", null, "/t-0");
+ gce.addDirReference("5", null, "/t-0");
+ gce.addDirReference("6", null, "hdfs://foo.com:6000/user/foo/tables/6/t-0");
+ gce.addDirReference("7", null, "hdfs://foo.com:6000/user/foo/tables/7/t-0");
+
+ gce.addFileReference("8", "m", "/t-0/F00.rf");
+ gce.addFileReference("9", "m", "/t-0/F00.rf");
+
+ gce.addFileReference("a", "m", "hdfs://foo.com:6000/user/foo/tables/a/t-0/F00.rf");
+ gce.addFileReference("b", "m", "hdfs://foo.com:6000/user/foo/tables/b/t-0/F00.rf");
+
+ gce.addFileReference("e", "m", "../c/t-0/F00.rf");
+ gce.addFileReference("f", "m", "../d/t-0/F00.rf");
+
+ GarbageCollectionAlgorithm gca = new GarbageCollectionAlgorithm();
+
+ // A directory reference does not preclude a candidate file beneath that directory from deletion
+ gca.collect(gce);
+ assertRemoved(gce, "/4/t-0/F002.rf");
+
+ // Removing the dir reference for a table will delete all tablet directories
+ gce.removeDirReference("5", null);
+ gca.collect(gce);
+ assertRemoved(gce, "hdfs://foo.com:6000/user/foo/tables/5/t-0");
+
+ gce.removeDirReference("4", null);
+ gca.collect(gce);
+ assertRemoved(gce, "/4/t-0");
+
+ gce.removeDirReference("6", null);
+ gce.removeDirReference("7", null);
+ gca.collect(gce);
+ assertRemoved(gce, "/6/t-0", "hdfs://foo:6000/user/foo/tables/7/t-0/");
+
+ gce.removeFileReference("8", "m", "/t-0/F00.rf");
+ gce.removeFileReference("9", "m", "/t-0/F00.rf");
+ gce.removeFileReference("a", "m", "hdfs://foo.com:6000/user/foo/tables/a/t-0/F00.rf");
+ gce.removeFileReference("b", "m", "hdfs://foo.com:6000/user/foo/tables/b/t-0/F00.rf");
+ gce.removeFileReference("e", "m", "../c/t-0/F00.rf");
+ gce.removeFileReference("f", "m", "../d/t-0/F00.rf");
+ gca.collect(gce);
+ assertRemoved(gce, "/8/t-0", "hdfs://foo:6000/user/foo/tables/9/t-0", "/a/t-0", "hdfs://foo:6000/user/foo/tables/b/t-0", "/c/t-0",
+ "hdfs://foo:6000/user/foo/tables/d/t-0");
+
+ gca.collect(gce);
+ assertRemoved(gce);
+ }
+
private void badRefTest(String ref) {
TestGCE gce = new TestGCE();
[02/14] git commit: Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by el...@apache.org.
Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Conflicts:
test/src/test/java/org/apache/accumulo/test/DeleteRowsTest.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/fb17d458
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/fb17d458
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/fb17d458
Branch: refs/heads/ACCUMULO-2061
Commit: fb17d4588d597e1124a47a4dd29dfc728bd1dc49
Parents: da7f937 6c05091
Author: Josh Elser <el...@apache.org>
Authored: Tue Mar 11 00:26:39 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Tue Mar 11 00:26:39 2014 -0400
----------------------------------------------------------------------
test/src/test/java/org/apache/accumulo/test/DeleteRowsIT.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/fb17d458/test/src/test/java/org/apache/accumulo/test/DeleteRowsIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/DeleteRowsIT.java
index 190f693,0000000..c116e30
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/DeleteRowsIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/DeleteRowsIT.java
@@@ -1,42 -1,0 +1,42 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.test.functional.FunctionalTestUtils;
+import org.apache.accumulo.test.functional.SimpleMacIT;
+import org.junit.Test;
+
+public class DeleteRowsIT extends SimpleMacIT {
+
- @Test(timeout = 60 * 1000)
++ @Test(timeout = 5 * 60 * 1000)
+ public void test() throws Exception {
+ Connector c = getConnector();
+ String[] tableNames = this.getTableNames(20);
+ for (String tableName : tableNames) {
+ c.tableOperations().create(tableName);
+ c.tableOperations().deleteRows(tableName, null, null);
+ Scanner scanner = c.createScanner(tableName, Authorizations.EMPTY);
+ assertEquals(0, FunctionalTestUtils.count(scanner));
+ }
+ }
+
+}
[11/14] git commit: ACCUMULO-2267 Add missing license headers.
Posted by el...@apache.org.
ACCUMULO-2267 Add missing license headers.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/46bf772d
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/46bf772d
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/46bf772d
Branch: refs/heads/ACCUMULO-2061
Commit: 46bf772d6b49022afd58e3b16580d7ee12773cc5
Parents: 3ed0bbd
Author: Josh Elser <el...@apache.org>
Authored: Tue Mar 11 15:48:04 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Tue Mar 11 15:48:04 2014 -0400
----------------------------------------------------------------------
init/src/main/packaging/etc/accumulo/gc | 15 +++++++++++++++
init/src/main/packaging/etc/accumulo/masters | 15 +++++++++++++++
init/src/main/packaging/etc/accumulo/monitors | 15 +++++++++++++++
init/src/main/packaging/etc/accumulo/slaves | 15 +++++++++++++++
init/src/main/packaging/etc/accumulo/tracers | 14 ++++++++++++++
5 files changed, 74 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/46bf772d/init/src/main/packaging/etc/accumulo/gc
----------------------------------------------------------------------
diff --git a/init/src/main/packaging/etc/accumulo/gc b/init/src/main/packaging/etc/accumulo/gc
index 2fbb50c..63fb8bb 100644
--- a/init/src/main/packaging/etc/accumulo/gc
+++ b/init/src/main/packaging/etc/accumulo/gc
@@ -1 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
localhost
http://git-wip-us.apache.org/repos/asf/accumulo/blob/46bf772d/init/src/main/packaging/etc/accumulo/masters
----------------------------------------------------------------------
diff --git a/init/src/main/packaging/etc/accumulo/masters b/init/src/main/packaging/etc/accumulo/masters
index 2fbb50c..63fb8bb 100644
--- a/init/src/main/packaging/etc/accumulo/masters
+++ b/init/src/main/packaging/etc/accumulo/masters
@@ -1 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
localhost
http://git-wip-us.apache.org/repos/asf/accumulo/blob/46bf772d/init/src/main/packaging/etc/accumulo/monitors
----------------------------------------------------------------------
diff --git a/init/src/main/packaging/etc/accumulo/monitors b/init/src/main/packaging/etc/accumulo/monitors
index 2fbb50c..63fb8bb 100644
--- a/init/src/main/packaging/etc/accumulo/monitors
+++ b/init/src/main/packaging/etc/accumulo/monitors
@@ -1 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
localhost
http://git-wip-us.apache.org/repos/asf/accumulo/blob/46bf772d/init/src/main/packaging/etc/accumulo/slaves
----------------------------------------------------------------------
diff --git a/init/src/main/packaging/etc/accumulo/slaves b/init/src/main/packaging/etc/accumulo/slaves
index 2fbb50c..63fb8bb 100644
--- a/init/src/main/packaging/etc/accumulo/slaves
+++ b/init/src/main/packaging/etc/accumulo/slaves
@@ -1 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
localhost
http://git-wip-us.apache.org/repos/asf/accumulo/blob/46bf772d/init/src/main/packaging/etc/accumulo/tracers
----------------------------------------------------------------------
diff --git a/init/src/main/packaging/etc/accumulo/tracers b/init/src/main/packaging/etc/accumulo/tracers
index 8b13789..09697dc 100644
--- a/init/src/main/packaging/etc/accumulo/tracers
+++ b/init/src/main/packaging/etc/accumulo/tracers
@@ -1 +1,15 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
[05/14] git commit: Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by el...@apache.org.
Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/6b2704c7
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/6b2704c7
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/6b2704c7
Branch: refs/heads/ACCUMULO-2061
Commit: 6b2704c7ecb90268e0ccd89a6cb42cf5258a82f7
Parents: fb17d45 19231a2
Author: Bill Havanki <bh...@cloudera.com>
Authored: Tue Mar 11 12:19:29 2014 -0400
Committer: Bill Havanki <bh...@cloudera.com>
Committed: Tue Mar 11 12:19:29 2014 -0400
----------------------------------------------------------------------
----------------------------------------------------------------------
[03/14] git commit: ACCUMULO-2450 Backport of ACCUMULO-2438 to 1.4:
ensure that we watch separate sets of tables for merges (cherry picked from
commit 12b53b043b60368fb647db9c0a15916171d4504b)
Posted by el...@apache.org.
ACCUMULO-2450 Backport of ACCUMULO-2438 to 1.4: ensure that we watch separate sets of tables for merges
(cherry picked from commit 12b53b043b60368fb647db9c0a15916171d4504b)
Original authors: Eric Newton (1.6+), Josh Elser (1.5 backport)
This backport does not include the new DeleteRowsTest, as no location for it is obvious
under the 1.4 development branch.
Conflicts:
src/server/src/main/java/org/apache/accumulo/server/master/Master.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/117dce76
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/117dce76
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/117dce76
Branch: refs/heads/ACCUMULO-2061
Commit: 117dce76568bbe6355eae2ca058bdc3f81e53578
Parents: 6faac42
Author: Josh Elser <el...@apache.org>
Authored: Sun Mar 9 23:34:18 2014 -0400
Committer: Bill Havanki <bh...@cloudera.com>
Committed: Tue Mar 11 11:54:47 2014 -0400
----------------------------------------------------------------------
.../main/java/org/apache/accumulo/server/master/Master.java | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/117dce76/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/master/Master.java b/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
index aa8305b..fb7be51 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
@@ -1293,9 +1293,10 @@ public class Master implements LiveTServerSet.Listener, LoggerWatcher, TableObse
int unloaded = 0;
try {
Map<Text,MergeStats> mergeStatsCache = new HashMap<Text,MergeStats>();
+ Map<Text,MergeStats> currentMerges = new HashMap<Text,MergeStats>();
for (MergeInfo merge : merges()) {
if (merge.getRange() != null) {
- mergeStatsCache.put(merge.getRange().getTableId(), new MergeStats(merge));
+ currentMerges.put(merge.getRange().getTableId(), new MergeStats(merge));
}
}
@@ -1341,7 +1342,11 @@ public class Master implements LiveTServerSet.Listener, LoggerWatcher, TableObse
Text tableId = tls.extent.getTableId();
MergeStats mergeStats = mergeStatsCache.get(tableId);
if (mergeStats == null) {
- mergeStatsCache.put(tableId, mergeStats = new MergeStats(getMergeInfo(tableId)));
+ mergeStats = currentMerges.get(tableId);
+ if (mergeStats == null) {
+ mergeStats = new MergeStats(getMergeInfo(tableId));
+ }
+ mergeStatsCache.put(tableId, mergeStats);
}
TabletGoalState goal = getGoalState(tls, mergeStats.getMergeInfo());
TServerInstance server = tls.getServer();
[12/14] git commit: ACCUMULO-2455 small work arounds to increase the
chance that zookeeper updates are in place before tservers need to see them
Posted by el...@apache.org.
ACCUMULO-2455 small work arounds to increase the chance that zookeeper updates are in place before tservers need to see them
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/09307322
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/09307322
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/09307322
Branch: refs/heads/ACCUMULO-2061
Commit: 09307322a3e9f45cd37bbd4c63fa66ccbe2cc0a2
Parents: 46bf772
Author: Eric Newton <er...@gmail.com>
Authored: Tue Mar 11 16:30:45 2014 -0400
Committer: Eric Newton <er...@gmail.com>
Committed: Tue Mar 11 16:30:45 2014 -0400
----------------------------------------------------------------------
.../apache/accumulo/proxy/SimpleProxyIT.java | 13 ++++-----
.../org/apache/accumulo/test/NamespacesIT.java | 8 +++---
.../accumulo/test/functional/BloomFilterIT.java | 30 ++++++++++++--------
3 files changed, 28 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/09307322/proxy/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java
----------------------------------------------------------------------
diff --git a/proxy/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java b/proxy/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java
index efe36b9..f554d63 100644
--- a/proxy/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java
+++ b/proxy/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java
@@ -772,15 +772,14 @@ public class SimpleProxyIT {
// create a table that's very slow, so we can look for scans/compactions
client.createTable(creds, "slow", true, TimeType.MILLIS);
+ IteratorSetting setting = new IteratorSetting(100, "slow", SlowIterator.class.getName(), Collections.singletonMap("sleepTime", "250"));
+ client.attachIterator(creds, "slow", setting, EnumSet.allOf(IteratorScope.class));
- // Should take 5 seconds to read every record
- for (int i = 0; i < 20; i++) {
+ // Should take 10 seconds to read every record
+ for (int i = 0; i < 40; i++) {
client.updateAndFlush(creds, "slow", mutation("row" + i, "cf", "cq", "value"));
}
- IteratorSetting setting = new IteratorSetting(100, "slow", SlowIterator.class.getName(), Collections.singletonMap("sleepTime", "250"));
- client.attachIterator(creds, "slow", setting, EnumSet.allOf(IteratorScope.class));
-
// scan
Thread t = new Thread() {
@Override
@@ -798,7 +797,7 @@ public class SimpleProxyIT {
};
t.start();
- // look for the scan
+ // look for the scan many times
List<ActiveScan> scans = new ArrayList<ActiveScan>();
for (int i = 0; i < 100 && scans.isEmpty(); i++) {
for (String tserver : client.getTabletServers(creds)) {
@@ -811,7 +810,7 @@ public class SimpleProxyIT {
if (!scans.isEmpty())
break;
- UtilWaitThread.sleep(10);
+ UtilWaitThread.sleep(100);
}
}
t.join();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/09307322/test/src/test/java/org/apache/accumulo/test/NamespacesIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/NamespacesIT.java b/test/src/test/java/org/apache/accumulo/test/NamespacesIT.java
index a8d5f7f..af9274b 100644
--- a/test/src/test/java/org/apache/accumulo/test/NamespacesIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/NamespacesIT.java
@@ -321,17 +321,17 @@ public class NamespacesIT extends SimpleMacIT {
}
IteratorSetting setting2 = c.namespaceOperations().getIteratorSetting(namespace, setting.getName(), IteratorScope.scan);
assertEquals(setting, setting2);
- s = c.createScanner(t1, Authorizations.EMPTY);
- assertFalse(s.iterator().hasNext());
assertTrue(c.namespaceOperations().listIterators(namespace).containsKey(iterName));
assertTrue(c.tableOperations().listIterators(t1).containsKey(iterName));
+ s = c.createScanner(t1, Authorizations.EMPTY);
+ assertFalse(s.iterator().hasNext());
// verify can see inserted entry again
c.namespaceOperations().removeIterator(namespace, setting.getName(), EnumSet.allOf(IteratorScope.class));
- s = c.createScanner(t1, Authorizations.EMPTY);
- assertTrue(s.iterator().hasNext());
assertFalse(c.namespaceOperations().listIterators(namespace).containsKey(iterName));
assertFalse(c.tableOperations().listIterators(t1).containsKey(iterName));
+ s = c.createScanner(t1, Authorizations.EMPTY);
+ assertTrue(s.iterator().hasNext());
}
@Test
http://git-wip-us.apache.org/repos/asf/accumulo/blob/09307322/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java b/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java
index 27aab91..50ca776 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java
@@ -39,6 +39,7 @@ import org.apache.accumulo.core.file.keyfunctor.ColumnFamilyFunctor;
import org.apache.accumulo.core.file.keyfunctor.ColumnQualifierFunctor;
import org.apache.accumulo.core.file.keyfunctor.RowFunctor;
import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.fate.util.UtilWaitThread;
import org.apache.accumulo.minicluster.MemoryUnit;
import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
import org.apache.hadoop.conf.Configuration;
@@ -56,6 +57,7 @@ public class BloomFilterIT extends ConfigurableMacIT {
siteConfig.put(Property.TABLE_BLOOM_SIZE.getKey(), "2000000");
siteConfig.put(Property.TABLE_BLOOM_ERRORRATE.getKey(), "1%");
siteConfig.put(Property.TABLE_BLOOM_LOAD_THRESHOLD.getKey(), "0");
+ siteConfig.put(Property.TSERV_MUTATION_QUEUE_MAX.getKey(), "10M");
siteConfig.put(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(), "1G");
cfg.setSiteConfig(siteConfig );
}
@@ -69,9 +71,9 @@ public class BloomFilterIT extends ConfigurableMacIT {
c.tableOperations().setProperty(table, Property.TABLE_BLOCKCACHE_ENABLED.getKey(), "false");
}
log.info("Writing");
- write(c, "bt1", 1, 0, 2000000000, 1000);
- write(c, "bt2", 2, 0, 2000000000, 1000);
- write(c, "bt3", 3, 0, 2000000000, 1000);
+ write(c, "bt1", 1, 0, 2000000000, 500);
+ write(c, "bt2", 2, 0, 2000000000, 500);
+ write(c, "bt3", 3, 0, 2000000000, 500);
log.info("Writing complete");
// test inserting an empty key
@@ -94,35 +96,39 @@ public class BloomFilterIT extends ConfigurableMacIT {
// these queries should only run quickly if bloom filters are working, so lets get a base
log.info("Base query");
- long t1 = query(c, "bt1", 1, 0, 2000000000, 100000, 1000);
- long t2 = query(c, "bt2", 2, 0, 2000000000, 100000, 1000);
- long t3 = query(c, "bt3", 3, 0, 2000000000, 100000, 1000);
+ long t1 = query(c, "bt1", 1, 0, 2000000000, 100000, 500);
+ long t2 = query(c, "bt2", 2, 0, 2000000000, 100000, 500);
+ long t3 = query(c, "bt3", 3, 0, 2000000000, 100000, 500);
log.info("Base query complete");
log.info("Rewriting with bloom filters");
c.tableOperations().setProperty("bt1", Property.TABLE_BLOOM_ENABLED.getKey(), "true");
c.tableOperations().setProperty("bt1", Property.TABLE_BLOOM_KEY_FUNCTOR.getKey(), RowFunctor.class.getName());
- c.tableOperations().compact("bt1", null, null, false, true);
c.tableOperations().setProperty("bt2", Property.TABLE_BLOOM_ENABLED.getKey(), "true");
c.tableOperations().setProperty("bt2", Property.TABLE_BLOOM_KEY_FUNCTOR.getKey(), ColumnFamilyFunctor.class.getName());
- c.tableOperations().compact("bt2", null, null, false, true);
c.tableOperations().setProperty("bt3", Property.TABLE_BLOOM_ENABLED.getKey(), "true");
c.tableOperations().setProperty("bt3", Property.TABLE_BLOOM_KEY_FUNCTOR.getKey(), ColumnQualifierFunctor.class.getName());
- c.tableOperations().compact("bt3", null, null, false, true);
c.tableOperations().setProperty("bt4", Property.TABLE_BLOOM_ENABLED.getKey(), "true");
c.tableOperations().setProperty("bt4", Property.TABLE_BLOOM_KEY_FUNCTOR.getKey(), RowFunctor.class.getName());
+
+ // ensure the updates to zookeeper propogate
+ UtilWaitThread.sleep(500);
+
c.tableOperations().compact("bt4", null, null, false, true);
+ c.tableOperations().compact("bt1", null, null, false, true);
+ c.tableOperations().compact("bt2", null, null, false, true);
+ c.tableOperations().compact("bt3", null, null, false, true);
log.info("Rewriting with bloom filters complete");
// these queries should only run quickly if bloom
// filters are working
log.info("Bloom query");
- long tb1 = query(c, "bt1", 1, 0, 2000000000, 100000, 1000);
- long tb2 = query(c, "bt2", 2, 0, 2000000000, 100000, 1000);
- long tb3 = query(c, "bt3", 3, 0, 2000000000, 100000, 1000);
+ long tb1 = query(c, "bt1", 1, 0, 2000000000, 100000, 500);
+ long tb2 = query(c, "bt2", 2, 0, 2000000000, 100000, 500);
+ long tb3 = query(c, "bt3", 3, 0, 2000000000, 100000, 500);
log.info("Bloom query complete");
timeCheck(t1 + t2 + t3, tb1 + tb2 + tb3);
[04/14] git commit: Merge branch '1.4.5-SNAPSHOT' into 1.5.2-SNAPSHOT
(-sours)
Posted by el...@apache.org.
Merge branch '1.4.5-SNAPSHOT' into 1.5.2-SNAPSHOT (-sours)
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/19231a25
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/19231a25
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/19231a25
Branch: refs/heads/ACCUMULO-2061
Commit: 19231a25c0f27862e4320a60a7dc1158eb907be8
Parents: 6c05091 117dce7
Author: Bill Havanki <bh...@cloudera.com>
Authored: Tue Mar 11 12:19:10 2014 -0400
Committer: Bill Havanki <bh...@cloudera.com>
Committed: Tue Mar 11 12:19:10 2014 -0400
----------------------------------------------------------------------
----------------------------------------------------------------------
[06/14] git commit: ACCUMULO-2433 It would help to actually pass the
error message to the assertion
Posted by el...@apache.org.
ACCUMULO-2433 It would help to actually pass the error message to the assertion
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/43cebf8c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/43cebf8c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/43cebf8c
Branch: refs/heads/ACCUMULO-2061
Commit: 43cebf8c0389ce178fc02c7a541671d335e8e45e
Parents: 19231a2
Author: Josh Elser <el...@apache.org>
Authored: Tue Mar 11 14:50:11 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Tue Mar 11 14:50:11 2014 -0400
----------------------------------------------------------------------
.../test/java/org/apache/accumulo/test/ShellServerTest.java | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/43cebf8c/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java b/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
index d66f509..caa3dd1 100644
--- a/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
+++ b/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
@@ -158,7 +158,7 @@ public class ShellServerTest {
Shell.log.debug(output.get());
if (0 != shell.getExitCode()) {
String errorMsg = callback.getErrorMessage();
- assertEquals(0, shell.getExitCode());
+ assertEquals(errorMsg, 0, shell.getExitCode());
}
if (s.length() > 0)
@@ -167,7 +167,11 @@ public class ShellServerTest {
void assertBadExit(String s, boolean stringPresent, ErrorMessageCallback callback) {
Shell.log.debug(output.get());
- assertTrue(shell.getExitCode() > 0);
+ if (0 == shell.getExitCode()) {
+ String errorMsg = callback.getErrorMessage();
+ assertTrue(errorMsg, shell.getExitCode() > 0);
+ }
+
if (s.length() > 0)
assertEquals(s + " present in " + output.get() + " was not " + stringPresent, stringPresent, output.get().contains(s));
shell.resetExitCode();
[07/14] git commit: Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by el...@apache.org.
Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/97607425
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/97607425
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/97607425
Branch: refs/heads/ACCUMULO-2061
Commit: 976074254b0013f9daa221934ab965c8f0db9708
Parents: 6b2704c 43cebf8
Author: Josh Elser <el...@apache.org>
Authored: Tue Mar 11 14:50:54 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Tue Mar 11 14:50:54 2014 -0400
----------------------------------------------------------------------
test/src/test/java/org/apache/accumulo/test/ShellServerIT.java | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/97607425/test/src/test/java/org/apache/accumulo/test/ShellServerIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/ShellServerIT.java
index e0ad1bd,0000000..92bae48
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/ShellServerIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/ShellServerIT.java
@@@ -1,1238 -1,0 +1,1242 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import jline.console.ConsoleReader;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.client.impl.Namespaces;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.core.file.FileSKVWriter;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.util.shell.Shell;
+import org.apache.accumulo.test.functional.FunctionalTestUtils;
+import org.apache.accumulo.test.functional.SimpleMacIT;
+import org.apache.accumulo.tracer.TraceServer;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.tools.DistCp;
+import org.apache.log4j.Logger;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+
+public class ShellServerIT extends SimpleMacIT {
+ public static class TestOutputStream extends OutputStream {
+ StringBuilder sb = new StringBuilder();
+
+ @Override
+ public void write(int b) throws IOException {
+ sb.append((char) (0xff & b));
+ }
+
+ public String get() {
+ return sb.toString();
+ }
+
+ public void clear() {
+ sb.setLength(0);
+ }
+ }
+
+ private static final Logger log = Logger.getLogger(ShellServerIT.class);
+
+ public static class StringInputStream extends InputStream {
+ private String source = "";
+ private int offset = 0;
+
+ @Override
+ public int read() throws IOException {
+ if (offset == source.length())
+ return '\n';
+ else
+ return source.charAt(offset++);
+ }
+
+ public void set(String other) {
+ source = other;
+ offset = 0;
+ }
+ }
+
+ private static abstract class ErrorMessageCallback {
+ public abstract String getErrorMessage();
+ }
+
+ private static class NoOpErrorMessageCallback extends ErrorMessageCallback {
+ private static final String empty = "";
+ public String getErrorMessage() {
+ return empty;
+ }
+ }
+
+ private static final NoOpErrorMessageCallback noop = new NoOpErrorMessageCallback();
+
+ public TestOutputStream output;
+ public StringInputStream input;
+ public Shell shell;
+ private static Process traceProcess;
+
+ @Rule
+ public TestName name = new TestName();
+
+ String exec(String cmd) throws IOException {
+ output.clear();
+ shell.execCommand(cmd, true, true);
+ return output.get();
+ }
+
+ String exec(String cmd, boolean expectGoodExit) throws IOException {
+ return exec(cmd, expectGoodExit, noop);
+ }
+
+ String exec(String cmd, boolean expectGoodExit, ErrorMessageCallback callback) throws IOException {
+ String result = exec(cmd);
+ if (expectGoodExit)
+ assertGoodExit("", true, callback);
+ else
+ assertBadExit("", true, callback);
+ return result;
+ }
+
+ String exec(String cmd, boolean expectGoodExit, String expectString) throws IOException {
+ return exec(cmd, expectGoodExit, expectString, noop);
+ }
+
+ String exec(String cmd, boolean expectGoodExit, String expectString, ErrorMessageCallback callback) throws IOException {
+ return exec(cmd, expectGoodExit, expectString, true, callback);
+ }
+
+ String exec(String cmd, boolean expectGoodExit, String expectString, boolean stringPresent) throws IOException {
+ return exec(cmd, expectGoodExit, expectString, stringPresent, noop);
+ }
+
+ String exec(String cmd, boolean expectGoodExit, String expectString, boolean stringPresent, ErrorMessageCallback callback) throws IOException {
+ String result = exec(cmd);
+ if (expectGoodExit)
+ assertGoodExit(expectString, stringPresent, callback);
+ else
+ assertBadExit(expectString, stringPresent, callback);
+ return result;
+ }
+
+ void assertGoodExit(String s, boolean stringPresent) {
+ assertGoodExit(s, stringPresent, noop);
+ }
+
+ void assertGoodExit(String s, boolean stringPresent, ErrorMessageCallback callback) {
+ Shell.log.info(output.get());
+ if (0 != shell.getExitCode()) {
+ String errorMsg = callback.getErrorMessage();
+ assertEquals(errorMsg, 0, shell.getExitCode());
+ }
+
+ if (s.length() > 0)
+ assertEquals(s + " present in " + output.get() + " was not " + stringPresent, stringPresent, output.get().contains(s));
+ }
+
+ void assertBadExit(String s, boolean stringPresent, ErrorMessageCallback callback) {
+ Shell.log.debug(output.get());
- assertTrue(shell.getExitCode() > 0);
++ if (0 == shell.getExitCode()) {
++ String errorMsg = callback.getErrorMessage();
++ assertTrue(errorMsg, shell.getExitCode() > 0);
++ }
++
+ if (s.length() > 0)
+ assertEquals(s + " present in " + output.get() + " was not " + stringPresent, stringPresent, output.get().contains(s));
+ shell.resetExitCode();
+ }
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ // history file is updated in $HOME
+ System.setProperty("HOME", getFolder().getAbsolutePath());
+ System.setProperty("hadoop.tmp.dir", System.getProperty("user.dir") + "/target/hadoop-tmp");
+
+ // use reflection to call this method so it does not need to be made public
+ Method method = getStaticCluster().getClass().getDeclaredMethod("exec", Class.class, String[].class);
+ method.setAccessible(true);
+ traceProcess = (Process) method.invoke(getStaticCluster(), TraceServer.class, new String[0]);
+
+ // give the tracer some time to start
+ UtilWaitThread.sleep(1000);
+ }
+
+ @Before
+ public void setupShell() throws Exception {
+ // start the shell
+ output = new TestOutputStream();
+ input = new StringInputStream();
+ PrintWriter pw = new PrintWriter(new OutputStreamWriter(output));
+ shell = new Shell(new ConsoleReader(input, output), pw);
+ shell.setLogErrorsToConsole();
+ shell.config("-u", "root", "-p", ROOT_PASSWORD, "-z", getStaticCluster().getConfig().getInstanceName(), getStaticCluster().getConfig().getZooKeepers(),
+ "--config-file", getStaticCluster().getConfig().getClientConfFile().getAbsolutePath());
+ exec("quit", true);
+ shell.start();
+ shell.setExit(false);
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ traceProcess.destroy();
+ }
+
+ @After
+ public void deleteTables() throws Exception {
+ Connector c = getConnector();
+ for (String table : c.tableOperations().list()) {
+ if (!table.equals(MetadataTable.NAME) && !table.equals(RootTable.NAME) && !table.equals("trace"))
+ try {
+ c.tableOperations().delete(table);
+ } catch (TableNotFoundException e) {
+ // don't care
+ }
+ }
+ }
+
+ @After
+ public void tearDownShell() {
+ shell.shutdown();
+ }
+
+ @Test(timeout = 60000)
+ public void exporttableImporttable() throws Exception {
+ final String table = name.getMethodName(), table2 = table + "2";
+
+ // exporttable / importtable
+ exec("createtable " + table + " -evc", true);
+ make10();
+ exec("addsplits row5", true);
+ exec("config -t " + table + " -s table.split.threshold=345M", true);
+ exec("offline " + table, true);
+ String export = "file://" + new File(getFolder(), "ShellServerIT.export").toString();
+ exec("exporttable -t " + table + " " + export, true);
+ DistCp cp = newDistCp();
+ String import_ = "file://" + new File(getFolder(), "ShellServerIT.import").toString();
+ cp.run(new String[] {"-f", export + "/distcp.txt", import_});
+ exec("importtable " + table2 + " " + import_, true);
+ exec("config -t " + table2 + " -np", true, "345M", true);
+ exec("getsplits -t " + table2, true, "row5", true);
+ exec("constraint --list -t " + table2, true, "VisibilityConstraint=2", true);
+ exec("onlinetable " + table, true);
+ exec("deletetable -f " + table, true);
+ exec("deletetable -f " + table2, true);
+ }
+
+ private DistCp newDistCp() {
+ try {
+ @SuppressWarnings("unchecked")
+ Constructor<DistCp>[] constructors = (Constructor<DistCp>[]) DistCp.class.getConstructors();
+ for (Constructor<DistCp> constructor : constructors) {
+ Class<?>[] parameterTypes = constructor.getParameterTypes();
+ if (parameterTypes.length > 0 && parameterTypes[0].equals(Configuration.class)) {
+ if (parameterTypes.length == 1) {
+ return constructor.newInstance(new Configuration());
+ } else if (parameterTypes.length == 2) {
+ return constructor.newInstance(new Configuration(), null);
+ }
+ }
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ throw new RuntimeException("Unexpected constructors for DistCp");
+ }
+
+ @Test(timeout = 45000)
+ public void setscaniterDeletescaniter() throws Exception {
+ final String table = name.getMethodName();
+
+ // setscaniter, deletescaniter
+ exec("createtable " + table);
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ input.set("true\n\n\nSTRING");
+ exec("setscaniter -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -n name", true);
+ exec("scan", true, "3", true);
+ exec("deletescaniter -n name", true);
+ exec("scan", true, "1", true);
+ exec("deletetable -f " + table);
+
+ }
+
+ @Test(timeout = 45000)
+ public void execfile() throws Exception {
+ // execfile
+ File file = File.createTempFile("ShellServerIT.execfile", ".conf", getFolder());
+ PrintWriter writer = new PrintWriter(file.getAbsolutePath());
+ writer.println("about");
+ writer.close();
+ exec("execfile " + file.getAbsolutePath(), true, Constants.VERSION, true);
+
+ }
+
+ @Test(timeout = 45000)
+ public void egrep() throws Exception {
+ final String table = name.getMethodName();
+
+ // egrep
+ exec("createtable " + table);
+ make10();
+ String lines = exec("egrep row[123]", true);
+ assertTrue(lines.split("\n").length - 1 == 3);
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void du() throws Exception {
+ final String table = name.getMethodName();
+
+ // create and delete a table so we get out of a table context in the shell
+ exec("notable", true);
+
+ // Calling du not in a table context shouldn't throw an error
+ output.clear();
+ exec("du", true, "", true);
+
+ output.clear();
+ exec("createtable " + table);
+ make10();
+ exec("flush -t " + table + " -w");
+ exec("du " + table, true, " [" + table + "]", true);
+ output.clear();
+ shell.execCommand("du -h", false, false);
+ String o = output.get();
+ // for some reason, there's a bit of fluctuation
+ assertTrue("Output did not match regex: '" + o + "'", o.matches(".*[1-9][0-9][0-9]\\s\\[" + table + "\\]\\n"));
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 1000)
+ public void debug() throws Exception {
+ exec("debug", true, "off", true);
+ exec("debug on", true);
+ exec("debug", true, "on", true);
+ exec("debug off", true);
+ exec("debug", true, "off", true);
+ exec("debug debug", false);
+ exec("debug debug debug", false);
+ }
+
+ @Test(timeout = 45000)
+ public void user() throws Exception {
+ final String table = name.getMethodName();
+
+ // createuser, deleteuser, user, users, droptable, grant, revoke
+ input.set("secret\nsecret\n");
+ exec("createuser xyzzy", true);
+ exec("users", true, "xyzzy", true);
+ String perms = exec("userpermissions -u xyzzy", true);
+ assertTrue(perms.contains("Table permissions (" + MetadataTable.NAME + "): Table.READ"));
+ exec("grant -u xyzzy -s System.CREATE_TABLE", true);
+ perms = exec("userpermissions -u xyzzy", true);
+ assertTrue(perms.contains(""));
+ exec("grant -u root -t " + MetadataTable.NAME + " Table.WRITE", true);
+ exec("grant -u root -t " + MetadataTable.NAME + " Table.GOOFY", false);
+ exec("grant -u root -s foo", false);
+ exec("grant -u xyzzy -t " + MetadataTable.NAME + " foo", false);
+ input.set("secret\nsecret\n");
+ exec("user xyzzy", true);
+ exec("createtable " + table, true, "xyzzy@", true);
+ exec("insert row1 cf cq 1", true);
+ exec("scan", true, "row1", true);
+ exec("droptable -f " + table, true);
+ exec("deleteuser xyzzy", false, "delete yourself", true);
+ input.set(ROOT_PASSWORD + "\n" + ROOT_PASSWORD + "\n");
+ exec("user root", true);
+ exec("revoke -u xyzzy -s System.CREATE_TABLE", true);
+ exec("revoke -u xyzzy -s System.GOOFY", false);
+ exec("revoke -u xyzzy -s foo", false);
+ exec("revoke -u xyzzy -t " + MetadataTable.NAME + " Table.WRITE", true);
+ exec("revoke -u xyzzy -t " + MetadataTable.NAME + " Table.GOOFY", false);
+ exec("revoke -u xyzzy -t " + MetadataTable.NAME + " foo", false);
+ exec("deleteuser xyzzy", true);
+ exec("users", true, "xyzzy", false);
+ }
+
+ @Test(timeout = 45000)
+ public void iter() throws Exception {
+ final String table = name.getMethodName();
+
+ // setshelliter, listshelliter, deleteshelliter
+ exec("createtable " + table);
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ input.set("true\n\n\nSTRING\n");
+ exec("setshelliter -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -pn sum -n name", true);
+ exec("setshelliter -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 11 -pn sum -n name", false);
+ exec("setshelliter -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -pn sum -n other", false);
+ input.set("true\n\n\nSTRING\n");
+ exec("setshelliter -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 11 -pn sum -n xyzzy", true);
+ exec("scan -pn sum", true, "3", true);
+ exec("listshelliter", true, "Iterator name", true);
+ exec("listshelliter", true, "Iterator xyzzy", true);
+ exec("listshelliter", true, "Profile : sum", true);
+ exec("deleteshelliter -pn sum -n name", true);
+ exec("listshelliter", true, "Iterator name", false);
+ exec("listshelliter", true, "Iterator xyzzy", true);
+ exec("deleteshelliter -pn sum -a", true);
+ exec("listshelliter", true, "Iterator xyzzy", false);
+ exec("listshelliter", true, "Profile : sum", false);
+ exec("deletetable -f " + table);
+ // list iter
+ exec("createtable " + table);
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ input.set("true\n\n\nSTRING\n");
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -n name", true);
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 11 -n name", false);
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -n other", false);
+ input.set("true\n\n\nSTRING\n");
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 11 -n xyzzy", true);
+ exec("scan", true, "3", true);
+ exec("listiter -scan", true, "Iterator name", true);
+ exec("listiter -scan", true, "Iterator xyzzy", true);
+ exec("listiter -minc", true, "Iterator name", false);
+ exec("listiter -minc", true, "Iterator xyzzy", false);
+ exec("deleteiter -scan -n name", true);
+ exec("listiter -scan", true, "Iterator name", false);
+ exec("listiter -scan", true, "Iterator xyzzy", true);
+ exec("deletetable -f " + table);
+
+ }
+
+ @Test(timeout = 45000)
+ public void setIterOptionPrompt() throws Exception {
+ Connector conn = getConnector();
+ String tableName = name.getMethodName();
+
+ exec("createtable " + tableName);
+ input.set("\n\n");
+ // Setting a non-optiondescriber with no name should fail
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.ColumnFamilyCounter -p 30", false);
+
+ // Name as option will work
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.ColumnFamilyCounter -p 30 -name cfcounter", true);
+
+ String expectedKey = "table.iterator.scan.cfcounter";
+ String expectedValue = "30,org.apache.accumulo.core.iterators.ColumnFamilyCounter";
+ TableOperations tops = conn.tableOperations();
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+
+ exec("deletetable " + tableName, true);
+ tableName = tableName + "1";
+
+ exec("createtable " + tableName, true);
+
+ input.set("customcfcounter\n\n");
+
+ // Name on the CLI should override OptionDescriber (or user input name, in this case)
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.ColumnFamilyCounter -p 30", true);
+ expectedKey = "table.iterator.scan.customcfcounter";
+ expectedValue = "30,org.apache.accumulo.core.iterators.ColumnFamilyCounter";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+
+ exec("deletetable " + tableName, true);
+ tableName = tableName + "1";
+
+ exec("createtable " + tableName, true);
+
+ input.set("customcfcounter\nname1 value1\nname2 value2\n\n");
+
+ // Name on the CLI should override OptionDescriber (or user input name, in this case)
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.ColumnFamilyCounter -p 30", true);
+ expectedKey = "table.iterator.scan.customcfcounter";
+ expectedValue = "30,org.apache.accumulo.core.iterators.ColumnFamilyCounter";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+ expectedKey = "table.iterator.scan.customcfcounter.opt.name1";
+ expectedValue = "value1";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+ expectedKey = "table.iterator.scan.customcfcounter.opt.name2";
+ expectedValue = "value2";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+
+ exec("deletetable " + tableName, true);
+ tableName = tableName + "1";
+
+ exec("createtable " + tableName, true);
+
+ input.set("\nname1 value1.1,value1.2,value1.3\nname2 value2\n\n");
+
+ // Name on the CLI should override OptionDescriber (or user input name, in this case)
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.ColumnFamilyCounter -p 30 -name cfcounter", true);
+ expectedKey = "table.iterator.scan.cfcounter";
+ expectedValue = "30,org.apache.accumulo.core.iterators.ColumnFamilyCounter";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+ expectedKey = "table.iterator.scan.cfcounter.opt.name1";
+ expectedValue = "value1.1,value1.2,value1.3";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+ expectedKey = "table.iterator.scan.cfcounter.opt.name2";
+ expectedValue = "value2";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+ }
+
+ protected void checkTableForProperty(TableOperations tops, String tableName, String expectedKey, String expectedValue) throws Exception {
+ for (int i = 0; i < 5; i++) {
+ for (Entry<String,String> entry : tops.getProperties(tableName)) {
+ if (expectedKey.equals(entry.getKey())) {
+ assertEquals(expectedValue, entry.getValue());
+ return;
+ }
+ }
+ Thread.sleep(500);
+ }
+
+ fail("Failed to find expected property on " + tableName + ": " + expectedKey + "=" + expectedValue);
+ }
+
+ @Test(timeout = 45000)
+ public void notable() throws Exception {
+ final String table = name.getMethodName();
+
+ // notable
+ exec("createtable " + table, true);
+ exec("scan", true, " " + table + ">", true);
+ assertTrue(output.get().contains(" " + table + ">"));
+ exec("notable", true);
+ exec("scan", false, "Not in a table context.", true);
+ assertFalse(output.get().contains(" " + table + ">"));
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void sleep() throws Exception {
+ // sleep
+ long now = System.currentTimeMillis();
+ exec("sleep 0.2", true);
+ long diff = System.currentTimeMillis() - now;
+ assertTrue("Diff was actually " + diff, diff >= 200);
+ assertTrue("Diff was actually " + diff, diff < 600);
+ }
+
+ @Test(timeout = 45000)
+ public void addauths() throws Exception {
+ final String table = name.getMethodName();
+ // addauths
+ exec("createtable " + table + " -evc");
+ exec("insert a b c d -l foo", false, "does not have authorization", true, new ErrorMessageCallback() {
+ public String getErrorMessage() {
+ try {
+ Connector c = getConnector();
+ return "Current auths for root are: " + c.securityOperations().getUserAuthorizations("root").toString();
+ } catch (Exception e) {
+ return "Could not check authorizations";
+ }
+ }
+ });
+ exec("addauths -s foo,bar", true);
+ exec("getauths", true, "foo", true);
+ exec("getauths", true, "bar", true);
+ exec("insert a b c d -l foo");
+ exec("scan", true, "[foo]");
+ exec("scan -s bar", true, "[foo]", false);
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void byeQuitExit() throws Exception {
+ // bye, quit, exit
+ for (String cmd : "bye quit exit".split(" ")) {
+ assertFalse(shell.getExit());
+ exec(cmd);
+ assertTrue(shell.getExit());
+ shell.setExit(false);
+ }
+ }
+
+ @Test(timeout = 45000)
+ public void classpath() throws Exception {
+ // classpath
+ exec("classpath", true, "Level 2: Java Classloader (loads everything defined by java classpath) URL classpath items are", true);
+ }
+
+ @Test(timeout = 45000)
+ public void clearCls() throws Exception {
+ // clear/cls
+ if (shell.getReader().getTerminal().isAnsiSupported()) {
+ exec("cls", true, "[1;1H");
+ exec("clear", true, "[2J");
+ } else {
+ exec("cls", false, "does not support");
+ exec("clear", false, "does not support");
+ }
+ }
+
+ @Test(timeout = 45000)
+ public void clonetable() throws Exception {
+ final String table = name.getMethodName(), clone = table + "_clone";
+
+ // clonetable
+ exec("createtable " + table + " -evc");
+ exec("config -t " + table + " -s table.split.threshold=123M", true);
+ exec("addsplits -t " + table + " a b c", true);
+ exec("insert a b c value");
+ exec("scan", true, "value", true);
+ exec("clonetable " + table + " " + clone);
+ // verify constraint, config, and splits were cloned
+ exec("constraint --list -t " + clone, true, "VisibilityConstraint=2", true);
+ exec("config -t " + clone + " -np", true, "123M", true);
+ exec("getsplits -t " + clone, true, "a\nb\nc\n");
+ exec("deletetable -f " + table);
+ exec("deletetable -f " + clone);
+ }
+
+ @Test(timeout = 45000)
+ public void testCompactions() throws Exception {
+ final String table = name.getMethodName();
+
+ // compact
+ exec("createtable " + table);
+
+ String tableId = getTableId(table);
+
+ // make two files
+ exec("insert a b c d");
+ exec("flush -w");
+ exec("insert x y z v");
+ exec("flush -w");
+ int oldCount = countFiles(tableId);
+ // merge two files into one
+ exec("compact -t " + table + " -w");
+ assertTrue(countFiles(tableId) < oldCount);
+ exec("addsplits -t " + table + " f");
+ // make two more files:
+ exec("insert m 1 2 3");
+ exec("flush -w");
+ exec("insert n 1 2 3");
+ exec("flush -w");
+ List<String> oldFiles = getFiles(tableId);
+
+ // at this point there are 4 files in the default tablet
+ assertEquals("Files that were found: " + oldFiles, 4, oldFiles.size());
+
+ // compact some data:
+ exec("compact -b g -e z -w");
+ assertEquals(2, countFiles(tableId));
+ exec("compact -w");
+ assertEquals(2, countFiles(tableId));
+ exec("merge --all -t " + table);
+ exec("compact -w");
+ assertEquals(1, countFiles(tableId));
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void constraint() throws Exception {
+ final String table = name.getMethodName();
+
+ // constraint
+ exec("constraint -l -t " + MetadataTable.NAME + "", true, "MetadataConstraints=1", true);
+ exec("createtable " + table + " -evc");
+
+ // Make sure the table is fully propagated through zoocache
+ getTableId(table);
+
+ exec("constraint -l -t " + table, true, "VisibilityConstraint=2", true);
+ exec("constraint -t " + table + " -d 2", true, "Removed constraint 2 from table " + table);
+ // wait for zookeeper updates to propagate
+ UtilWaitThread.sleep(1000);
+ exec("constraint -l -t " + table, true, "VisibilityConstraint=2", false);
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void deletemany() throws Exception {
+ final String table = name.getMethodName();
+
+ // deletemany
+ exec("createtable " + table);
+ make10();
+ assertEquals(10, countkeys(table));
+ exec("deletemany -f -b row8");
+ assertEquals(8, countkeys(table));
+ exec("scan -t " + table + " -np", true, "row8", false);
+ make10();
+ exec("deletemany -f -b row4 -e row5");
+ assertEquals(8, countkeys(table));
+ make10();
+ exec("deletemany -f -c cf:col4,cf:col5");
+ assertEquals(8, countkeys(table));
+ make10();
+ exec("deletemany -f -r row3");
+ assertEquals(9, countkeys(table));
+ make10();
+ exec("deletemany -f -r row3");
+ assertEquals(9, countkeys(table));
+ make10();
+ exec("deletemany -f -b row3 -be -e row5 -ee");
+ assertEquals(9, countkeys(table));
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void deleterows() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table);
+ final String tableId = getTableId(table);
+
+ // deleterows
+ int base = countFiles(tableId);
+ assertEquals(0, base);
+
+ exec("addsplits row5 row7");
+ make10();
+ exec("flush -w -t " + table);
+ List<String> files = getFiles(tableId);
+ assertEquals("Found the following files: " + files, 3, files.size());
+ exec("deleterows -t " + table + " -b row5 -e row7", true);
+ assertEquals(2, countFiles(tableId));
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void groups() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table);
+ exec("setgroups -t " + table + " alpha=a,b,c num=3,2,1");
+ exec("getgroups -t " + table, true, "alpha=a,b,c", true);
+ exec("getgroups -t " + table, true, "num=1,2,3", true);
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void grep() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table, true);
+ make10();
+ exec("grep row[123]", true, "row1", false);
+ exec("grep row5", true, "row5", true);
+ exec("deletetable -f " + table, true);
+ }
+
+ @Test(timeout = 45000)
+ public void help() throws Exception {
+ exec("help -np", true, "Help Commands", true);
+ exec("?", true, "Help Commands", true);
+ for (String c : ("bye exit quit " + "about help info ? " + "deleteiter deletescaniter listiter setiter setscaniter "
+ + "grant revoke systempermissions tablepermissions userpermissions " + "execfile history " + "authenticate cls clear notable sleep table user whoami "
+ + "clonetable config createtable deletetable droptable du exporttable importtable offline online renametable tables "
+ + "addsplits compact constraint flush getgropus getsplits merge setgroups " + "addauths createuser deleteuser dropuser getauths passwd setauths users "
+ + "delete deletemany deleterows egrep formatter interpreter grep importdirectory insert maxrow scan").split(" ")) {
+ exec("help " + c, true);
+ }
+ }
+
+ // @Test(timeout = 45000)
+ public void history() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("history -c", true);
+ exec("createtable " + table);
+ exec("deletetable -f " + table);
+ exec("history", true, table, true);
+ exec("history", true, "history", true);
+ }
+
+ @Test(timeout = 45000)
+ public void importDirectory() throws Exception {
+ final String table = name.getMethodName();
+
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.get(conf);
+ File importDir = new File(getFolder(), "import");
+ importDir.mkdir();
+ String even = new File(importDir, "even.rf").toString();
+ String odd = new File(importDir, "odd.rf").toString();
+ File errorsDir = new File(getFolder(), "errors");
+ errorsDir.mkdir();
+ fs.mkdirs(new Path(errorsDir.toString()));
+ AccumuloConfiguration aconf = AccumuloConfiguration.getDefaultConfiguration();
+ FileSKVWriter evenWriter = FileOperations.getInstance().openWriter(even, fs, conf, aconf);
+ evenWriter.startDefaultLocalityGroup();
+ FileSKVWriter oddWriter = FileOperations.getInstance().openWriter(odd, fs, conf, aconf);
+ oddWriter.startDefaultLocalityGroup();
+ long ts = System.currentTimeMillis();
+ Text cf = new Text("cf");
+ Text cq = new Text("cq");
+ Value value = new Value("value".getBytes());
+ for (int i = 0; i < 100; i += 2) {
+ Key key = new Key(new Text(String.format("%8d", i)), cf, cq, ts);
+ evenWriter.append(key, value);
+ key = new Key(new Text(String.format("%8d", i + 1)), cf, cq, ts);
+ oddWriter.append(key, value);
+ }
+ evenWriter.close();
+ oddWriter.close();
+ assertEquals(0, shell.getExitCode());
+ exec("createtable " + table, true);
+ exec("importdirectory " + importDir + " " + errorsDir + " true", true);
+ exec("scan -r 00000000", true, "00000000", true);
+ exec("scan -r 00000099", true, "00000099", true);
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void info() throws Exception {
+ exec("info", true, Constants.VERSION, true);
+ }
+
+ @Test(timeout = 45000)
+ public void interpreter() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table, true);
+ exec("interpreter -l", true, "HexScan", false);
+ exec("insert \\x02 cf cq value", true);
+ exec("scan -b 02", true, "value", false);
+ exec("interpreter -i org.apache.accumulo.core.util.interpret.HexScanInterpreter", true);
+ // Need to allow time for this to propagate through zoocache/zookeeper
+ UtilWaitThread.sleep(3000);
+
+ exec("interpreter -l", true, "HexScan", true);
+ exec("scan -b 02", true, "value", true);
+ exec("deletetable -f " + table, true);
+ }
+
+ @Test(timeout = 45000)
+ public void listcompactions() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table, true);
+ exec("config -t " + table + " -s table.iterator.minc.slow=30,org.apache.accumulo.test.functional.SlowIterator", true);
+ exec("config -t " + table + " -s table.iterator.minc.slow.opt.sleepTime=1000", true);
+ exec("insert a cf cq value", true);
+ exec("insert b cf cq value", true);
+ exec("insert c cf cq value", true);
+ exec("insert d cf cq value", true);
+ exec("flush -t " + table, true);
+ exec("sleep 0.2", true);
+ exec("listcompactions", true, "default_tablet");
+ String[] lines = output.get().split("\n");
+ String last = lines[lines.length - 1];
+ String[] parts = last.split("\\|");
+ assertEquals(12, parts.length);
+ exec("deletetable -f " + table, true);
+ }
+
+ @Test(timeout = 45000)
+ public void maxrow() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table, true);
+ exec("insert a cf cq value", true);
+ exec("insert b cf cq value", true);
+ exec("insert ccc cf cq value", true);
+ exec("insert zzz cf cq value", true);
+ exec("maxrow", true, "zzz", true);
+ exec("delete zzz cf cq", true);
+ exec("maxrow", true, "ccc", true);
+ exec("deletetable -f " + table, true);
+ }
+
+ @Test(timeout = 45000)
+ public void merge() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table);
+ exec("addsplits a m z");
+ exec("getsplits", true, "z", true);
+ exec("merge --all", true);
+ exec("getsplits", true, "z", false);
+ exec("deletetable -f " + table);
+ exec("getsplits -t " + MetadataTable.NAME + "", true);
+ assertEquals(2, output.get().split("\n").length);
+ exec("getsplits -t accumulo.root", true);
+ assertEquals(1, output.get().split("\n").length);
+ exec("merge --all -t " + MetadataTable.NAME + "");
+ exec("getsplits -t " + MetadataTable.NAME + "", true);
+ assertEquals(1, output.get().split("\n").length);
+ }
+
+ @Test(timeout = 45000)
+ public void ping() throws Exception {
+ for (int i = 0; i < 10; i++) {
+ exec("ping", true, "OK", true);
+ // wait for both tservers to start up
+ if (output.get().split("\n").length == 3)
+ break;
+ UtilWaitThread.sleep(1000);
+
+ }
+ assertEquals(3, output.get().split("\n").length);
+ }
+
+ @Test(timeout = 45000)
+ public void renametable() throws Exception {
+ final String table = name.getMethodName() + "1", rename = name.getMethodName() + "2";
+
+ exec("createtable " + table);
+ exec("insert this is a value");
+ exec("renametable " + table + " " + rename);
+ exec("tables", true, rename, true);
+ exec("tables", true, table, false);
+ exec("scan -t " + rename, true, "value", true);
+ exec("deletetable -f " + rename, true);
+ }
+
+ @Test(timeout = 30000)
+ public void tables() throws Exception {
+ final String table = name.getMethodName(), table1 = table + "_z", table2 = table + "_a";
+ exec("createtable " + table1);
+ exec("createtable " + table2);
+ exec("notable");
+ String lst = exec("tables -l");
+ assertTrue(lst.indexOf(table2) < lst.indexOf(table1));
+ lst = exec("tables -l -s");
+ assertTrue(lst.indexOf(table1) < lst.indexOf(table2));
+ }
+
+ @Test(timeout = 45000)
+ public void systempermission() throws Exception {
+ exec("systempermissions");
+ assertEquals(11, output.get().split("\n").length - 1);
+ exec("tablepermissions", true);
+ assertEquals(6, output.get().split("\n").length - 1);
+ }
+
+ @Test(timeout = 45000)
+ public void listscans() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table, true);
+
+ // Should be about a 3 second scan
+ for (int i = 0; i < 6; i++) {
+ exec("insert " + i + " cf cq value", true);
+ }
+ exec("config -t " + table + " -s table.iterator.scan.slow=30,org.apache.accumulo.test.functional.SlowIterator", true);
+ exec("config -t " + table + " -s table.iterator.scan.slow.opt.sleepTime=500", true);
+ Thread thread = new Thread() {
+ @Override
+ public void run() {
+ try {
+ Connector connector = getConnector();
+ Scanner s = connector.createScanner(table, Authorizations.EMPTY);
+ FunctionalTestUtils.count(s);
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+ };
+ thread.start();
+
+ List<String> scans = new ArrayList<String>();
+ // Try to find the active scan for about 5seconds
+ for (int i = 0; i < 50 && scans.isEmpty(); i++) {
+ String currentScans = exec("listscans", true);
+ String[] lines = currentScans.split("\n");
+ for (int scanOffset = 2; i < lines.length; i++) {
+ String currentScan = lines[scanOffset];
+ if (currentScan.contains(table)) {
+ scans.add(currentScan);
+ }
+ }
+ UtilWaitThread.sleep(100);
+ }
+ thread.join();
+
+ assertFalse("Could not find any active scans over table " + table, scans.isEmpty());
+
+ for (String scan : scans) {
+ assertTrue("Scan does not appear to be a 'RUNNING' scan: '" + scan + "'", scan.contains("RUNNING"));
+ String parts[] = scan.split("\\|");
+ assertEquals("Expected 13 colums, but found " + parts.length + " instead for '" + Arrays.toString(parts) + "'", 13, parts.length);
+ String tserver = parts[0].trim();
+ // TODO: any way to tell if the client address is accurate? could be local IP, host, loopback...?
+ String hostPortPattern = ".+:\\d+";
+ assertTrue(tserver.matches(hostPortPattern));
+ assertTrue(getConnector().instanceOperations().getTabletServers().contains(tserver));
+ String client = parts[1].trim();
+ assertTrue(client.matches(hostPortPattern));
+ }
+
+ exec("deletetable -f " + table, true);
+ }
+
+ @Test(timeout = 45000)
+ public void testPertableClasspath() throws Exception {
+ final String table = name.getMethodName();
+
+ File fooFilterJar = File.createTempFile("FooFilter", ".jar", getFolder());
+
+ FileUtils.copyURLToFile(this.getClass().getResource("/FooFilter.jar"), fooFilterJar);
+ fooFilterJar.deleteOnExit();
+
+ File fooConstraintJar = File.createTempFile("FooConstraint", ".jar", getFolder());
+ FileUtils.copyURLToFile(this.getClass().getResource("/FooConstraint.jar"), fooConstraintJar);
+ fooConstraintJar.deleteOnExit();
+
+ exec(
+ "config -s " + Property.VFS_CONTEXT_CLASSPATH_PROPERTY.getKey() + "cx1=" + fooFilterJar.toURI().toString() + "," + fooConstraintJar.toURI().toString(),
+ true);
+
+ exec("createtable " + table, true);
+ exec("config -t " + table + " -s " + Property.TABLE_CLASSPATH.getKey() + "=cx1", true);
+
+ UtilWaitThread.sleep(200);
+
+ // We can't use the setiter command as Filter implements OptionDescriber which
+ // forces us to enter more input that I don't know how to input
+ // Instead, we can just manually set the property on the table.
+ exec("config -t " + table + " -s " + Property.TABLE_ITERATOR_PREFIX.getKey() + "scan.foo=10,org.apache.accumulo.test.FooFilter");
+
+ exec("insert foo f q v", true);
+
+ UtilWaitThread.sleep(100);
+
+ exec("scan -np", true, "foo", false);
+
+ exec("constraint -a FooConstraint", true);
+
+ exec("offline -w " + table);
+ exec("online -w " + table);
+
+ exec("table " + table, true);
+ exec("insert foo f q v", false);
+ exec("insert ok foo q v", true);
+
+ exec("deletetable -f " + table, true);
+ exec("config -d " + Property.VFS_CONTEXT_CLASSPATH_PROPERTY.getKey() + "cx1");
+
+ }
+
+ @Test(timeout = 45000)
+ public void trace() throws Exception {
+ // Make sure to not collide with the "trace" table
+ final String table = name.getMethodName() + "Test";
+
+ exec("trace on", true);
+ exec("createtable " + table, true);
+ exec("insert a b c value", true);
+ exec("scan -np", true, "value", true);
+ exec("deletetable -f " + table);
+ exec("sleep 1");
+ String trace = exec("trace off");
+ System.out.println(trace);
+ assertTrue(trace.contains("sendMutations"));
+ assertTrue(trace.contains("startScan"));
+ assertTrue(trace.contains("DeleteTable"));
+ }
+
+ @Test(timeout = 30000)
+ public void badLogin() throws Exception {
+ input.set(ROOT_PASSWORD + "\n");
+ String err = exec("user NoSuchUser", false);
+ assertTrue(err.contains("BAD_CREDENTIALS for user NoSuchUser"));
+ }
+
+ @Test(timeout = 30000)
+ public void namespaces() throws Exception {
+ exec("namespaces", true, "\"\"", true); // default namespace, displayed as quoted empty string
+ exec("namespaces", true, Namespaces.ACCUMULO_NAMESPACE, true);
+ exec("createnamespace thing1", true);
+ String namespaces = exec("namespaces");
+ assertTrue(namespaces.contains("thing1"));
+
+ exec("renamenamespace thing1 thing2");
+ namespaces = exec("namespaces");
+ assertTrue(namespaces.contains("thing2"));
+ assertTrue(!namespaces.contains("thing1"));
+
+ // can't delete a namespace that still contains tables, unless you do -f
+ exec("createtable thing2.thingy", true);
+ exec("deletenamespace thing2");
+ exec("y");
+ exec("namespaces", true, "thing2", true);
+
+ exec("du -ns thing2", true, "thing2.thingy", true);
+
+ // all "TableOperation" commands can take a namespace
+ exec("offline -ns thing2", true);
+ exec("online -ns thing2", true);
+ exec("flush -ns thing2", true);
+ exec("compact -ns thing2", true);
+ exec("createnamespace testers3", true);
+ exec("createtable testers3.1", true);
+ exec("createtable testers3.2", true);
+ exec("deletetable -ns testers3 -f", true);
+ exec("tables", true, "testers3.1", false);
+ exec("namespaces", true, "testers3", true);
+ exec("deletenamespace testers3 -f", true);
+ input.set("true\n\n\nSTRING\n");
+ exec("setiter -ns thing2 -scan -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -n name", true);
+ exec("listiter -ns thing2 -scan", true, "Summing", true);
+ exec("deleteiter -ns thing2 -n name -scan", true);
+ exec("createuser dude");
+ exec("pass");
+ exec("pass");
+ exec("grant Namespace.CREATE_TABLE -ns thing2 -u dude", true);
+ exec("revoke Namespace.CREATE_TABLE -ns thing2 -u dude", true);
+
+ // properties override and such
+ exec("config -ns thing2 -s table.file.max=44444", true);
+ exec("config -ns thing2", true, "44444", true);
+ exec("config -t thing2.thingy", true, "44444", true);
+ exec("config -t thing2.thingy -s table.file.max=55555", true);
+ exec("config -t thing2.thingy", true, "55555", true);
+
+ // can copy properties when creating
+ exec("createnamespace thing3 -cc thing2", true);
+ exec("config -ns thing3", true, "44444", true);
+
+ exec("deletenamespace -f thing2", true);
+ exec("namespaces", true, "thing2", false);
+ exec("tables", true, "thing2.thingy", false);
+
+ // put constraints on a namespace
+ exec("constraint -ns thing3 -a org.apache.accumulo.examples.simple.constraints.NumericValueConstraint", true);
+ exec("createtable thing3.constrained", true);
+ exec("table thing3.constrained", true);
+ exec("constraint -d 1");
+ // should fail
+ exec("constraint -l", true, "NumericValueConstraint", true);
+ exec("insert r cf cq abc", false);
+ exec("constraint -ns thing3 -d 1");
+ exec("sleep 1");
+ exec("insert r cf cq abc", true);
+ }
+
+ private int countkeys(String table) throws IOException {
+ exec("scan -np -t " + table);
+ return output.get().split("\n").length - 1;
+ }
+
+ @Test(timeout = 30000)
+ public void scans() throws Exception {
+ exec("createtable t");
+ make10();
+ String result = exec("scan -np -b row1 -e row1");
+ assertEquals(2, result.split("\n").length);
+ result = exec("scan -np -b row3 -e row5");
+ assertEquals(4, result.split("\n").length);
+ result = exec("scan -np -r row3");
+ assertEquals(2, result.split("\n").length);
+ result = exec("scan -np -b row:");
+ assertEquals(1, result.split("\n").length);
+ result = exec("scan -np -b row");
+ assertEquals(11, result.split("\n").length);
+ result = exec("scan -np -e row:");
+ assertEquals(11, result.split("\n").length);
+ exec("deletetable -f t");
+ }
+
+ @Test(timeout = 30000)
+ public void whoami() throws Exception {
+ assertTrue(exec("whoami", true).contains("root"));
+ input.set("secret\nsecret\n");
+ exec("createuser test_user");
+ exec("setauths -u test_user -s 12,3,4");
+ String auths = exec("getauths -u test_user");
+ assertTrue(auths.contains("3") && auths.contains("12") && auths.contains("4"));
+ input.set("secret\n");
+ exec("user test_user", true);
+ assertTrue(exec("whoami", true).contains("test_user"));
+ input.set(ROOT_PASSWORD + "\n");
+ exec("user root", true);
+ }
+
+ private void make10() throws IOException {
+ for (int i = 0; i < 10; i++) {
+ exec(String.format("insert row%d cf col%d value", i, i));
+ }
+ }
+
+ private List<String> getFiles(String tableId) throws IOException {
+ output.clear();
+
+ exec("scan -t " + MetadataTable.NAME + " -np -c file -b " + tableId + " -e " + tableId + "~");
+
+ log.debug("countFiles(): " + output.get());
+
+ String[] lines = StringUtils.split(output.get(), "\n");
+ output.clear();
+
+ if (0 == lines.length) {
+ return Collections.emptyList();
+ }
+
+ return Arrays.asList(Arrays.copyOfRange(lines, 1, lines.length));
+ }
+
+ private int countFiles(String tableId) throws IOException {
+ return getFiles(tableId).size();
+ }
+
+ private String getTableId(String tableName) throws Exception {
+ Connector conn = getConnector();
+
+ for (int i = 0; i < 5; i++) {
+ Map<String,String> nameToId = conn.tableOperations().tableIdMap();
+ if (nameToId.containsKey(tableName)) {
+ return nameToId.get(tableName);
+ } else {
+ Thread.sleep(1000);
+ }
+ }
+
+ fail("Could not find ID for table: " + tableName);
+ // Will never get here
+ return null;
+ }
+
+}
[09/14] git commit: Merge branch '1.6.0-SNAPSHOT' of
https://git-wip-us.apache.org/repos/asf/accumulo into 1.6.0-SNAPSHOT
Posted by el...@apache.org.
Merge branch '1.6.0-SNAPSHOT' of https://git-wip-us.apache.org/repos/asf/accumulo into 1.6.0-SNAPSHOT
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/3ed0bbd6
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/3ed0bbd6
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/3ed0bbd6
Branch: refs/heads/ACCUMULO-2061
Commit: 3ed0bbd65032e3f78bbf1e27dd6d3a56a911f08f
Parents: 9760742 fc4de90
Author: Josh Elser <el...@apache.org>
Authored: Tue Mar 11 14:57:34 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Tue Mar 11 14:57:34 2014 -0400
----------------------------------------------------------------------
assemble/pom.xml | 7 ++
bin/accumulo | 5 +
core/pom.xml | 2 +-
init/pom.xml | 98 ++++++++++++++++++++
.../main/packaging/etc/accumulo/accumulo-env.sh | 65 +++++++++++++
.../packaging/etc/accumulo/accumulo-site.xml | 87 +++++++++++++++++
.../main/packaging/etc/accumulo/auditLog.xml | 41 ++++++++
init/src/main/packaging/etc/accumulo/gc | 1 +
.../packaging/etc/accumulo/generic_logger.xml | 83 +++++++++++++++++
.../packaging/etc/accumulo/log4j.properties | 41 ++++++++
init/src/main/packaging/etc/accumulo/masters | 1 +
.../packaging/etc/accumulo/monitor_logger.xml | 64 +++++++++++++
init/src/main/packaging/etc/accumulo/monitors | 1 +
init/src/main/packaging/etc/accumulo/slaves | 1 +
init/src/main/packaging/etc/accumulo/tracers | 1 +
pom.xml | 13 +++
server/base/pom.xml | 7 +-
.../src/main/packaging/etc/init.d/accumulo-gc | 6 +-
.../main/packaging/etc/init.d/accumulo-master | 6 +-
.../main/packaging/etc/init.d/accumulo-monitor | 6 +-
.../main/packaging/etc/init.d/accumulo-tracer | 6 +-
.../main/packaging/etc/init.d/accumulo-tserver | 9 +-
22 files changed, 532 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
[13/14] git commit: ACCUMULO-2061 Excessive logging
Posted by el...@apache.org.
ACCUMULO-2061 Excessive logging
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/85b5caf4
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/85b5caf4
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/85b5caf4
Branch: refs/heads/ACCUMULO-2061
Commit: 85b5caf418e2693f1b2e554dbbbf671d8f15ab6f
Parents: bf5da44
Author: Josh Elser <el...@apache.org>
Authored: Wed Mar 12 01:06:23 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Wed Mar 12 01:06:23 2014 -0400
----------------------------------------------------------------------
.../java/org/apache/accumulo/server/client/HdfsZooInstance.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/85b5caf4/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java b/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
index 6bff9c5..ee928f3 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
@@ -135,7 +135,7 @@ public class HdfsZooInstance implements Instance {
throw new RuntimeException(e);
}
Volume randVolume = fs.getVolumes().iterator().next();
- log.info("Looking for instanceId from " + randVolume);
+ log.trace("Looking for instanceId from " + randVolume);
String instanceIdFromFile = ZooUtil.getInstanceIDFromHdfs(ServerConstants.getInstanceIdLocation(randVolume), acuConf);
instanceId = instanceIdFromFile;
}