You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2013/11/06 23:25:53 UTC
[1/8] git commit: ACCUMULO-1835 Add -f,
-x and -r docs to test/system/auto/README
Updated Branches:
refs/heads/master c7697db33 -> 31c8b5ab7
ACCUMULO-1835 Add -f, -x and -r docs to test/system/auto/README
Signed-off-by: Eric Newton <er...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/62c6a227
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/62c6a227
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/62c6a227
Branch: refs/heads/master
Commit: 62c6a227a67264c5298d424ffb739077af32e4a5
Parents: 345b0a6
Author: Bill Havanki <bh...@cloudera.com>
Authored: Mon Nov 4 17:01:21 2013 -0500
Committer: Eric Newton <er...@gmail.com>
Committed: Tue Nov 5 14:12:05 2013 -0500
----------------------------------------------------------------------
test/system/auto/README | 14 ++++++++++++++
1 file changed, 14 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/62c6a227/test/system/auto/README
----------------------------------------------------------------------
diff --git a/test/system/auto/README b/test/system/auto/README
index df447f8..4773393 100644
--- a/test/system/auto/README
+++ b/test/system/auto/README
@@ -21,6 +21,10 @@ You can select tests using a case-insensitive regular expression:
$ ./test/system/auto/run.py -t simple
$ ./test/system/auto/run.py -t SunnyDay
+To run tests repeatedly:
+
+ $ ./test/system/auto/run.py -r 3
+
If you are attempting to debug what is causing a test to fail, you can run the tests in "verbose" mode:
$ python test/system/auto/run.py -t SunnyDay -v 10
@@ -34,6 +38,16 @@ If the test suite hangs, and you would like to re-run the tests starting with th
$ ./test/system/auto/run.py -s start.over.test
+If tests tend to time out (on slower hardware, for example), you can scale up the timeout values by a
+multiplier. This example triples timeouts:
+
+ $ ./test/system/auto/run.py -f 3
+
+Test results are normally printed to the console, but you can send them to XML files compatible with
+Jenkins:
+
+ $ ./test/system/auto/run.py -x
+
The full test suite can take nearly an hour. If you have a larger hadoop cluster at your disposal, you can
run the tests as a map-reduce job:
[2/8] git commit: ACCUMULO-1835 merge to 1.5
Posted by el...@apache.org.
ACCUMULO-1835 merge to 1.5
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/806f00e7
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/806f00e7
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/806f00e7
Branch: refs/heads/master
Commit: 806f00e7e8baf967f14be529cdddd6f7e9a17a78
Parents: e328eab 62c6a22
Author: Eric Newton <er...@gmail.com>
Authored: Tue Nov 5 14:14:17 2013 -0500
Committer: Eric Newton <er...@gmail.com>
Committed: Tue Nov 5 14:14:17 2013 -0500
----------------------------------------------------------------------
test/system/auto/README | 16 +++++++++++++++-
1 file changed, 15 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/806f00e7/test/system/auto/README
----------------------------------------------------------------------
[3/8] git commit: ACCUMULO-1832 fixed build issue
Posted by el...@apache.org.
ACCUMULO-1832 fixed build issue
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f8df8da2
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f8df8da2
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f8df8da2
Branch: refs/heads/master
Commit: f8df8da2f4cca769ba2eb9dd65a8a870ad4c7bdf
Parents: dcb5591
Author: Keith Turner <kt...@apache.org>
Authored: Wed Nov 6 14:53:17 2013 -0500
Committer: Keith Turner <kt...@apache.org>
Committed: Wed Nov 6 14:53:17 2013 -0500
----------------------------------------------------------------------
server/base/pom.xml | 10 +++++-----
server/extras/pom.xml | 9 ++++-----
2 files changed, 9 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f8df8da2/server/base/pom.xml
----------------------------------------------------------------------
diff --git a/server/base/pom.xml b/server/base/pom.xml
index 1e3164f..6f5f4a7 100644
--- a/server/base/pom.xml
+++ b/server/base/pom.xml
@@ -100,17 +100,17 @@
<artifactId>jetty</artifactId>
</dependency>
<dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- <scope>compile</scope>
- </dependency>
- <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<scope>test</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f8df8da2/server/extras/pom.xml
----------------------------------------------------------------------
diff --git a/server/extras/pom.xml b/server/extras/pom.xml
index 8413f4b..b97103f 100644
--- a/server/extras/pom.xml
+++ b/server/extras/pom.xml
@@ -36,13 +36,12 @@
<artifactId>accumulo-tserver</artifactId>
</dependency>
<dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
[6/8] git commit: ACCUMULO-1857 Clean up the README to be more
informative about what users must do instead of guessing at what their
distribution might do.
Posted by el...@apache.org.
ACCUMULO-1857 Clean up the README to be more informative about what
users must do instead of guessing at what their distribution might do.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/b0ec17de
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/b0ec17de
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/b0ec17de
Branch: refs/heads/master
Commit: b0ec17de67e71174cea7c80e986f99ef0e9831b5
Parents: 806f00e
Author: Josh Elser <el...@apache.org>
Authored: Wed Nov 6 17:13:24 2013 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Wed Nov 6 17:13:24 2013 -0500
----------------------------------------------------------------------
README | 21 +++++++++------------
1 file changed, 9 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b0ec17de/README
----------------------------------------------------------------------
diff --git a/README b/README
index 4b29864..2c47ed3 100644
--- a/README
+++ b/README
@@ -81,18 +81,15 @@ You will need to have hadoop installed and configured on your system. Accumulo
you must enable HDFS durable sync. How you enable this depends on your version
of Hadoop. Please consult the table below for information regarding your version.
If you need to set the coniguration, please be sure to restart HDFS. See
-ACCUMULO-623 for more information.
-
-HADOOP RELEASE VERSION SYNC NAME DEFAULT
-Apache Hadoop 0.20.205 dfs.support.append false
-Apache Hadoop 0.23.x dfs.support.append true
-Apache Hadoop 1.0.x dfs.support.append false
-Apache Hadoop 1.1.x dfs.durable.sync true
-Apache Hadoop 2.0.0-2.0.4 dfs.support.append true
-Cloudera CDH 3u0-3u3 ???? true
-Cloudera CDH 3u4 dfs.support.append true
-Hortonworks HDP `1.0 dfs.support.append false
-Hortonworks HDP `1.1 dfs.support.append false
+ACCUMULO-623 and ACCUMULO-1637 for more information.
+
+The following releases of Apache Hadoop require special configuration to ensure
+that data is not inadvertently lost; however, in all releases of Apache Hadoop,
+`dfs.durable.sync` and `dfs.support.append` should *not* be configured as `false`.
+
+VERSION NAME=VALUE
+0.20.205.0 - dfs.support.append=true
+1.0.x - dfs.support.append=true
The example accumulo configuration files are placed in directories based on the
memory footprint for the accumulo processes. If you are using native libraries
[4/8] git commit: ACCUMULO-1761 learn table directories from the
!METADATA table scan
Posted by el...@apache.org.
ACCUMULO-1761 learn table directories from the !METADATA table scan
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/aa26200f
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/aa26200f
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/aa26200f
Branch: refs/heads/master
Commit: aa26200fa978fdb3554aab189c9cd6724775d7f1
Parents: 82a5bc2
Author: Eric Newton <er...@gmail.com>
Authored: Wed Nov 6 16:29:52 2013 -0500
Committer: Eric Newton <er...@gmail.com>
Committed: Wed Nov 6 16:30:51 2013 -0500
----------------------------------------------------------------------
.../java/org/apache/accumulo/server/util/TableDiskUsage.java | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/aa26200f/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java b/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
index b7019e6..6b80ccb 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
@@ -43,6 +43,7 @@ import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.util.NumUtil;
+import org.apache.accumulo.core.util.StringUtil;
import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.cli.ClientOpts;
import org.apache.accumulo.server.fs.VolumeManager;
@@ -147,6 +148,7 @@ public class TableDiskUsage {
HashSet<String> tablesReferenced = new HashSet<String>(tableIds);
HashSet<String> emptyTableIds = new HashSet<String>();
+ HashSet<String> nameSpacesReferenced = new HashSet<String>();
for (String tableId : tableIds) {
Scanner mdScanner = null;
@@ -171,6 +173,10 @@ public class TableDiskUsage {
if (!ref.equals(tableId)) {
tablesReferenced.add(ref);
}
+ if (file.contains(":") && parts.length > 3) {
+ List<String> base = Arrays.asList(Arrays.copyOf(parts, parts.length - 3));
+ nameSpacesReferenced.add(StringUtil.join(base, "/"));
+ }
}
tdu.linkFileAndTable(tableId, uniqueName);
@@ -178,7 +184,7 @@ public class TableDiskUsage {
}
for (String tableId : tablesReferenced) {
- for (String tableDir : ServerConstants.getTablesDirs()) {
+ for (String tableDir : nameSpacesReferenced) {
FileStatus[] files = fs.globStatus(new Path(tableDir + "/" + tableId + "/*/*"));
if (files != null) {
for (FileStatus fileStatus : files) {
[8/8] git commit: Merge branch '1.6.0-SNAPSHOT'
Posted by el...@apache.org.
Merge branch '1.6.0-SNAPSHOT'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/31c8b5ab
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/31c8b5ab
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/31c8b5ab
Branch: refs/heads/master
Commit: 31c8b5ab79d9527109c260ff6a6c3063cf08ce8e
Parents: c7697db 7a83c14
Author: Josh Elser <el...@apache.org>
Authored: Wed Nov 6 17:21:24 2013 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Wed Nov 6 17:21:24 2013 -0500
----------------------------------------------------------------------
README | 21 +++++++++------------
server/base/pom.xml | 10 +++++-----
server/extras/pom.xml | 9 ++++-----
3 files changed, 18 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/31c8b5ab/server/base/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/31c8b5ab/server/extras/pom.xml
----------------------------------------------------------------------
[7/8] git commit: Merge branch '1.5.1-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by el...@apache.org.
Merge branch '1.5.1-SNAPSHOT' into 1.6.0-SNAPSHOT
Conflicts:
test/system/auto/README
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7a83c145
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7a83c145
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7a83c145
Branch: refs/heads/master
Commit: 7a83c145b47f323680abbf9f237f5838330c2731
Parents: aa26200 b0ec17d
Author: Josh Elser <el...@apache.org>
Authored: Wed Nov 6 17:18:39 2013 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Wed Nov 6 17:18:39 2013 -0500
----------------------------------------------------------------------
README | 21 +++++++++------------
1 file changed, 9 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7a83c145/README
----------------------------------------------------------------------
[5/8] git commit: Revert "ACCUMULO-1761 use file size information
from !METADATA instead of asking the namenode,
avoids using ServerConstants.getTablesDirs()"
Posted by el...@apache.org.
Revert "ACCUMULO-1761 use file size information from !METADATA instead of asking the namenode, avoids using ServerConstants.getTablesDirs()"
This reverts commit dcb559165b587eca9e5bf0b64df8031c748d3f37.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/82a5bc25
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/82a5bc25
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/82a5bc25
Branch: refs/heads/master
Commit: 82a5bc25984e2ea6c35842883863931b61e85d77
Parents: f8df8da
Author: Eric Newton <er...@gmail.com>
Authored: Wed Nov 6 16:17:55 2013 -0500
Committer: Eric Newton <er...@gmail.com>
Committed: Wed Nov 6 16:30:51 2013 -0500
----------------------------------------------------------------------
.../accumulo/server/util/TableDiskUsage.java | 26 +++++++++++++-------
1 file changed, 17 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/82a5bc25/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java b/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
index 2d87a43..b7019e6 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
@@ -43,9 +43,12 @@ import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.util.NumUtil;
+import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.cli.ClientOpts;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Logger;
@@ -155,13 +158,11 @@ public class TableDiskUsage {
mdScanner.fetchColumnFamily(DataFileColumnFamily.NAME);
mdScanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
- Iterator<Entry<Key,Value>> mdIter = mdScanner.iterator();
- if (!mdIter.hasNext()) {
+ if (!mdScanner.iterator().hasNext()) {
emptyTableIds.add(tableId);
}
- while (mdIter.hasNext()) {
- Entry<Key,Value> entry = mdIter.next();
+ for (Entry<Key,Value> entry : mdScanner) {
String file = entry.getKey().getColumnQualifier().toString();
String parts[] = file.split("/");
String uniqueName = parts[parts.length - 1];
@@ -173,11 +174,18 @@ public class TableDiskUsage {
}
tdu.linkFileAndTable(tableId, uniqueName);
- String sizeKeys = entry.getValue().toString();
- parts = sizeKeys.split(",");
- // defensive: all file entries should have a size
- if (parts.length == 2) {
- tdu.addFileSize(uniqueName, Long.parseLong(parts[0]));
+ }
+ }
+
+ for (String tableId : tablesReferenced) {
+ for (String tableDir : ServerConstants.getTablesDirs()) {
+ FileStatus[] files = fs.globStatus(new Path(tableDir + "/" + tableId + "/*/*"));
+ if (files != null) {
+ for (FileStatus fileStatus : files) {
+ // Assumes that all filenames are unique
+ String name = fileStatus.getPath().getName();
+ tdu.addFileSize(name, fileStatus.getLen());
+ }
}
}
}