You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kt...@apache.org on 2016/07/22 20:33:36 UTC
[1/3] accumulo git commit: ACCUMULO-4324 Fixed issues found while
attempting to run upgrade test
Repository: accumulo
Updated Branches:
refs/heads/master 7c1d58a41 -> 1ac95b6c6
ACCUMULO-4324 Fixed issues found while attempting to run upgrade test
With these fixes was able to run the update test script going from 1.7.1 and
1.6.5 to 1.8.0-SNAP. Ran script with clean and dirty shutdown.
Also removed option to upgrade from 1.5 and ealier since this was not tested.
Closes apache/accumulo#120, apache/accumulo#121
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a11b28b3
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a11b28b3
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a11b28b3
Branch: refs/heads/master
Commit: a11b28b356011acabc46c23393621012a080fac9
Parents: c8aa1fc
Author: Keith Turner <kt...@apache.org>
Authored: Tue Jul 12 17:24:24 2016 -0400
Committer: Keith Turner <kt...@apache.org>
Committed: Fri Jul 22 16:15:39 2016 -0400
----------------------------------------------------------------------
.../apache/accumulo/server/ServerConstants.java | 2 +-
.../accumulo/server/log/WalStateManager.java | 19 +++++++++++++--
.../java/org/apache/accumulo/master/Master.java | 4 ++++
test/system/upgrade_test.sh | 25 ++++++++++----------
4 files changed, 35 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a11b28b3/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
index e840ea2..a14b8fc 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
@@ -72,7 +72,7 @@ public class ServerConstants {
public static final int LOGGING_TO_HDFS = 4;
public static final BitSet CAN_UPGRADE = new BitSet();
static {
- for (int i : new int[] {DATA_VERSION, MOVE_TO_ROOT_TABLE, MOVE_DELETE_MARKERS, LOGGING_TO_HDFS}) {
+ for (int i : new int[] {DATA_VERSION, MOVE_TO_REPLICATION_TABLE, MOVE_TO_ROOT_TABLE}) {
CAN_UPGRADE.set(i);
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a11b28b3/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java b/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java
index 25d9168..f08bcc4 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java
@@ -86,18 +86,33 @@ public class WalStateManager {
private final Instance instance;
private final ZooReaderWriter zoo;
+ private volatile boolean checkedExistance = false;
+
public WalStateManager(Instance instance, ZooReaderWriter zoo) {
this.instance = instance;
this.zoo = zoo;
}
- private String root() {
- return ZooUtil.getRoot(instance) + ZWALS;
+ private String root() throws WalMarkerException {
+ String root = ZooUtil.getRoot(instance) + ZWALS;
+
+ try {
+ if (!checkedExistance && !zoo.exists(root)) {
+ zoo.putPersistentData(root, new byte[0], NodeExistsPolicy.SKIP);
+ }
+
+ checkedExistance = true;
+ } catch (KeeperException | InterruptedException e) {
+ throw new WalMarkerException(e);
+ }
+
+ return root;
}
// Tablet server exists
public void initWalMarker(TServerInstance tsi) throws WalMarkerException {
byte[] data = new byte[0];
+
try {
zoo.putPersistentData(root() + "/" + tsi.toString(), data, NodeExistsPolicy.FAIL);
} catch (KeeperException | InterruptedException e) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a11b28b3/server/master/src/main/java/org/apache/accumulo/master/Master.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/Master.java b/server/master/src/main/java/org/apache/accumulo/master/Master.java
index 76fafb5..9633a9d 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/Master.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java
@@ -433,6 +433,10 @@ public class Master extends AccumuloServerContext implements LiveTServerSet.List
// add the currlog location for root tablet current logs
zoo.putPersistentData(ZooUtil.getRoot(getInstance()) + RootTable.ZROOT_TABLET_CURRENT_LOGS, new byte[0], NodeExistsPolicy.SKIP);
+
+ // create tablet server wal logs node in ZK
+ zoo.putPersistentData(ZooUtil.getRoot(getInstance()) + WalStateManager.ZWALS, new byte[0], NodeExistsPolicy.SKIP);
+
haveUpgradedZooKeeper = true;
} catch (Exception ex) {
// ACCUMULO-3651 Changed level to error and added FATAL to message for slf4j compatibility
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a11b28b3/test/system/upgrade_test.sh
----------------------------------------------------------------------
diff --git a/test/system/upgrade_test.sh b/test/system/upgrade_test.sh
index 590c07c..651755d 100755
--- a/test/system/upgrade_test.sh
+++ b/test/system/upgrade_test.sh
@@ -28,21 +28,22 @@ fi
#TODO could support multinode configs, this script assumes single node config
-PREV=../../../../accumulo-1.5.0
+PREV=../../../accumulo-1.7.1
CURR=../../
DIR=/accumulo
BULK=/tmp/upt
+INSTANCE=testUp
pkill -f accumulo.start
hadoop fs -rmr "$DIR"
hadoop fs -rmr "$BULK"
-hadoop fs -mkdir "$BULK/fail"
+hadoop fs -mkdir -p "$BULK/fail"
-"$PREV/bin/accumulo" init --clear-instance-name --instance-name testUp --password secret
+"$PREV/bin/accumulo" init --clear-instance-name --instance-name $INSTANCE --password secret
"$PREV/bin/start-all.sh"
-"$PREV/bin/accumulo" org.apache.accumulo.test.TestIngest -u root -p secret --timestamp 1 --size 50 --random 56 --rows 200000 --start 0 --cols 1 --createTable --splits 10
-"$PREV/bin/accumulo" org.apache.accumulo.test.TestIngest --rfile $BULK/bulk/test --timestamp 1 --size 50 --random 56 --rows 200000 --start 200000 --cols 1
+"$PREV/bin/accumulo" org.apache.accumulo.test.TestIngest -i $INSTANCE -u root -p secret --timestamp 1 --size 50 --random 56 --rows 200000 --start 0 --cols 1 --createTable --splits 10
+"$PREV/bin/accumulo" org.apache.accumulo.test.TestIngest -i $INSTANCE -u root -p secret --rfile $BULK/bulk/test --timestamp 1 --size 50 --random 56 --rows 200000 --start 200000 --cols 1
echo -e "table test_ingest\nimportdirectory $BULK/bulk $BULK/fail false" | $PREV/bin/accumulo shell -u root -p secret
if [[ $1 == dirty ]]; then
@@ -54,23 +55,23 @@ fi
echo "==== Starting Current ==="
"$CURR/bin/start-all.sh"
-"$CURR/bin/accumulo" org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 1 --random 56 --rows 400000 --start 0 --cols 1 -u root -p secret
+"$CURR/bin/accumulo" org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 1 --random 56 --rows 400000 --start 0 --cols 1 -i $INSTANCE -u root -p secret
echo "compact -t test_ingest -w" | $CURR/bin/accumulo shell -u root -p secret
-"$CURR/bin/accumulo" org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 1 --random 56 --rows 400000 --start 0 --cols 1 -u root -p secret
+"$CURR/bin/accumulo" org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 1 --random 56 --rows 400000 --start 0 --cols 1 -i $INSTANCE -u root -p secret
-"$CURR/bin/accumulo" org.apache.accumulo.test.TestIngest --timestamp 2 --size 50 --random 57 --rows 500000 --start 0 --cols 1 -u root -p secret
-"$CURR/bin/accumulo" org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2 --random 57 --rows 500000 --start 0 --cols 1 -u root -p secret
+"$CURR/bin/accumulo" org.apache.accumulo.test.TestIngest --timestamp 2 --size 50 --random 57 --rows 500000 --start 0 --cols 1 -i $INSTANCE -u root -p secret
+"$CURR/bin/accumulo" org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2 --random 57 --rows 500000 --start 0 --cols 1 -i $INSTANCE -u root -p secret
echo "compact -t test_ingest -w" | $CURR/bin/accumulo shell -u root -p secret
-"$CURR/bin/accumulo" org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2 --random 57 --rows 500000 --start 0 --cols 1 -u root -p secret
+"$CURR/bin/accumulo" org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2 --random 57 --rows 500000 --start 0 --cols 1 -i $INSTANCE -u root -p secret
"$CURR/bin/stop-all.sh"
"$CURR/bin/start-all.sh"
-"$CURR/bin/accumulo" org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2 --random 57 --rows 500000 --start 0 --cols 1 -u root -p secret
+"$CURR/bin/accumulo" org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2 --random 57 --rows 500000 --start 0 --cols 1 -i $INSTANCE -u root -p secret
pkill -9 -f accumulo.start
"$CURR/bin/start-all.sh"
-"$CURR/bin/accumulo" org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2 --random 57 --rows 500000 --start 0 --cols 1 -u root -p secret
+"$CURR/bin/accumulo" org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2 --random 57 --rows 500000 --start 0 --cols 1 -i $INSTANCE -u root -p secret
[2/3] accumulo git commit: ACCUMULO-4324 update data version
Posted by kt...@apache.org.
ACCUMULO-4324 update data version
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c8aa1fc1
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c8aa1fc1
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c8aa1fc1
Branch: refs/heads/master
Commit: c8aa1fc110b4fb31f78ec5a59932046d79fbe875
Parents: 61e28dc
Author: Michael Wall <mj...@gmail.com>
Authored: Tue Jun 21 14:50:52 2016 -0400
Committer: Keith Turner <kt...@apache.org>
Committed: Fri Jul 22 16:15:39 2016 -0400
----------------------------------------------------------------------
.../main/java/org/apache/accumulo/server/ServerConstants.java | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c8aa1fc1/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
index 8c64d50..e840ea2 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
@@ -47,13 +47,17 @@ public class ServerConstants {
public static final Integer WIRE_VERSION = 3;
/**
+ * version (8) reflects changes to RFile index (ACCUMULO-1124) in version 1.8.0
+ */
+ public static final int SHORTEN_RFILE_KEYS = 8;
+ /**
* version (7) also reflects the addition of a replication table
*/
public static final int MOVE_TO_REPLICATION_TABLE = 7;
/**
* this is the current data version
*/
- public static final int DATA_VERSION = MOVE_TO_REPLICATION_TABLE;
+ public static final int DATA_VERSION = SHORTEN_RFILE_KEYS;
/**
* version (6) reflects the addition of a separate root table (ACCUMULO-1481) in version 1.6.0
*/
[3/3] accumulo git commit: Merge branch '1.8'
Posted by kt...@apache.org.
Merge branch '1.8'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/1ac95b6c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/1ac95b6c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/1ac95b6c
Branch: refs/heads/master
Commit: 1ac95b6c6ea0a09cc34eaa8d42a1fa2f5d60d49f
Parents: 7c1d58a a11b28b
Author: Keith Turner <kt...@apache.org>
Authored: Fri Jul 22 16:17:43 2016 -0400
Committer: Keith Turner <kt...@apache.org>
Committed: Fri Jul 22 16:17:43 2016 -0400
----------------------------------------------------------------------
.../apache/accumulo/server/ServerConstants.java | 8 +++++--
.../accumulo/server/log/WalStateManager.java | 19 +++++++++++++--
.../java/org/apache/accumulo/master/Master.java | 4 ++++
test/system/upgrade_test.sh | 25 ++++++++++----------
4 files changed, 40 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/1ac95b6c/server/master/src/main/java/org/apache/accumulo/master/Master.java
----------------------------------------------------------------------