You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kt...@apache.org on 2013/02/09 03:10:15 UTC
svn commit: r1444313 - in /accumulo/trunk:
core/src/main/java/org/apache/accumulo/core/Constants.java
server/src/main/java/org/apache/accumulo/server/master/Master.java
test/system/upgrade_test.sh
Author: kturner
Date: Sat Feb 9 02:10:15 2013
New Revision: 1444313
URL: http://svn.apache.org/r1444313
Log:
ACCUMULO-635 ACCUMULO-1010 got basic upgrade and upgrade test script working...
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/Constants.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java
accumulo/trunk/test/system/upgrade_test.sh
Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/Constants.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/Constants.java?rev=1444313&r1=1444312&r2=1444313&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/Constants.java (original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/Constants.java Sat Feb 9 02:10:15 2013
@@ -35,8 +35,8 @@ public class Constants {
// versions should never be negative
public static final Integer WIRE_VERSION = 2;
- public static final int DATA_VERSION = 4;
- public static final int PREV_DATA_VERSION = 3;
+ public static final int DATA_VERSION = 5;
+ public static final int PREV_DATA_VERSION = 4;
// Zookeeper locations
public static final String ZROOT = "/accumulo";
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java?rev=1444313&r1=1444312&r2=1444313&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java Sat Feb 9 02:10:15 2013
@@ -98,7 +98,6 @@ import org.apache.accumulo.fate.zookeepe
import org.apache.accumulo.server.Accumulo;
import org.apache.accumulo.server.client.HdfsZooInstance;
import org.apache.accumulo.server.conf.ServerConfiguration;
-import org.apache.accumulo.server.iterators.MetadataBulkLoadFilter;
import org.apache.accumulo.server.master.LiveTServerSet.TServerConnection;
import org.apache.accumulo.server.master.balancer.DefaultLoadBalancer;
import org.apache.accumulo.server.master.balancer.TabletBalancer;
@@ -157,7 +156,6 @@ import org.apache.accumulo.server.zookee
import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
import org.apache.accumulo.trace.instrument.thrift.TraceWrap;
import org.apache.accumulo.trace.thrift.TInfo;
-import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DataInputBuffer;
@@ -171,8 +169,6 @@ import org.apache.zookeeper.KeeperExcept
import org.apache.zookeeper.KeeperException.NoNodeException;
import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.ZooDefs;
-import org.apache.zookeeper.data.ACL;
import org.apache.zookeeper.data.Stat;
/**
@@ -279,53 +275,21 @@ public class Master implements LiveTServ
log.info("Upgrading zookeeper");
IZooReaderWriter zoo = ZooReaderWriter.getInstance();
-
- TablePropUtil.setTableProperty(Constants.METADATA_TABLE_ID, Property.TABLE_ITERATOR_PREFIX.getKey() + "majc.bulkLoadFilter", "20,"
- + MetadataBulkLoadFilter.class.getName());
-
- zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZTABLE_LOCKS, new byte[0], NodeExistsPolicy.SKIP);
- zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZHDFS_RESERVATIONS, new byte[0], NodeExistsPolicy.SKIP);
- zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZNEXT_FILE, new byte[] {'0'}, NodeExistsPolicy.SKIP);
-
- String[] tablePropsToDelete = new String[] {"table.scan.cache.size", "table.scan.cache.enable"};
-
+
+ zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZRECOVERY, new byte[] {'0'}, NodeExistsPolicy.SKIP);
+
for (String id : Tables.getIdToNameMap(instance).keySet()) {
- zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + id + Constants.ZTABLE_FLUSH_ID, "0".getBytes(), NodeExistsPolicy.SKIP);
- zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + id + Constants.ZTABLE_COMPACT_ID, "0".getBytes(), NodeExistsPolicy.SKIP);
+
zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + id + Constants.ZTABLE_COMPACT_CANCEL_ID, "0".getBytes(),
NodeExistsPolicy.SKIP);
-
- for (String prop : tablePropsToDelete) {
- String propPath = ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + id + Constants.ZTABLE_CONF + "/" + prop;
- if (zoo.exists(propPath))
- zoo.delete(propPath, -1);
- }
}
-
- setACLs(zoo, ZooUtil.getRoot(instance), ZooUtil.getRoot(instance) + Constants.ZUSERS);
-
} catch (Exception ex) {
log.fatal("Error performing upgrade", ex);
System.exit(1);
}
}
}
-
- private void setACLs(IZooReaderWriter zoo, String root, String users) throws Exception {
- Stat stat = new Stat();
- List<ACL> acls = zoo.getZooKeeper().getACL(root, stat);
- if (acls.equals(ZooDefs.Ids.OPEN_ACL_UNSAFE)) {
- if (root.startsWith(users)) {
- zoo.getZooKeeper().setACL(root, ZooUtil.PRIVATE, -1);
- } else {
- zoo.getZooKeeper().setACL(root, ZooUtil.PUBLIC, -1);
- }
- for (String child : zoo.getChildren(root)) {
- setACLs(zoo, root + "/" + child, users);
- }
- }
- }
-
+
private final AtomicBoolean upgradeMetadataRunning = new AtomicBoolean(false);
private final ServerConfiguration serverConfig;
@@ -337,22 +301,6 @@ public class Master implements LiveTServ
@Override
public void run() {
try {
- // add delete entries to metadata table for bulk dirs
-
- log.info("Adding bulk dir delete entries to !METADATA table for upgrade");
-
- BatchWriter bw = getConnector().createBatchWriter(Constants.METADATA_TABLE_NAME, new BatchWriterConfig());
-
- FileStatus[] tables = fs.globStatus(new Path(Constants.getTablesDir(getSystemConfiguration()) + "/*"));
- for (FileStatus tableDir : tables) {
- FileStatus[] bulkDirs = fs.globStatus(new Path(tableDir.getPath() + "/bulk_*"));
- for (FileStatus bulkDir : bulkDirs) {
- bw.addMutation(MetadataTable.createDeleteMutation(tableDir.getPath().getName(), "/" + bulkDir.getPath().getName()));
- }
- }
-
- bw.close();
-
Accumulo.updateAccumuloVersion(fs);
log.info("Upgrade complete");
Modified: accumulo/trunk/test/system/upgrade_test.sh
URL: http://svn.apache.org/viewvc/accumulo/trunk/test/system/upgrade_test.sh?rev=1444313&r1=1444312&r2=1444313&view=diff
==============================================================================
--- accumulo/trunk/test/system/upgrade_test.sh (original)
+++ accumulo/trunk/test/system/upgrade_test.sh Sat Feb 9 02:10:15 2013
@@ -22,59 +22,53 @@ then
exit -1
fi
-#this script test upgrade from 1.3 to 1.4. This script is not self verifying, its output must be inspected for correctness.
+#this script test upgrade from 1.4 to 1.5. This script is not self verifying, its output must be inspected for correctness.
-#set the following to point to configured 1.3 and 1.4 accumulo dirs. Ensure both point to the same walogs
+#set the following to point to configured 1.4 and 1.5 accumulo dirs. Ensure both point to the same walogs
#TODO could support multinode configs, this script assumes single node config
-ONE_THREE_DIR=../../../accumulo-1.3.5-SNAPSHOT
-ONE_FOUR_DIR=../../
+ONE_FOUR_DIR=../../../accumulo-1.4
+ONE_FIVE_DIR=../../
pkill -f accumulo.start
hadoop fs -rmr /accumulo
hadoop fs -rmr /testmf
hadoop fs -rmr /testmfFail
-echo "uptest\nsecret\nsecret" | $ONE_THREE_DIR/bin/accumulo init --clear-instance-name
-$ONE_THREE_DIR/bin/start-all.sh
-$ONE_THREE_DIR/bin/accumulo org.apache.accumulo.test.TestIngest -u root -p secret --timestamp 1 --size 50 --random 56 --rows 100000 --start 0 --cols 1
-$ONE_THREE_DIR/bin/accumulo org.apache.accumulo.test.TestIngest --rfile /testmf/mf01 --timestamp 1 --size 50 --random 56 --rows 100000 --start 100000 --cols 1
-$ONE_THREE_DIR/bin/accumulo org.apache.accumulo.test.BulkImportDirectory -u root -p secret -t test_ingest --source /testmf --failures /testmfFail
+echo -e "uptest\nsecret\nsecret" | $ONE_FOUR_DIR/bin/accumulo init --clear-instance-name
+$ONE_FOUR_DIR/bin/start-all.sh
+$ONE_FOUR_DIR/bin/accumulo 'org.apache.accumulo.server.test.TestIngest$CreateTable' 0 200000 10 root secret
+$ONE_FOUR_DIR/bin/accumulo org.apache.accumulo.server.test.TestIngest -timestamp 1 -size 50 -random 56 100000 0 1
+$ONE_FOUR_DIR/bin/accumulo org.apache.accumulo.server.test.TestIngest -rfile /testmf/mf01 -timestamp 1 -size 50 -random 56 100000 100000 1
+$ONE_FOUR_DIR/bin/accumulo org.apache.accumulo.server.test.BulkImportDirectory root secret test_ingest /testmf /testmfFail
if [ $1 == "dirty" ]; then
pkill -9 -f accumulo.start
else
- $ONE_THREE_DIR/bin/stop-all.sh
+ $ONE_FOUR_DIR/bin/stop-all.sh
fi
-echo "==== Starting 1.4 ==="
+echo "==== Starting 1.5 ==="
-#Test some new 1.4 features against an upgraded instance
-#TODO need to try following operations in different orders
-#TODO test delete range
-$ONE_FOUR_DIR/bin/start-all.sh
-$ONE_FOUR_DIR/bin/accumulo org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 1 --random 56 --rows 200000 --start 0 --cols 1
-echo "compact -t test_ingest -w" | $ONE_FOUR_DIR/bin/accumulo shell -u root -p secret
-$ONE_FOUR_DIR/bin/accumulo org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 1 --random 56 --rows 200000 --start 0 --cols 1
-echo "merge -t test_ingest -s 1G" | $ONE_FOUR_DIR/bin/accumulo shell -u root -p secret
-$ONE_FOUR_DIR/bin/accumulo org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 1 --random 56 --rows 200000 --start 0 --cols 1
-echo "clonetable test_ingest tmp\ndeletetable test_ingest\nrenametable tmp test_ingest" | $ONE_FOUR_DIR/bin/accumulo shell -u root -p secret
-$ONE_FOUR_DIR/bin/accumulo org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 1 --random 56 --rows 200000 --start 0 --cols 1
-
-#test overwriting data writting in 1.3
-$ONE_FOUR_DIR/bin/accumulo org.apache.accumulo.test.TestIngest --timestamp 2 --size 50 --random 57 --rows 300000 --start 0 --cols 1
-$ONE_FOUR_DIR/bin/accumulo org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2 --random 57 --rows 300000 --start 0 --cols 1
-echo "compact -t test_ingest -w" | $ONE_FOUR_DIR/bin/accumulo shell -u root -p secret
-$ONE_FOUR_DIR/bin/accumulo org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2 --random 57 --rows 300000 --start 0 --cols 1
+$ONE_FIVE_DIR/bin/start-all.sh
+$ONE_FIVE_DIR/bin/accumulo org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 1 --random 56 --rows 200000 --start 0 --cols 1
+echo "compact -t test_ingest -w" | $ONE_FIVE_DIR/bin/accumulo shell -u root -p secret
+$ONE_FIVE_DIR/bin/accumulo org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 1 --random 56 --rows 200000 --start 0 --cols 1
-$ONE_FOUR_DIR/bin/stop-all.sh
-$ONE_FOUR_DIR/bin/start-all.sh
+#test overwriting data writting in 1.4
+$ONE_FIVE_DIR/bin/accumulo org.apache.accumulo.test.TestIngest --timestamp 2 --size 50 --random 57 --rows 300000 --start 0 --cols 1 -cv "L1&L2&G1&GROUP2"
+$ONE_FIVE_DIR/bin/accumulo org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2 --random 57 --rows 300000 --start 0 --cols 1
+echo "compact -t test_ingest -w" | $ONE_FIVE_DIR/bin/accumulo shell -u root -p secret
+$ONE_FIVE_DIR/bin/accumulo org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2 --random 57 --rows 300000 --start 0 --cols 1
-$ONE_FOUR_DIR/bin/accumulo org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2 --random 57 --rows 300000 --start 0 --cols 1
+$ONE_FIVE_DIR/bin/stop-all.sh
+$ONE_FIVE_DIR/bin/start-all.sh
-pkill -f accumulo.start
-$ONE_FOUR_DIR/bin/start-all.sh
+$ONE_FIVE_DIR/bin/accumulo org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2 --random 57 --rows 300000 --start 0 --cols 1
+
+pkill -9 -f accumulo.start
+$ONE_FIVE_DIR/bin/start-all.sh
-$ONE_FOUR_DIR/bin/accumulo org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2 --random 57 --rows 300000 --start 0 --cols 1
+$ONE_FIVE_DIR/bin/accumulo org.apache.accumulo.test.VerifyIngest --size 50 --timestamp 2 --random 57 --rows 300000 --start 0 --cols 1