You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kt...@apache.org on 2011/11/22 03:26:30 UTC
svn commit: r1204786 - in /incubator/accumulo/branches/1.4:
src/core/src/main/java/org/apache/accumulo/core/
src/server/src/main/java/org/apache/accumulo/server/
src/server/src/main/java/org/apache/accumulo/server/master/
src/server/src/main/java/org/a...
Author: kturner
Date: Tue Nov 22 02:26:29 2011
New Revision: 1204786
URL: http://svn.apache.org/viewvc?rev=1204786&view=rev
Log:
ACCUMULO-169 initial checkin of 1.3 to 1.4 upgrade code w/ a test
Added:
incubator/accumulo/branches/1.4/test/system/upgrade_test.sh (with props)
Removed:
incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/upgrade/
Modified:
incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/Constants.java
incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java
incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
Modified: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/Constants.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/Constants.java?rev=1204786&r1=1204785&r2=1204786&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/Constants.java (original)
+++ incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/Constants.java Tue Nov 22 02:26:29 2011
@@ -29,7 +29,8 @@ import org.apache.hadoop.io.Text;
public class Constants {
public static final String VERSION = "1.4.0-incubating-SNAPSHOT";
- public static final int DATA_VERSION = 3;
+ public static final int DATA_VERSION = 4;
+ public static final int PREV_DATA_VERSION = 3;
// Zookeeper locations
public static final String ZROOT = "/accumulo";
Modified: incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java?rev=1204786&r1=1204785&r2=1204786&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java (original)
+++ incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java Tue Nov 22 02:26:29 2011
@@ -36,6 +36,7 @@ import org.apache.accumulo.server.zookee
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction;
import org.apache.log4j.Logger;
@@ -48,6 +49,23 @@ public class Accumulo {
private static final Logger log = Logger.getLogger(Accumulo.class);
private static Integer dataVersion = null;
+ public static synchronized void updateAccumuloVersion() {
+ Configuration conf = CachedConfiguration.getInstance();
+ try {
+ if (getAccumuloPersistentVersion() == Constants.PREV_DATA_VERSION) {
+ FileSystem fs = TraceFileSystem.wrap(FileUtil.getFileSystem(conf, ServerConfiguration.getSiteConfiguration()));
+
+ fs.create(new Path(ServerConstants.getDataVersionLocation() + "/" + Constants.DATA_VERSION));
+ fs.delete(new Path(ServerConstants.getDataVersionLocation() + "/" + Constants.PREV_DATA_VERSION), false);
+
+ dataVersion = null;
+ }
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to set accumulo version: an error occurred.", e);
+ }
+
+ }
+
public static synchronized int getAccumuloPersistentVersion() {
if (dataVersion != null)
return dataVersion;
@@ -113,7 +131,7 @@ public class Accumulo {
int dataVersion = Accumulo.getAccumuloPersistentVersion();
Version codeVersion = new Version(Constants.VERSION);
- if (dataVersion != Constants.DATA_VERSION) {
+ if (dataVersion != Constants.DATA_VERSION && dataVersion != Constants.PREV_DATA_VERSION) {
throw new RuntimeException("This version of accumulo (" + codeVersion + ") is not compatible with files stored using data version " + dataVersion);
}
Modified: incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/master/Master.java?rev=1204786&r1=1204785&r2=1204786&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/master/Master.java (original)
+++ incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/master/Master.java Tue Nov 22 02:26:29 2011
@@ -35,6 +35,7 @@ import java.util.SortedMap;
import java.util.TimerTask;
import java.util.TreeMap;
import java.util.TreeSet;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.accumulo.core.Constants;
@@ -52,7 +53,6 @@ import org.apache.accumulo.core.client.i
import org.apache.accumulo.core.client.impl.thrift.TableOperation;
import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType;
import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.KeyExtent;
@@ -143,7 +143,6 @@ import org.apache.accumulo.server.util.A
import org.apache.accumulo.server.util.DefaultMap;
import org.apache.accumulo.server.util.Halt;
import org.apache.accumulo.server.util.MetadataTable;
-import org.apache.accumulo.server.util.OfflineMetadataScanner;
import org.apache.accumulo.server.util.SystemPropUtil;
import org.apache.accumulo.server.util.TServerUtils;
import org.apache.accumulo.server.util.TablePropUtil;
@@ -156,6 +155,7 @@ import org.apache.accumulo.server.zookee
import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
import org.apache.accumulo.server.zookeeper.ZooReaderWriter.Mutator;
import org.apache.accumulo.start.classloader.AccumuloClassLoader;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DataInputBuffer;
@@ -308,62 +308,85 @@ public class Master implements LiveTServ
}, 100l, 1000l);
}
- if (oldState != newState && (newState == MasterState.SAFE_MODE || newState == MasterState.NORMAL)) {
- upgradeSettings();
+ if (oldState != newState && (newState == MasterState.HAVE_LOCK)) {
+ upgradeZookeeper();
+ }
+
+ if (oldState != newState && (newState == MasterState.NORMAL)) {
+ upgradeMetadata();
}
}
- private void upgradeSettings() {
- AccumuloConfiguration conf = ServerConfiguration.getTableConfiguration(Constants.METADATA_TABLE_ID);
- IZooReaderWriter zoo = ZooReaderWriter.getInstance();
- if (!conf.getBoolean(Property.TABLE_BLOCKCACHE_ENABLED)) {
+ private void upgradeZookeeper() {
+ if (Accumulo.getAccumuloPersistentVersion() == Constants.PREV_DATA_VERSION) {
+ // TODO check if tablets are loaded, if so abort?
+
try {
- // make sure the last shutdown was clean
- OfflineMetadataScanner scanner = new OfflineMetadataScanner();
- scanner.fetchColumnFamily(Constants.METADATA_LOG_COLUMN_FAMILY);
- boolean fail = false;
- for (Entry<Key,Value> entry : scanner) {
- log.error(String.format("Unable to upgrade: extent %s has log entry %s", entry.getKey().getRow(), entry.getValue()));
- fail = true;
- }
- if (fail)
- throw new Exception("Upgrade requires a clean shutdown");
+ // TODO compare zookeeper dump of 1.3 and 1.4 zookeeper init to make sure no more zookeeper updates are needed
+ // TODO need to update ACL in zookeeper
+ log.info("Upgrading zookeeper");
+
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
- // perform 1.2 -> 1.3 settings
- zset(Property.TABLE_LOCALITY_GROUP_PREFIX.getKey() + "tablet",
- String.format("%s,%s", Constants.METADATA_TABLET_COLUMN_FAMILY.toString(), Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY.toString()));
- zset(Property.TABLE_LOCALITY_GROUP_PREFIX.getKey() + "server", String.format("%s,%s,%s,%s", Constants.METADATA_DATAFILE_COLUMN_FAMILY.toString(),
- Constants.METADATA_LOG_COLUMN_FAMILY.toString(), Constants.METADATA_SERVER_COLUMN_FAMILY.toString(),
- Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY.toString()));
- zset(Property.TABLE_LOCALITY_GROUPS.getKey(), "tablet,server");
- zset(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY.getKey(), "");
- zset(Property.TABLE_INDEXCACHE_ENABLED.getKey(), "true");
- zset(Property.TABLE_BLOCKCACHE_ENABLED.getKey(), "true");
- for (String id : Tables.getIdToNameMap(instance).keySet())
- zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + id + "/state", "ONLINE".getBytes(), NodeExistsPolicy.OVERWRITE);
+ zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZTABLE_LOCKS, new byte[0], NodeExistsPolicy.SKIP);
+ zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZHDFS_RESERVATIONS, new byte[0], NodeExistsPolicy.SKIP);
+ zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZNEXT_FILE, new byte[] {'0'}, NodeExistsPolicy.SKIP);
+
+ for (String id : Tables.getIdToNameMap(instance).keySet()) {
+ zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + id + Constants.ZTABLE_FLUSH_ID, "0".getBytes(), NodeExistsPolicy.SKIP);
+ zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + id + Constants.ZTABLE_COMPACT_ID, "0".getBytes(), NodeExistsPolicy.SKIP);
+ }
} catch (Exception ex) {
log.fatal("Error performing upgrade", ex);
System.exit(1);
}
-
}
- String zkInstanceRoot = ZooUtil.getRoot(instance);
- try {
- if (!zoo.exists(zkInstanceRoot + Constants.ZTABLE_LOCKS)) {
- zoo.putPersistentData(zkInstanceRoot + Constants.ZTABLE_LOCKS, new byte[0], NodeExistsPolicy.FAIL);
- zoo.putPersistentData(zkInstanceRoot + Constants.ZHDFS_RESERVATIONS, new byte[0], NodeExistsPolicy.FAIL);
- }
- } catch (Exception ex) {
- log.fatal("Error performing upgrade", ex);
- System.exit(1);
- }
-
}
- private static void zset(String property, String value) throws KeeperException, InterruptedException {
- TablePropUtil.setTableProperty(Constants.METADATA_TABLE_ID, property, value);
+ private AtomicBoolean upgradeMetadataRunning = new AtomicBoolean(false);
+
+ private void upgradeMetadata() {
+ if (Accumulo.getAccumuloPersistentVersion() == Constants.PREV_DATA_VERSION) {
+ if (upgradeMetadataRunning.compareAndSet(false, true)) {
+ Runnable upgradeTask = new Runnable() {
+ @Override
+ public void run() {
+ try {
+ // add delete entries to metadata table for bulk dirs
+
+ log.info("Adding bulk dir delete entries to !METADATA table for upgrade");
+
+ BatchWriter bw = getConnector().createBatchWriter(Constants.METADATA_TABLE_NAME, 10000000, 60000l, 4);
+
+ FileStatus[] tables = fs.globStatus(new Path(Constants.getTablesDir(ServerConfiguration.getSystemConfiguration()) + "/*"));
+ for (FileStatus tableDir : tables) {
+ FileStatus[] bulkDirs = fs.globStatus(new Path(tableDir.getPath() + "/bulk_*"));
+ for (FileStatus bulkDir : bulkDirs) {
+ bw.addMutation(MetadataTable.createDeleteMutation(tableDir.getPath().getName(), "/" + bulkDir.getPath().getName()));
+ }
+ }
+
+ bw.close();
+
+ Accumulo.updateAccumuloVersion();
+
+ log.info("Upgrade complete");
+
+ } catch (Exception ex) {
+ log.fatal("Error performing upgrade", ex);
+ System.exit(1);
+ }
+
+ }
+ };
+
+ // need to run this in a separate thread because a lock is held that prevents !METADATA tablets from being assigned and this task writes to the
+ // !METADATA table
+ new Thread(upgradeTask).start();
+ }
+ }
}
-
+
private int assignedOrHosted(Text tableId) {
int result = 0;
for (TabletGroupWatcher watcher : watchers) {
Modified: incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java?rev=1204786&r1=1204785&r2=1204786&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java (original)
+++ incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java Tue Nov 22 02:26:29 2011
@@ -525,7 +525,6 @@ public class MetadataTable extends org.a
public static Mutation createDeleteMutation(String tableId, String pathToRemove) {
Mutation delFlag;
- ;
if (pathToRemove.startsWith("../"))
delFlag = new Mutation(new Text(Constants.METADATA_DELETE_FLAG_PREFIX + pathToRemove.substring(2)));
else
Added: incubator/accumulo/branches/1.4/test/system/upgrade_test.sh
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/test/system/upgrade_test.sh?rev=1204786&view=auto
==============================================================================
--- incubator/accumulo/branches/1.4/test/system/upgrade_test.sh (added)
+++ incubator/accumulo/branches/1.4/test/system/upgrade_test.sh Tue Nov 22 02:26:29 2011
@@ -0,0 +1,81 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+if [ $# -ne 1 ]
+then
+ echo "Usage: `basename $0` clean|dirty"
+ exit -1
+fi
+
+#this script test upgrade from 1.3 to 1.4. This script is not self verifying, its output must be inspected for correctness.
+
+#set the following to point to configured 1.3 and 1.4 accumulo dirs. Ensure both point to the same walogs
+
+#TODO could support multinode configs, this script assumes single node config
+
+ONE_THREE_DIR=../../../accumulo-1.3.5-incubating-SNAPSHOT
+ONE_FOUR_DIR=../../
+
+pkill -f accumulo.start
+hadoop fs -rmr /accumulo
+hadoop fs -rmr /testmf
+hadoop fs -rmr /testmfFail
+
+echo "uptest\nsecret\nsecret" | $ONE_THREE_DIR/bin/accumulo init --clear-instance-name
+$ONE_THREE_DIR/bin/start-all.sh
+$ONE_THREE_DIR/bin/accumulo 'org.apache.accumulo.server.test.TestIngest$CreateTable' 0 200000 10 root secret
+$ONE_THREE_DIR/bin/accumulo org.apache.accumulo.server.test.TestIngest -timestamp 1 -size 50 -random 56 100000 0 1
+$ONE_THREE_DIR/bin/accumulo org.apache.accumulo.server.test.TestIngest -mapFile /testmf/mf01 -timestamp 1 -size 50 -random 56 100000 100000 1
+$ONE_THREE_DIR/bin/accumulo org.apache.accumulo.server.test.BulkImportDirectory root secret test_ingest /testmf /testmfFail
+if [ $1 == "dirty" ]; then
+ pkill -9 -f accumulo.start
+else
+ $ONE_THREE_DIR/bin/stop-all.sh
+fi
+
+echo "==== Starting 1.4 ==="
+
+#Test some new 1.4 features against an upgraded instance
+#TODO need to try following operations in different orders
+#TODO test delete range
+
+$ONE_FOUR_DIR/bin/start-all.sh
+$ONE_FOUR_DIR/bin/accumulo org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 1 -random 56 200000 0 1
+echo "compact -t test_ingest -w" | $ONE_FOUR_DIR/bin/accumulo shell -u root -p secret
+$ONE_FOUR_DIR/bin/accumulo org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 1 -random 56 200000 0 1
+echo "merge -t test_ingest -s 1G" | $ONE_FOUR_DIR/bin/accumulo shell -u root -p secret
+$ONE_FOUR_DIR/bin/accumulo org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 1 -random 56 200000 0 1
+echo "clonetable test_ingest tmp\ndeletetable test_ingest\nrenametable tmp test_ingest" | $ONE_FOUR_DIR/bin/accumulo shell -u root -p secret
+$ONE_FOUR_DIR/bin/accumulo org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 1 -random 56 200000 0 1
+
+#test overwriting data writting in 1.3
+$ONE_FOUR_DIR/bin/accumulo org.apache.accumulo.server.test.TestIngest -timestamp 2 -size 50 -random 57 300000 0 1
+$ONE_FOUR_DIR/bin/accumulo org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 2 -random 57 300000 0 1
+echo "compact -t test_ingest -w" | $ONE_FOUR_DIR/bin/accumulo shell -u root -p secret
+$ONE_FOUR_DIR/bin/accumulo org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 2 -random 57 300000 0 1
+
+$ONE_FOUR_DIR/bin/stop-all.sh
+$ONE_FOUR_DIR/bin/start-all.sh
+
+$ONE_FOUR_DIR/bin/accumulo org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 2 -random 57 300000 0 1
+
+pkill -f accumulo.start
+$ONE_FOUR_DIR/bin/start-all.sh
+
+$ONE_FOUR_DIR/bin/accumulo org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 2 -random 57 300000 0 1
+
Propchange: incubator/accumulo/branches/1.4/test/system/upgrade_test.sh
------------------------------------------------------------------------------
svn:executable = *