You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2013/10/28 22:15:02 UTC
git commit: ACCUMULO-1481 creates root table and copies files to the
directory on master start-up
Updated Branches:
refs/heads/master 8683295fc -> e04fd1902
ACCUMULO-1481 creates root table and copies files to the directory on master start-up
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e04fd190
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e04fd190
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e04fd190
Branch: refs/heads/master
Commit: e04fd1902b23b8bca8e1c30795f45d9c596f8623
Parents: 8683295
Author: Eric Newton <er...@gmail.com>
Authored: Mon Oct 28 17:15:23 2013 -0400
Committer: Eric Newton <er...@gmail.com>
Committed: Mon Oct 28 17:15:23 2013 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/core/conf/Property.java | 2 --
.../core/metadata/MetadataLocationObtainer.java | 2 +-
.../accumulo/core/metadata/RootTable.java | 1 +
.../apache/accumulo/server/master/Master.java | 25 ++++++++++++++++++--
.../server/master/recovery/HadoopLogCloser.java | 2 +-
.../master/state/MetaDataTableScanner.java | 6 +++--
.../accumulo/server/tabletserver/Tablet.java | 9 ++++---
.../server/tabletserver/TabletServer.java | 4 ++--
.../tabletserver/log/SortedLogRecovery.java | 9 ++++++-
.../apache/accumulo/server/util/Initialize.java | 15 ++++++++----
.../accumulo/server/util/MetadataTableUtil.java | 15 ++++--------
11 files changed, 61 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e04fd190/core/src/main/java/org/apache/accumulo/core/conf/Property.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 23bae10..134a037 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -594,12 +594,10 @@ public enum Property {
public static Map<String,String> getCompactionStrategyOptions(AccumuloConfiguration tableConf) {
Map<String,String> longNames = tableConf.getAllPropertiesWithPrefix(Property.TABLE_COMPACTION_STRATEGY_PREFIX);
- log.info("longNames " + longNames);
Map<String,String> result = new HashMap<String, String>();
for (Entry<String,String> entry : longNames.entrySet()) {
result.put(entry.getKey().substring(Property.TABLE_COMPACTION_STRATEGY_PREFIX.getKey().length()), entry.getValue());
}
- log.info("result " + result);
return result;
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e04fd190/core/src/main/java/org/apache/accumulo/core/metadata/MetadataLocationObtainer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/MetadataLocationObtainer.java b/core/src/main/java/org/apache/accumulo/core/metadata/MetadataLocationObtainer.java
index 7d6312d..0fb1a8e 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/MetadataLocationObtainer.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/MetadataLocationObtainer.java
@@ -113,7 +113,7 @@ public class MetadataLocationObtainer implements TabletLocationObtainer {
if (opTimer != null)
opTimer.stop("Got " + results.size() + " results from " + src.tablet_extent + " in %DURATION%");
- // System.out.println("results "+results.keySet());
+ //if (log.isTraceEnabled()) log.trace("results "+results);
return MetadataLocationObtainer.getMetadataLocationEntries(results);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e04fd190/core/src/main/java/org/apache/accumulo/core/metadata/RootTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/RootTable.java b/core/src/main/java/org/apache/accumulo/core/metadata/RootTable.java
index 402c1cc..a47f183 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/RootTable.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/RootTable.java
@@ -42,5 +42,6 @@ public class RootTable {
public static final String ZROOT_TABLET_WALOGS = ZROOT_TABLET + "/walogs";
public static final KeyExtent EXTENT = new KeyExtent(new Text(ID), null, null);
+ public static final KeyExtent OLD_EXTENT = new KeyExtent(new Text(MetadataTable.ID), KeyExtent.getMetadataEntry(new Text(MetadataTable.ID), null), null);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e04fd190/server/src/main/java/org/apache/accumulo/server/master/Master.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/master/Master.java b/server/src/main/java/org/apache/accumulo/server/master/Master.java
index 9cbfb51..2c19b6f 100644
--- a/server/src/main/java/org/apache/accumulo/server/master/Master.java
+++ b/server/src/main/java/org/apache/accumulo/server/master/Master.java
@@ -78,6 +78,7 @@ import org.apache.accumulo.core.security.Credentials;
import org.apache.accumulo.core.security.SecurityUtil;
import org.apache.accumulo.core.security.thrift.TCredentials;
import org.apache.accumulo.core.util.ByteBufferUtil;
+import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.Daemon;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.core.zookeeper.ZooUtil;
@@ -95,6 +96,7 @@ import org.apache.accumulo.server.ServerOpts;
import org.apache.accumulo.server.client.HdfsZooInstance;
import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManager.FileType;
import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.accumulo.server.master.LiveTServerSet.TServerConnection;
import org.apache.accumulo.server.master.balancer.DefaultLoadBalancer;
@@ -135,6 +137,7 @@ import org.apache.accumulo.server.security.SecurityOperation;
import org.apache.accumulo.server.security.SystemCredentials;
import org.apache.accumulo.server.util.DefaultMap;
import org.apache.accumulo.server.util.Halt;
+import org.apache.accumulo.server.util.Initialize;
import org.apache.accumulo.server.util.MetadataTableUtil;
import org.apache.accumulo.server.util.SystemPropUtil;
import org.apache.accumulo.server.util.TServerUtils;
@@ -146,6 +149,9 @@ import org.apache.accumulo.server.zookeeper.ZooLock;
import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
import org.apache.accumulo.trace.instrument.thrift.TraceWrap;
import org.apache.accumulo.trace.thrift.TInfo;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.Text;
@@ -265,9 +271,13 @@ public class Master implements LiveTServerSet.Listener, TableObserver, CurrentSt
IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ if (!Tables.exists(instance, RootTable.ID)) {
+ TableManager.prepareNewTableState(instance.getInstanceID(), RootTable.ID, RootTable.NAME, TableState.ONLINE, NodeExistsPolicy.FAIL);
+ Initialize.initMetadataConfig(RootTable.ID);
+ }
+
zoo.recursiveDelete(ZooUtil.getRoot(instance) + "/loggers", NodeMissingPolicy.SKIP);
zoo.recursiveDelete(ZooUtil.getRoot(instance) + "/dead/loggers", NodeMissingPolicy.SKIP);
-
zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZRECOVERY, new byte[] {'0'}, NodeExistsPolicy.SKIP);
for (String id : Tables.getIdToNameMap(instance).keySet()) {
@@ -293,7 +303,18 @@ public class Master implements LiveTServerSet.Listener, TableObserver, CurrentSt
@Override
public void run() {
try {
- MetadataTableUtil.convertRootTabletToRootTable(instance, SystemCredentials.get());
+ Path oldDir = fs.getFullPath(FileType.TABLE, ServerConstants.getDefaultBaseDir() + "/tables/!0/root_tablet");
+ for (FileStatus file : fs.listStatus(oldDir)) {
+ if (file.isFile()) {
+ Path newFile = new Path(ServerConstants.getRootTabletDir(), file.getPath().getName());
+ FileUtil.copy(
+ fs.getFileSystemByPath(file.getPath()), file.getPath(),
+ fs.getFileSystemByPath(newFile), newFile,
+ false,
+ true,
+ CachedConfiguration.getInstance());
+ }
+ }
MetadataTableUtil.moveMetaDeleteMarkers(instance, SystemCredentials.get());
Accumulo.updateAccumuloVersion(fs);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e04fd190/server/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java b/server/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java
index 3e7d406..3eda844 100644
--- a/server/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java
+++ b/server/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java
@@ -46,7 +46,7 @@ public class HadoopLogCloser implements LogCloser {
} catch (FileNotFoundException ex) {
throw ex;
} catch (Exception ex) {
- log.warn("Error recovery lease on " + source.toString(), ex);
+ log.warn("Error recovering lease on " + source.toString(), ex);
ns.append(source).close();
log.info("Recovered lease on " + source.toString() + " using append");
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e04fd190/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java b/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
index cec0bcf..1932958 100644
--- a/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
+++ b/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
@@ -37,6 +37,7 @@ import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.user.WholeRowIterator;
import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ChoppedColumnFamily;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily;
@@ -49,7 +50,7 @@ import org.apache.log4j.Logger;
public class MetaDataTableScanner implements Iterator<TabletLocationState> {
private static final Logger log = Logger.getLogger(MetaDataTableScanner.class);
- BatchScanner mdScanner;
+ BatchScanner mdScanner = null;
Iterator<Entry<Key,Value>> iter;
public MetaDataTableScanner(Instance instance, Credentials credentials, Range range, CurrentState state) {
@@ -65,7 +66,8 @@ public class MetaDataTableScanner implements Iterator<TabletLocationState> {
mdScanner.setRanges(Collections.singletonList(range));
iter = mdScanner.iterator();
} catch (Exception ex) {
- mdScanner.close();
+ if (mdScanner != null)
+ mdScanner.close();
throw new RuntimeException(ex);
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e04fd190/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java b/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
index c734ff7..72c78c3 100644
--- a/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
+++ b/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
@@ -1271,7 +1271,6 @@ public class Tablet {
this.lastCompactID = initCompactID;
if (extent.isRootTablet()) {
-
long rtime = Long.MIN_VALUE;
for (FileRef ref : datafiles.keySet()) {
Path path = ref.path();
@@ -1295,6 +1294,10 @@ public class Tablet {
}
}
}
+ if (time == null && datafiles.isEmpty() && extent.equals(RootTable.OLD_EXTENT)) {
+ // recovery... old root tablet has no data, so time doesn't matter:
+ time = TabletTime.LOGICAL_TIME_ID + "" + Long.MIN_VALUE;
+ }
this.tabletServer = tabletServer;
this.logId = tabletServer.createLogId(extent);
@@ -1430,7 +1433,7 @@ public class Tablet {
removeOldTemporaryFiles();
}
- log.log(TLevel.TABLET_HIST, extent + " opened ");
+ log.log(TLevel.TABLET_HIST, extent + " opened");
}
private void removeOldTemporaryFiles() {
@@ -3176,7 +3179,7 @@ public class Tablet {
droppedFiles.addAll(inputFiles);
if (plan != null)
droppedFiles.addAll(plan.deleteFiles);
- propogateDeletes = !(droppedFiles.equals(allFiles.keySet()));
+ propogateDeletes = droppedFiles.equals(allFiles.keySet());
log.debug("Major compaction plan: " + plan + " propogate deletes : " + propogateDeletes);
filesToCompact = new HashMap<FileRef,DataFileValue>(allFiles);
filesToCompact.keySet().retainAll(inputFiles);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e04fd190/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java b/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
index 92aabf3..c1df5d3 100644
--- a/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
+++ b/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
@@ -2987,7 +2987,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
long now = RelativeTime.currentTimeMillis();
List<String> logSet = new ArrayList<String>();
for (DfsLogger log : logs)
- logSet.add(log.toString());
+ logSet.add(log.getFileName());
MetadataTableUtil.LogEntry entry = new MetadataTableUtil.LogEntry();
entry.extent = extent;
entry.tabletId = id;
@@ -3376,7 +3376,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
throw new AccumuloException("Metadata entry does not have directory (" + metadataEntry + ")");
}
- if (time == null) {
+ if (time == null && !extent.equals(RootTable.OLD_EXTENT)) {
throw new AccumuloException("Metadata entry does not have time (" + metadataEntry + ")");
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e04fd190/server/src/main/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecovery.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecovery.java b/server/src/main/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecovery.java
index 840fad0..2f1a875 100644
--- a/server/src/main/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecovery.java
+++ b/server/src/main/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecovery.java
@@ -29,10 +29,13 @@ import java.util.Set;
import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.logger.LogFileKey;
import org.apache.accumulo.server.logger.LogFileValue;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
import org.apache.log4j.Logger;
/**
@@ -150,6 +153,10 @@ public class SortedLogRecovery {
throw new RuntimeException("COMPACTION_FINISH (without preceding COMPACTION_START) is not followed by a successful minor compaction.");
lastStartToFinish.update(key.tserverSession);
}
+ KeyExtent alternative = extent;
+ if (extent.isRootTablet()) {
+ alternative = RootTable.OLD_EXTENT;
+ }
LogFileKey defineKey = null;
@@ -159,7 +166,7 @@ public class SortedLogRecovery {
// LogReader.printEntry(entry);
if (key.event != DEFINE_TABLET)
break;
- if (key.tablet.equals(extent)) {
+ if (key.tablet.equals(extent) || key.tablet.equals(alternative)) {
if (tid != key.tid) {
tid = key.tid;
defineKey = key;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e04fd190/server/src/main/java/org/apache/accumulo/server/util/Initialize.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/util/Initialize.java b/server/src/main/java/org/apache/accumulo/server/util/Initialize.java
index f7ff87a..3f6ef19 100644
--- a/server/src/main/java/org/apache/accumulo/server/util/Initialize.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/Initialize.java
@@ -237,7 +237,7 @@ public class Initialize {
for (Path mtd : metadataTableDirs) {
try {
fstat = fs.getFileStatus(mtd);
- if (!fstat.isDir()) {
+ if (!fstat.isDirectory()) {
log.fatal("location " + mtd.toString() + " exists but is not a directory");
return;
}
@@ -252,7 +252,7 @@ public class Initialize {
// create root table and tablet
try {
fstat = fs.getFileStatus(rootTablet);
- if (!fstat.isDir()) {
+ if (!fstat.isDirectory()) {
log.fatal("location " + rootTablet.toString() + " exists but is not a directory");
return;
}
@@ -313,14 +313,14 @@ public class Initialize {
Path dir = new Path(s);
try {
fstat = fs.getFileStatus(dir);
- if (!fstat.isDir()) {
+ if (!fstat.isDirectory()) {
log.fatal("location " + dir.toString() + " exists but is not a directory");
return;
}
} catch (FileNotFoundException fnfe) {
try {
fstat = fs.getFileStatus(dir);
- if (!fstat.isDir()) {
+ if (!fstat.isDirectory()) {
log.fatal("location " + dir.toString() + " exists but is not a directory");
return;
}
@@ -432,7 +432,7 @@ public class Initialize {
opts.rootpass);
}
- protected static void initMetadataConfig() throws IOException {
+ public static void initMetadataConfig(String tableId) throws IOException {
try {
Configuration conf = CachedConfiguration.getInstance();
int max = conf.getInt("dfs.replication.max", 512);
@@ -454,6 +454,11 @@ public class Initialize {
}
}
+ protected static void initMetadataConfig() throws IOException {
+ initMetadataConfig(RootTable.ID);
+ initMetadataConfig(MetadataTable.ID);
+ }
+
private static void setMetadataReplication(int replication, String reason) throws IOException {
String rep = getConsoleReader().readLine(
"Your HDFS replication " + reason + " is not compatible with our default " + MetadataTable.NAME + " replication of 5. What do you want to set your "
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e04fd190/server/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java b/server/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index f5ed91b..b52ec9a 100644
--- a/server/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@ -810,7 +810,7 @@ public class MetadataTableUtil {
if (extent.isRootTablet()) {
getRootLogEntries(result);
Path rootDir = new Path(ServerConstants.getRootTabletDir());
- rootDir = rootDir.makeQualified(fs.getDefaultVolume());
+ rootDir = fs.getDefaultVolume().makeQualified(rootDir);
FileStatus[] files = fs.listStatus(rootDir);
for (FileStatus fileStatus : files) {
if (fileStatus.getPath().toString().endsWith("_tmp")) {
@@ -892,6 +892,8 @@ public class MetadataTableUtil {
LogEntry e = new LogEntry();
try {
e.fromBytes(zoo.getData(root + "/" + child, null));
+ // upgrade from !0;!0<< -> !!R<<
+ e.extent = RootTable.EXTENT;
result.add(e);
} catch (KeeperException.NoNodeException ex) {
continue;
@@ -1275,11 +1277,11 @@ public class MetadataTableUtil {
log.info("Moving " + filename + " marker in " + RootTable.NAME);
Mutation m = new Mutation(MetadataSchema.DeletesSection.getRowPrefix() + filename);
m.put(EMPTY_BYTES, EMPTY_BYTES, EMPTY_BYTES);
- update(creds, m, null);
+ update(creds, m, RootTable.EXTENT);
// remove the old entry
m = new Mutation(entry.getKey().getRow());
m.putDelete(EMPTY_BYTES, EMPTY_BYTES);
- update(creds, m, null);
+ update(creds, m, RootTable.OLD_EXTENT);
} else {
break;
}
@@ -1315,11 +1317,4 @@ public class MetadataTableUtil {
return tabletEntries;
}
- public static void convertRootTabletToRootTable(Instance instance, SystemCredentials systemCredentials) throws KeeperException, InterruptedException {
- ZooReaderWriter zoo = ZooReaderWriter.getInstance();
- if (zoo.exists(ZooUtil.getRoot(instance)+"/tables/" + RootTable.ID))
- return;
- TableManager.prepareNewTableState(instance.getInstanceID(), RootTable.ID, RootTable.NAME, TableState.ONLINE, NodeExistsPolicy.FAIL);
- }
-
}