You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2013/07/17 04:33:14 UTC
[12/50] [abbrv] ACCUMULO-1481 : Add tests for splitting/merging root
table;
refactor to consolidate metadata constants and structures in an organized way;
begin consolidating metadata ops into a servicer interface to abstract the
code that actually does
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/server/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java b/server/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
index 9201f1c..f219603 100644
--- a/server/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
+++ b/server/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
@@ -26,11 +26,11 @@ import java.util.TreeSet;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
import org.apache.accumulo.core.security.SystemPermission;
import org.apache.accumulo.core.security.TablePermission;
import org.apache.accumulo.core.security.thrift.TCredentials;
-import org.apache.accumulo.core.util.MetadataTable;
-import org.apache.accumulo.core.util.RootTable;
import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/server/src/main/java/org/apache/accumulo/server/tabletserver/Compactor.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/tabletserver/Compactor.java b/server/src/main/java/org/apache/accumulo/server/tabletserver/Compactor.java
index 03da11e..60ec7c0 100644
--- a/server/src/main/java/org/apache/accumulo/server/tabletserver/Compactor.java
+++ b/server/src/main/java/org/apache/accumulo/server/tabletserver/Compactor.java
@@ -46,12 +46,12 @@ import org.apache.accumulo.core.iterators.system.ColumnFamilySkippingIterator;
import org.apache.accumulo.core.iterators.system.DeletingIterator;
import org.apache.accumulo.core.iterators.system.MultiIterator;
import org.apache.accumulo.core.iterators.system.TimeSettingIterator;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction;
import org.apache.accumulo.core.tabletserver.thrift.CompactionReason;
import org.apache.accumulo.core.tabletserver.thrift.CompactionType;
import org.apache.accumulo.core.util.LocalityGroupUtil;
import org.apache.accumulo.core.util.LocalityGroupUtil.LocalityGroupConfigurationError;
-import org.apache.accumulo.core.util.MetadataTable.DataFileValue;
import org.apache.accumulo.server.conf.TableConfiguration;
import org.apache.accumulo.server.fs.FileRef;
import org.apache.accumulo.server.fs.VolumeManager;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/server/src/main/java/org/apache/accumulo/server/tabletserver/FileManager.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/tabletserver/FileManager.java b/server/src/main/java/org/apache/accumulo/server/tabletserver/FileManager.java
index 2d7989c..7349d51 100644
--- a/server/src/main/java/org/apache/accumulo/server/tabletserver/FileManager.java
+++ b/server/src/main/java/org/apache/accumulo/server/tabletserver/FileManager.java
@@ -41,7 +41,7 @@ import org.apache.accumulo.core.iterators.system.InterruptibleIterator;
import org.apache.accumulo.core.iterators.system.SourceSwitchingIterator;
import org.apache.accumulo.core.iterators.system.SourceSwitchingIterator.DataSource;
import org.apache.accumulo.core.iterators.system.TimeSettingIterator;
-import org.apache.accumulo.core.util.MetadataTable.DataFileValue;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.fs.FileRef;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/server/src/main/java/org/apache/accumulo/server/tabletserver/MinorCompactor.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/tabletserver/MinorCompactor.java b/server/src/main/java/org/apache/accumulo/server/tabletserver/MinorCompactor.java
index 4f8f398..49ffc65 100644
--- a/server/src/main/java/org/apache/accumulo/server/tabletserver/MinorCompactor.java
+++ b/server/src/main/java/org/apache/accumulo/server/tabletserver/MinorCompactor.java
@@ -25,7 +25,7 @@ import org.apache.accumulo.core.client.impl.Tables;
import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
import org.apache.accumulo.core.master.state.tables.TableState;
-import org.apache.accumulo.core.util.MetadataTable.DataFileValue;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.server.client.HdfsZooInstance;
import org.apache.accumulo.server.conf.TableConfiguration;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java b/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
index 1f6f38b..1305be6 100644
--- a/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
+++ b/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
@@ -78,15 +78,20 @@ import org.apache.accumulo.core.iterators.system.SourceSwitchingIterator.DataSou
import org.apache.accumulo.core.iterators.system.StatsIterator;
import org.apache.accumulo.core.iterators.system.VisibilityFilter;
import org.apache.accumulo.core.master.thrift.TabletLoadState;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.accumulo.core.security.CredentialHelper;
import org.apache.accumulo.core.security.thrift.TCredentials;
import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.LocalityGroupUtil;
-import org.apache.accumulo.core.util.MetadataTable.DataFileValue;
import org.apache.accumulo.core.util.Pair;
-import org.apache.accumulo.core.util.RootTable;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
import org.apache.accumulo.server.ServerConstants;
@@ -115,8 +120,8 @@ import org.apache.accumulo.server.tabletserver.mastermessage.TabletStatusMessage
import org.apache.accumulo.server.tabletserver.metrics.TabletServerMinCMetrics;
import org.apache.accumulo.server.util.FileUtil;
import org.apache.accumulo.server.util.MapCounter;
-import org.apache.accumulo.server.util.MetadataTable;
-import org.apache.accumulo.server.util.MetadataTable.LogEntry;
+import org.apache.accumulo.server.util.MetadataTableUtil;
+import org.apache.accumulo.server.util.MetadataTableUtil.LogEntry;
import org.apache.accumulo.server.util.TabletOperations;
import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
@@ -540,7 +545,7 @@ public class Tablet {
scanFileReservations.put(rid, absFilePaths);
- Map<FileRef,DataFileValue> ret = new HashMap<FileRef,MetadataTable.DataFileValue>();
+ Map<FileRef,DataFileValue> ret = new HashMap<FileRef,DataFileValue>();
for (FileRef path : absFilePaths) {
fileScanReferenceCounts.increment(path, 1);
@@ -578,7 +583,7 @@ public class Tablet {
if (filesToDelete.size() > 0) {
log.debug("Removing scan refs from metadata " + extent + " " + filesToDelete);
- MetadataTable.removeScanFiles(extent, filesToDelete, SecurityConstants.getSystemCredentials(), tabletServer.getLock());
+ MetadataTableUtil.removeScanFiles(extent, filesToDelete, SecurityConstants.getSystemCredentials(), tabletServer.getLock());
}
}
@@ -599,7 +604,7 @@ public class Tablet {
if (filesToDelete.size() > 0) {
log.debug("Removing scan refs from metadata " + extent + " " + filesToDelete);
- MetadataTable.removeScanFiles(extent, filesToDelete, SecurityConstants.getSystemCredentials(), tabletServer.getLock());
+ MetadataTableUtil.removeScanFiles(extent, filesToDelete, SecurityConstants.getSystemCredentials(), tabletServer.getLock());
}
}
@@ -645,7 +650,7 @@ public class Tablet {
String bulkDir = null;
- Map<FileRef,DataFileValue> paths = new HashMap<FileRef,MetadataTable.DataFileValue>();
+ Map<FileRef,DataFileValue> paths = new HashMap<FileRef,DataFileValue>();
for (Entry<FileRef,DataFileValue> entry : pathsString.entrySet())
paths.put(entry.getKey(), entry.getValue());
@@ -660,7 +665,7 @@ public class Tablet {
}
}
if (!inTheRightDirectory) {
- throw new IOException("Map file " + tpath + " not in table dirs");
+ throw new IOException("Data file " + tpath + " not in table dirs");
}
if (bulkDir == null)
@@ -683,7 +688,7 @@ public class Tablet {
throw new IOException(ex);
}
// Remove any bulk files we've previously loaded and compacted away
- List<FileRef> files = MetadataTable.getBulkFilesLoaded(conn, extent, tid);
+ List<FileRef> files = MetadataTableUtil.getBulkFilesLoaded(conn, extent, tid);
for (FileRef file : files)
if (paths.keySet().remove(file.path()))
@@ -705,7 +710,7 @@ public class Tablet {
if (bulkTime > persistedTime)
persistedTime = bulkTime;
- MetadataTable.updateTabletDataFile(tid, extent, paths, tabletTime.getMetadataValue(persistedTime), auths, tabletServer.getLock());
+ MetadataTableUtil.updateTabletDataFile(tid, extent, paths, tabletTime.getMetadataValue(persistedTime), auths, tabletServer.getLock());
}
}
}
@@ -833,7 +838,7 @@ public class Tablet {
// very important to write delete entries outside of log lock, because
// this !METADATA write does not go up... it goes sideways or to itself
if (absMergeFile != null)
- MetadataTable.addDeleteEntries(extent, Collections.singleton(absMergeFile), SecurityConstants.getSystemCredentials());
+ MetadataTableUtil.addDeleteEntries(extent, Collections.singleton(absMergeFile), SecurityConstants.getSystemCredentials());
Set<String> unusedWalLogs = beginClearingUnusedLogs();
try {
@@ -848,7 +853,7 @@ public class Tablet {
persistedTime = commitSession.getMaxCommittedTime();
String time = tabletTime.getMetadataValue(persistedTime);
- MetadataTable.updateTabletDataFile(extent, newDatafile, absMergeFile, dfv, time, creds, filesInUseByScans, tabletServer.getClientAddressString(),
+ MetadataTableUtil.updateTabletDataFile(extent, newDatafile, absMergeFile, dfv, time, creds, filesInUseByScans, tabletServer.getClientAddressString(),
tabletServer.getLock(), unusedWalLogs, lastLocation, flushId);
}
@@ -1032,7 +1037,7 @@ public class Tablet {
Set<FileRef> filesInUseByScans = waitForScansToFinish(oldDatafiles, false, 10000);
if (filesInUseByScans.size() > 0)
log.debug("Adding scan refs to metadata " + extent + " " + filesInUseByScans);
- MetadataTable.replaceDatafiles(extent, oldDatafiles, filesInUseByScans, newDatafile, compactionId, dfv, SecurityConstants.getSystemCredentials(),
+ MetadataTableUtil.replaceDatafiles(extent, oldDatafiles, filesInUseByScans, newDatafile, compactionId, dfv, SecurityConstants.getSystemCredentials(),
tabletServer.getClientAddressString(), lastLocation, tabletServer.getLock());
removeFilesAfterScan(filesInUseByScans);
}
@@ -1043,7 +1048,7 @@ public class Tablet {
public SortedMap<FileRef,DataFileValue> getDatafileSizes() {
synchronized (Tablet.this) {
- TreeMap<FileRef,DataFileValue> copy = new TreeMap<FileRef,MetadataTable.DataFileValue>(datafileSizes);
+ TreeMap<FileRef,DataFileValue> copy = new TreeMap<FileRef,DataFileValue>(datafileSizes);
return Collections.unmodifiableSortedMap(copy);
}
}
@@ -1090,7 +1095,7 @@ public class Tablet {
entries = new TreeMap<Key,Value>();
Text rowName = extent.getMetadataEntry();
for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
- if (entry.getKey().compareRow(rowName) == 0 && MetadataTable.TIME_COLUMN.hasColumns(entry.getKey())) {
+ if (entry.getKey().compareRow(rowName) == 0 && TabletsSection.ServerColumnFamily.TIME_COLUMN.hasColumns(entry.getKey())) {
entries.put(new Key(entry.getKey()), new Value(entry.getValue()));
}
}
@@ -1103,7 +1108,7 @@ public class Tablet {
return null;
}
- private static SortedMap<FileRef,DataFileValue> lookupDatafiles(AccumuloConfiguration conf, Text locText, VolumeManager fs, KeyExtent extent,
+ private static SortedMap<FileRef,DataFileValue> lookupDatafiles(AccumuloConfiguration conf, VolumeManager fs, KeyExtent extent,
SortedMap<Key,Value> tabletsKeyValues) throws IOException {
TreeMap<FileRef,DataFileValue> datafiles = new TreeMap<FileRef,DataFileValue>();
@@ -1134,7 +1139,7 @@ public class Tablet {
mdScanner.setBatchSize(1000);
// leave these in, again, now using endKey for safety
- mdScanner.fetchColumnFamily(MetadataTable.DATAFILE_COLUMN_FAMILY);
+ mdScanner.fetchColumnFamily(DataFileColumnFamily.NAME);
mdScanner.setRange(new Range(rowName));
@@ -1156,7 +1161,7 @@ public class Tablet {
if (ke.isMeta()) {
try {
- logEntries = MetadataTable.getLogEntries(SecurityConstants.getSystemCredentials(), ke);
+ logEntries = MetadataTableUtil.getLogEntries(SecurityConstants.getSystemCredentials(), ke);
} catch (Exception ex) {
throw new RuntimeException("Unable to read tablet log entries", ex);
}
@@ -1166,8 +1171,8 @@ public class Tablet {
for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
Key key = entry.getKey();
if (key.getRow().equals(row)) {
- if (key.getColumnFamily().equals(MetadataTable.LOG_COLUMN_FAMILY)) {
- logEntries.add(MetadataTable.entryFromKeyValue(key, entry.getValue()));
+ if (key.getColumnFamily().equals(LogColumnFamily.NAME)) {
+ logEntries.add(MetadataTableUtil.entryFromKeyValue(key, entry.getValue()));
}
}
}
@@ -1183,7 +1188,7 @@ public class Tablet {
Text row = extent.getMetadataEntry();
for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
Key key = entry.getKey();
- if (key.getRow().equals(row) && key.getColumnFamily().equals(MetadataTable.SCANFILE_COLUMN_FAMILY)) {
+ if (key.getRow().equals(row) && key.getColumnFamily().equals(ScanFileColumnFamily.NAME)) {
String meta = key.getColumnQualifier().toString();
Path path = fs.getFullPath(ServerConstants.getTablesDirs(), meta);
scanFiles.add(new FileRef(meta, path));
@@ -1197,7 +1202,7 @@ public class Tablet {
Text row = extent.getMetadataEntry();
for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
Key key = entry.getKey();
- if (key.getRow().equals(row) && MetadataTable.FLUSH_COLUMN.equals(key.getColumnFamily(), key.getColumnQualifier()))
+ if (key.getRow().equals(row) && TabletsSection.ServerColumnFamily.FLUSH_COLUMN.equals(key.getColumnFamily(), key.getColumnQualifier()))
return Long.parseLong(entry.getValue().toString());
}
@@ -1208,7 +1213,7 @@ public class Tablet {
Text row = extent.getMetadataEntry();
for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
Key key = entry.getKey();
- if (key.getRow().equals(row) && MetadataTable.COMPACT_COLUMN.equals(key.getColumnFamily(), key.getColumnQualifier()))
+ if (key.getRow().equals(row) && TabletsSection.ServerColumnFamily.COMPACT_COLUMN.equals(key.getColumnFamily(), key.getColumnQualifier()))
return Long.parseLong(entry.getValue().toString());
}
@@ -1217,14 +1222,14 @@ public class Tablet {
private Tablet(TabletServer tabletServer, Text location, KeyExtent extent, TabletResourceManager trm, Configuration conf, VolumeManager fs,
SortedMap<Key,Value> tabletsKeyValues) throws IOException {
- this(tabletServer, location, extent, trm, conf, fs, lookupLogEntries(extent, tabletsKeyValues), lookupDatafiles(tabletServer.getSystemConfiguration(),
- location, fs, extent, tabletsKeyValues), lookupTime(tabletServer.getSystemConfiguration(), extent, tabletsKeyValues), lookupLastServer(extent,
- tabletsKeyValues), lookupScanFiles(extent, tabletsKeyValues, fs), lookupFlushID(extent, tabletsKeyValues), lookupCompactID(extent, tabletsKeyValues));
+ this(tabletServer, location, extent, trm, conf, fs, lookupLogEntries(extent, tabletsKeyValues), lookupDatafiles(tabletServer.getSystemConfiguration(), fs,
+ extent, tabletsKeyValues), lookupTime(tabletServer.getSystemConfiguration(), extent, tabletsKeyValues), lookupLastServer(extent, tabletsKeyValues),
+ lookupScanFiles(extent, tabletsKeyValues, fs), lookupFlushID(extent, tabletsKeyValues), lookupCompactID(extent, tabletsKeyValues));
}
private static TServerInstance lookupLastServer(KeyExtent extent, SortedMap<Key,Value> tabletsKeyValues) {
for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
- if (entry.getKey().getColumnFamily().compareTo(MetadataTable.LAST_LOCATION_COLUMN_FAMILY) == 0) {
+ if (entry.getKey().getColumnFamily().compareTo(TabletsSection.LastLocationColumnFamily.NAME) == 0) {
return new TServerInstance(entry.getValue(), entry.getKey().getColumnQualifier());
}
}
@@ -1370,7 +1375,7 @@ public class Tablet {
tabletMemory.updateMemoryUsageStats();
if (count[0] == 0) {
- MetadataTable.removeUnusedWALEntries(extent, logEntries, tabletServer.getLock());
+ MetadataTableUtil.removeUnusedWALEntries(extent, logEntries, tabletServer.getLock());
logEntries.clear();
}
@@ -2211,7 +2216,7 @@ public class Tablet {
TCredentials creds = SecurityConstants.getSystemCredentials();
// if multiple threads were allowed to update this outside of a sync block, then it would be
// a race condition
- MetadataTable.updateTabletFlushID(extent, tableFlushID, creds, tabletServer.getLock());
+ MetadataTableUtil.updateTabletFlushID(extent, tableFlushID, creds, tabletServer.getLock());
} else if (initiateMinor)
initiateMinorCompaction(tableFlushID, MinorCompactionReason.USER);
@@ -2724,7 +2729,7 @@ public class Tablet {
}
try {
- Pair<List<LogEntry>,SortedMap<FileRef,DataFileValue>> fileLog = MetadataTable.getFileAndLogEntries(SecurityConstants.getSystemCredentials(), extent);
+ Pair<List<LogEntry>,SortedMap<FileRef,DataFileValue>> fileLog = MetadataTableUtil.getFileAndLogEntries(SecurityConstants.getSystemCredentials(), extent);
if (fileLog.getFirst().size() != 0) {
String msg = "Closed tablet " + extent + " has walog entries in " + MetadataTable.NAME + " " + fileLog.getFirst();
@@ -2734,7 +2739,7 @@ public class Tablet {
if (extent.isRootTablet()) {
if (!fileLog.getSecond().keySet().equals(datafileManager.getDatafileSizes().keySet())) {
- String msg = "Data file in " + MetadataTable.NAME + " differ from in memory data " + extent + " " + fileLog.getSecond().keySet() + " "
+ String msg = "Data file in " + RootTable.NAME + " differ from in memory data " + extent + " " + fileLog.getSecond().keySet() + " "
+ datafileManager.getDatafileSizes().keySet();
log.error(msg);
throw new RuntimeException(msg);
@@ -3319,7 +3324,7 @@ public class Tablet {
try {
majCStats = _majorCompact(reason);
if (reason == MajorCompactionReason.CHOP) {
- MetadataTable.chopped(getExtent(), this.tabletServer.getLock());
+ MetadataTableUtil.chopped(getExtent(), this.tabletServer.getLock());
tabletServer.enqueueMasterMessage(new TabletStatusMessage(TabletLoadState.CHOPPED, extent));
}
} catch (CompactionCanceledException mcce) {
@@ -3500,7 +3505,7 @@ public class Tablet {
SortedMap<FileRef,DataFileValue> highDatafileSizes = new TreeMap<FileRef,DataFileValue>();
List<FileRef> highDatafilesToRemove = new ArrayList<FileRef>();
- MetadataTable.splitDatafiles(extent.getTableId(), midRow, splitRatio, firstAndLastRows, datafileManager.getDatafileSizes(), lowDatafileSizes,
+ MetadataTableUtil.splitDatafiles(extent.getTableId(), midRow, splitRatio, firstAndLastRows, datafileManager.getDatafileSizes(), lowDatafileSizes,
highDatafileSizes, highDatafilesToRemove);
log.debug("Files for low split " + low + " " + lowDatafileSizes.keySet());
@@ -3511,12 +3516,12 @@ public class Tablet {
// it is possible that some of the bulk loading flags will be deleted after being read below because the bulk load
// finishes.... therefore split could propogate load flags for a finished bulk load... there is a special iterator
// on the !METADATA table to clean up this type of garbage
- Map<FileRef,Long> bulkLoadedFiles = MetadataTable.getBulkFilesLoaded(SecurityConstants.getSystemCredentials(), extent);
+ Map<FileRef,Long> bulkLoadedFiles = MetadataTableUtil.getBulkFilesLoaded(SecurityConstants.getSystemCredentials(), extent);
- MetadataTable.splitTablet(high, extent.getPrevEndRow(), splitRatio, SecurityConstants.getSystemCredentials(), tabletServer.getLock());
- MetadataTable.addNewTablet(low, lowDirectory, tabletServer.getTabletSession(), lowDatafileSizes, bulkLoadedFiles,
+ MetadataTableUtil.splitTablet(high, extent.getPrevEndRow(), splitRatio, SecurityConstants.getSystemCredentials(), tabletServer.getLock());
+ MetadataTableUtil.addNewTablet(low, lowDirectory, tabletServer.getTabletSession(), lowDatafileSizes, bulkLoadedFiles,
SecurityConstants.getSystemCredentials(), time, lastFlushID, lastCompactID, tabletServer.getLock());
- MetadataTable.finishSplit(high, highDatafileSizes, highDatafilesToRemove, SecurityConstants.getSystemCredentials(), tabletServer.getLock());
+ MetadataTableUtil.finishSplit(high, highDatafileSizes, highDatafilesToRemove, SecurityConstants.getSystemCredentials(), tabletServer.getLock());
log.log(TLevel.TABLET_HIST, extent + " split " + low + " " + high);
@@ -3802,7 +3807,7 @@ public class Tablet {
try {
// if multiple threads were allowed to update this outside of a sync block, then it would be
// a race condition
- MetadataTable.updateTabletCompactID(extent, compactionId, SecurityConstants.getSystemCredentials(), tabletServer.getLock());
+ MetadataTableUtil.updateTabletCompactID(extent, compactionId, SecurityConstants.getSystemCredentials(), tabletServer.getLock());
} finally {
synchronized (this) {
majorCompactionInProgress = false;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletIteratorEnvironment.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletIteratorEnvironment.java b/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletIteratorEnvironment.java
index 809ef4e..06c6395 100644
--- a/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletIteratorEnvironment.java
+++ b/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletIteratorEnvironment.java
@@ -28,7 +28,7 @@ import org.apache.accumulo.core.iterators.IteratorEnvironment;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
import org.apache.accumulo.core.iterators.system.MultiIterator;
-import org.apache.accumulo.core.util.MetadataTable.DataFileValue;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.server.fs.FileRef;
import org.apache.accumulo.server.tabletserver.FileManager.ScanFileManager;
import org.apache.hadoop.fs.Path;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java b/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
index 31bc0b9..fe2c2de 100644
--- a/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
+++ b/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
@@ -101,6 +101,9 @@ import org.apache.accumulo.core.master.thrift.MasterClientService;
import org.apache.accumulo.core.master.thrift.TableInfo;
import org.apache.accumulo.core.master.thrift.TabletLoadState;
import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.SecurityUtil;
import org.apache.accumulo.core.security.thrift.TCredentials;
@@ -121,7 +124,6 @@ import org.apache.accumulo.core.util.ColumnFQ;
import org.apache.accumulo.core.util.Daemon;
import org.apache.accumulo.core.util.LoggingRunnable;
import org.apache.accumulo.core.util.Pair;
-import org.apache.accumulo.core.util.RootTable;
import org.apache.accumulo.core.util.ServerServices;
import org.apache.accumulo.core.util.ServerServices.Service;
import org.apache.accumulo.core.util.SimpleThreadPool;
@@ -183,8 +185,8 @@ import org.apache.accumulo.server.tabletserver.metrics.TabletServerUpdateMetrics
import org.apache.accumulo.server.util.FileSystemMonitor;
import org.apache.accumulo.server.util.Halt;
import org.apache.accumulo.server.util.MapCounter;
-import org.apache.accumulo.server.util.MetadataTable;
-import org.apache.accumulo.server.util.MetadataTable.LogEntry;
+import org.apache.accumulo.server.util.MetadataTableUtil;
+import org.apache.accumulo.server.util.MetadataTableUtil.LogEntry;
import org.apache.accumulo.server.util.TServerUtils;
import org.apache.accumulo.server.util.TServerUtils.ServerPort;
import org.apache.accumulo.server.util.time.RelativeTime;
@@ -1254,6 +1256,9 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
tables.add(new String(keyExtent.getTable()));
}
+ if (tables.size() != 1)
+ throw new IllegalArgumentException("Cannot batch scan over multiple tables");
+
// check if user has permission to the tables
Authorizations userauths = null;
for (String table : tables)
@@ -1265,21 +1270,10 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
if (!userauths.contains(ByteBufferUtil.toBytes(auth)))
throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.BAD_AUTHORIZATIONS);
- KeyExtent threadPoolExtent = null;
-
Map<KeyExtent,List<Range>> batch = Translator.translate(tbatch, Translator.TKET, new Translator.ListTranslator<TRange,Range>(Translator.TRT));
- for (KeyExtent keyExtent : batch.keySet()) {
- if (threadPoolExtent == null) {
- threadPoolExtent = keyExtent;
- } else if (keyExtent.isRootTablet()) {
- throw new IllegalArgumentException("Cannot batch query root tablet with other tablets " + threadPoolExtent + " " + keyExtent);
- } else if (keyExtent.isMeta() && !threadPoolExtent.isMeta()) {
- throw new IllegalArgumentException("Cannot batch query " + MetadataTable.NAME + " and non " + MetadataTable.NAME + " tablets " + threadPoolExtent
- + " " + keyExtent);
- }
-
- }
+ // This is used to determine which thread pool to use
+ KeyExtent threadPoolExtent = batch.keySet().iterator().next();
if (waitForWrites)
writeTracker.waitForWrites(TabletType.type(batch.keySet()));
@@ -2597,14 +2591,14 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
List<String> logSet = new ArrayList<String>();
for (DfsLogger log : logs)
logSet.add(log.toString());
- MetadataTable.LogEntry entry = new MetadataTable.LogEntry();
+ MetadataTableUtil.LogEntry entry = new MetadataTableUtil.LogEntry();
entry.extent = extent;
entry.tabletId = id;
entry.timestamp = now;
entry.server = logs.get(0).getLogger();
entry.filename = logs.get(0).getFileName();
entry.logSet = logSet;
- MetadataTable.addLogEntry(SecurityConstants.getSystemCredentials(), entry, getLock());
+ MetadataTableUtil.addLogEntry(SecurityConstants.getSystemCredentials(), entry, getLock());
}
private int startServer(AccumuloConfiguration conf, Property portHint, TProcessor processor, String threadName) throws UnknownHostException {
@@ -2873,7 +2867,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
throw new AccumuloException("Root tablet already has a location set");
}
- return new Pair<Text,KeyExtent>(new Text(RootTable.ZROOT_TABLET), null);
+ return new Pair<Text,KeyExtent>(new Text(RootTable.ROOT_TABLET_LOCATION), null);
}
public static Pair<Text,KeyExtent> verifyTabletInformation(KeyExtent extent, TServerInstance instance, SortedMap<Key,Value> tabletsKeyValues,
@@ -2887,8 +2881,9 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
if (extent.isMeta())
tableToVerify = RootTable.ID;
- List<ColumnFQ> columnsToFetch = Arrays.asList(new ColumnFQ[] {MetadataTable.DIRECTORY_COLUMN, MetadataTable.PREV_ROW_COLUMN,
- MetadataTable.SPLIT_RATIO_COLUMN, MetadataTable.OLD_PREV_ROW_COLUMN, MetadataTable.TIME_COLUMN});
+ List<ColumnFQ> columnsToFetch = Arrays.asList(new ColumnFQ[] {TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN,
+ TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN, TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN,
+ TabletsSection.TabletColumnFamily.OLD_PREV_ROW_COLUMN, TabletsSection.ServerColumnFamily.TIME_COLUMN});
ScannerImpl scanner = new ScannerImpl(HdfsZooInstance.getInstance(), SecurityConstants.getSystemCredentials(), tableToVerify, Authorizations.EMPTY);
scanner.setRange(extent.toMetadataRange());
@@ -2913,18 +2908,18 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
Value oldPrevEndRow = null;
for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
- if (MetadataTable.OLD_PREV_ROW_COLUMN.hasColumns(entry.getKey())) {
+ if (TabletsSection.TabletColumnFamily.OLD_PREV_ROW_COLUMN.hasColumns(entry.getKey())) {
oldPrevEndRow = entry.getValue();
}
}
if (oldPrevEndRow != null) {
SortedMap<Text,SortedMap<ColumnFQ,Value>> tabletEntries;
- tabletEntries = MetadataTable.getTabletEntries(tabletsKeyValues, columnsToFetch);
+ tabletEntries = MetadataTableUtil.getTabletEntries(tabletsKeyValues, columnsToFetch);
KeyExtent fke;
try {
- fke = MetadataTable.fixSplit(metadataEntry, tabletEntries.get(metadataEntry), instance, SecurityConstants.getSystemCredentials(), lock);
+ fke = MetadataTableUtil.fixSplit(metadataEntry, tabletEntries.get(metadataEntry), instance, SecurityConstants.getSystemCredentials(), lock);
} catch (IOException e) {
log.error("Error fixing split " + metadataEntry);
throw new AccumuloException(e.toString());
@@ -2956,19 +2951,19 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
return null;
}
Text cf = key.getColumnFamily();
- if (cf.equals(MetadataTable.FUTURE_LOCATION_COLUMN_FAMILY)) {
+ if (cf.equals(TabletsSection.FutureLocationColumnFamily.NAME)) {
if (future != null) {
throw new AccumuloException("Tablet has multiple future locations " + extent);
}
future = new TServerInstance(entry.getValue(), key.getColumnQualifier());
- } else if (cf.equals(MetadataTable.CURRENT_LOCATION_COLUMN_FAMILY)) {
+ } else if (cf.equals(TabletsSection.CurrentLocationColumnFamily.NAME)) {
log.info("Tablet seems to be already assigned to " + new TServerInstance(entry.getValue(), key.getColumnQualifier()));
return null;
- } else if (MetadataTable.PREV_ROW_COLUMN.hasColumns(key)) {
+ } else if (TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(key)) {
prevEndRow = entry.getValue();
- } else if (MetadataTable.DIRECTORY_COLUMN.hasColumns(key)) {
+ } else if (TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.hasColumns(key)) {
dir = entry.getValue();
- } else if (MetadataTable.TIME_COLUMN.hasColumns(key)) {
+ } else if (TabletsSection.ServerColumnFamily.TIME_COLUMN.hasColumns(key)) {
time = entry.getValue();
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServerResourceManager.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServerResourceManager.java b/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServerResourceManager.java
index 2fdc4cc..78313e7 100644
--- a/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServerResourceManager.java
+++ b/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServerResourceManager.java
@@ -41,11 +41,11 @@ import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.file.blockfile.cache.LruBlockCache;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.core.util.Daemon;
import org.apache.accumulo.core.util.LoggingRunnable;
-import org.apache.accumulo.core.util.MetadataTable.DataFileValue;
import org.apache.accumulo.core.util.NamingThreadFactory;
-import org.apache.accumulo.core.util.RootTable;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.fs.FileRef;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/server/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java b/server/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
index 7255765..813d54c 100644
--- a/server/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
@@ -26,12 +26,12 @@ import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.PartialKey;
-import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.util.MetadataTable;
-import org.apache.accumulo.core.util.RootTable;
import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.cli.ClientOpts;
import org.apache.hadoop.conf.Configuration;
@@ -53,18 +53,16 @@ public class AddFilesWithMissingEntries {
}
/**
- * A utility to add files to the {@value MetadataTable#NAME} table that are not listed in the root tablet. This is a recovery tool for someone who
- * knows what they are doing. It might be better to save off files, and recover your instance by re-initializing and importing the existing files.
+ * A utility to add files to the {@value MetadataTable#NAME} table that are not listed in the root tablet. This is a recovery tool for someone who knows what
+ * they are doing. It might be better to save off files, and recover your instance by re-initializing and importing the existing files.
*/
public static void main(String[] args) throws Exception {
Opts opts = new Opts();
BatchWriterOpts bwOpts = new BatchWriterOpts();
opts.parseArgs(AddFilesWithMissingEntries.class.getName(), args, bwOpts);
- final Key rootTableEnd = new Key(RootTable.EXTENT.getEndRow());
- final Range range = new Range(rootTableEnd.followingKey(PartialKey.ROW), true, MetadataTable.RESERVED_RANGE_START_KEY, false);
final Scanner scanner = opts.getConnector().createScanner(MetadataTable.NAME, Authorizations.EMPTY);
- scanner.setRange(range);
+ scanner.setRange(MetadataSchema.TabletsSection.getRange());
final Configuration conf = new Configuration();
final FileSystem fs = FileSystem.get(conf);
@@ -89,10 +87,10 @@ public class AddFilesWithMissingEntries {
knownFiles.clear();
last = ke;
}
- if (MetadataTable.DIRECTORY_COLUMN.hasColumns(key)) {
+ if (TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.hasColumns(key)) {
directory = entry.getValue().toString();
log.debug("Found directory " + directory + " for row " + key.getRow().toString());
- } else if (key.compareColumnFamily(MetadataTable.DATAFILE_COLUMN_FAMILY) == 0) {
+ } else if (key.compareColumnFamily(DataFileColumnFamily.NAME) == 0) {
String filename = key.getColumnQualifier().toString();
knownFiles.add(filename);
log.debug("METADATA file found: " + filename);
@@ -124,7 +122,7 @@ public class AddFilesWithMissingEntries {
String size = Long.toString(file.getLen());
String entries = "1"; // lie
String value = size + "," + entries;
- m.put(MetadataTable.DATAFILE_COLUMN_FAMILY, new Text(filename), new Value(value.getBytes()));
+ m.put(DataFileColumnFamily.NAME, new Text(filename), new Value(value.getBytes()));
if (update) {
writer.getBatchWriter(MetadataTable.NAME).addMutation(m);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/server/src/main/java/org/apache/accumulo/server/util/Admin.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/util/Admin.java b/server/src/main/java/org/apache/accumulo/server/util/Admin.java
index 617053a..3bb801a 100644
--- a/server/src/main/java/org/apache/accumulo/server/util/Admin.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/Admin.java
@@ -31,9 +31,9 @@ import org.apache.accumulo.core.client.impl.ClientExec;
import org.apache.accumulo.core.client.impl.MasterClient;
import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
import org.apache.accumulo.core.master.thrift.MasterClientService;
+import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.security.CredentialHelper;
import org.apache.accumulo.core.security.thrift.TCredentials;
-import org.apache.accumulo.core.util.MetadataTable;
import org.apache.accumulo.server.cli.ClientOpts;
import org.apache.accumulo.server.client.HdfsZooInstance;
import org.apache.accumulo.server.security.SecurityConstants;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/server/src/main/java/org/apache/accumulo/server/util/CheckForMetadataProblems.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/util/CheckForMetadataProblems.java b/server/src/main/java/org/apache/accumulo/server/util/CheckForMetadataProblems.java
index 9c80927..fde9610 100644
--- a/server/src/main/java/org/apache/accumulo/server/util/CheckForMetadataProblems.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/CheckForMetadataProblems.java
@@ -30,6 +30,9 @@ import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.CredentialHelper;
import org.apache.accumulo.core.tabletserver.thrift.ConstraintViolationException;
@@ -85,7 +88,7 @@ public class CheckForMetadataProblems {
if (broke && opts.fix) {
KeyExtent ke = new KeyExtent(tabke);
ke.setPrevEndRow(lastEndRow);
- MetadataTable.updateTabletPrevEndRow(ke, CredentialHelper.create(opts.principal, opts.getToken(), opts.instance));
+ MetadataTableUtil.updateTabletPrevEndRow(ke, CredentialHelper.create(opts.principal, opts.getToken(), opts.instance));
System.out.println("KE " + tabke + " has been repaired to " + ke);
}
@@ -108,9 +111,9 @@ public class CheckForMetadataProblems {
scanner = opts.getConnector().createScanner(MetadataTable.NAME, Authorizations.EMPTY);
}
- scanner.setRange(MetadataTable.KEYSPACE);
- MetadataTable.PREV_ROW_COLUMN.fetch(scanner);
- scanner.fetchColumnFamily(MetadataTable.CURRENT_LOCATION_COLUMN_FAMILY);
+ scanner.setRange(MetadataSchema.TabletsSection.getRange());
+ TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner);
+ scanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME);
Text colf = new Text();
Text colq = new Text();
@@ -140,16 +143,16 @@ public class CheckForMetadataProblems {
tables.put(tableName, tablets);
}
- if (MetadataTable.PREV_ROW_COLUMN.equals(colf, colq)) {
+ if (TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.equals(colf, colq)) {
KeyExtent tabletKe = new KeyExtent(entry.getKey().getRow(), entry.getValue());
tablets.add(tabletKe);
justLoc = false;
- } else if (colf.equals(MetadataTable.CURRENT_LOCATION_COLUMN_FAMILY)) {
+ } else if (colf.equals(TabletsSection.CurrentLocationColumnFamily.NAME)) {
if (justLoc) {
System.out.println("Problem at key " + entry.getKey());
sawProblems = true;
if (opts.fix) {
- Writer t = MetadataTable.getMetadataTable(CredentialHelper.create(opts.principal, opts.getToken(), opts.instance));
+ Writer t = MetadataTableUtil.getMetadataTable(CredentialHelper.create(opts.principal, opts.getToken(), opts.instance));
Key k = entry.getKey();
Mutation m = new Mutation(k.getRow());
m.putDelete(k.getColumnFamily(), k.getColumnQualifier());
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java b/server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
index 80b1a7c..de27112 100644
--- a/server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
@@ -23,8 +23,8 @@ import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.conf.DefaultConfiguration;
import org.apache.accumulo.core.master.state.tables.TableState;
-import org.apache.accumulo.core.util.MetadataTable;
-import org.apache.accumulo.core.util.RootTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
import org.apache.accumulo.server.cli.ClientOpts;
import org.apache.accumulo.server.master.LiveTServerSet;
import org.apache.accumulo.server.master.LiveTServerSet.Listener;
@@ -48,10 +48,10 @@ public class FindOfflineTablets {
opts.parseArgs(FindOfflineTablets.class.getName(), args);
final AtomicBoolean scanning = new AtomicBoolean(false);
Instance instance = opts.getInstance();
- MetaDataTableScanner rootScanner = new MetaDataTableScanner(instance, SecurityConstants.getSystemCredentials(), RootTable.METADATA_TABLETS_RANGE);
- MetaDataTableScanner metaScanner = new MetaDataTableScanner(instance, SecurityConstants.getSystemCredentials(), MetadataTable.NON_ROOT_KEYSPACE);
+ MetaDataTableScanner rootScanner = new MetaDataTableScanner(instance, SecurityConstants.getSystemCredentials(), MetadataSchema.TabletsSection.getRange());
+ MetaDataTableScanner metaScanner = new MetaDataTableScanner(instance, SecurityConstants.getSystemCredentials(), MetadataSchema.TabletsSection.getRange());
@SuppressWarnings("unchecked")
- Iterator<TabletLocationState> scanner = (Iterator<TabletLocationState>)new IteratorChain(rootScanner, metaScanner);
+ Iterator<TabletLocationState> scanner = new IteratorChain(rootScanner, metaScanner);
LiveTServerSet tservers = new LiveTServerSet(instance, DefaultConfiguration.getDefaultConfiguration(), new Listener() {
@Override
public void update(LiveTServerSet current, Set<TServerInstance> deleted, Set<TServerInstance> added) {
@@ -66,7 +66,8 @@ public class FindOfflineTablets {
while (scanner.hasNext()) {
TabletLocationState locationState = scanner.next();
TabletState state = locationState.getState(tservers.getCurrentServers());
- if (state != null && state != TabletState.HOSTED && TableManager.getInstance().getTableState(locationState.extent.getTableId().toString()) != TableState.OFFLINE)
+ if (state != null && state != TabletState.HOSTED
+ && TableManager.getInstance().getTableState(locationState.extent.getTableId().toString()) != TableState.OFFLINE)
if (!locationState.extent.equals(RootTable.EXTENT))
System.out.println(locationState + " is " + state + " #walogs:" + locationState.walogs.size());
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/server/src/main/java/org/apache/accumulo/server/util/Initialize.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/util/Initialize.java b/server/src/main/java/org/apache/accumulo/server/util/Initialize.java
index 0b62c6c..7d4e6f2 100644
--- a/server/src/main/java/org/apache/accumulo/server/util/Initialize.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/Initialize.java
@@ -43,10 +43,14 @@ import org.apache.accumulo.core.file.FileSKVWriter;
import org.apache.accumulo.core.iterators.user.VersioningIterator;
import org.apache.accumulo.core.master.state.tables.TableState;
import org.apache.accumulo.core.master.thrift.MasterGoalState;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily;
import org.apache.accumulo.core.security.SecurityUtil;
import org.apache.accumulo.core.util.CachedConfiguration;
-import org.apache.accumulo.core.util.MetadataTable;
-import org.apache.accumulo.core.util.RootTable;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
@@ -81,6 +85,7 @@ import com.beust.jcommander.Parameter;
public class Initialize {
private static final Logger log = Logger.getLogger(Initialize.class);
private static final String DEFAULT_ROOT_USER = "root";
+ public static final String TABLE_TABLETS_TABLET_DIR = "/table_info";
private static ConsoleReader reader = null;
@@ -107,10 +112,9 @@ public class Initialize {
initialMetadataConf.put(Property.TABLE_ITERATOR_PREFIX.getKey() + "majc.bulkLoadFilter", "20," + MetadataBulkLoadFilter.class.getName());
initialMetadataConf.put(Property.TABLE_FAILURES_IGNORE.getKey(), "false");
initialMetadataConf.put(Property.TABLE_LOCALITY_GROUP_PREFIX.getKey() + "tablet",
- String.format("%s,%s", MetadataTable.TABLET_COLUMN_FAMILY.toString(), MetadataTable.CURRENT_LOCATION_COLUMN_FAMILY.toString()));
- initialMetadataConf.put(Property.TABLE_LOCALITY_GROUP_PREFIX.getKey() + "server", String.format("%s,%s,%s,%s",
- MetadataTable.DATAFILE_COLUMN_FAMILY.toString(), MetadataTable.LOG_COLUMN_FAMILY.toString(), MetadataTable.SERVER_COLUMN_FAMILY.toString(),
- MetadataTable.FUTURE_LOCATION_COLUMN_FAMILY.toString()));
+ String.format("%s,%s", TabletsSection.TabletColumnFamily.NAME, TabletsSection.CurrentLocationColumnFamily.NAME));
+ initialMetadataConf.put(Property.TABLE_LOCALITY_GROUP_PREFIX.getKey() + "server", String.format("%s,%s,%s,%s", DataFileColumnFamily.NAME,
+ LogColumnFamily.NAME, TabletsSection.ServerColumnFamily.NAME, TabletsSection.FutureLocationColumnFamily.NAME));
initialMetadataConf.put(Property.TABLE_LOCALITY_GROUPS.getKey(), "tablet,server");
initialMetadataConf.put(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY.getKey(), "");
initialMetadataConf.put(Property.TABLE_INDEXCACHE_ENABLED.getKey(), "true");
@@ -229,7 +233,7 @@ public class Initialize {
// the actual disk locations of the metadata table and tablets
final Path[] metadataTableDirs = paths(ServerConstants.getMetadataTableDirs());
- final Path[] tableMetadataTabletDirs = paths(ServerConstants.prefix(ServerConstants.getMetadataTableDirs(), MetadataTable.TABLE_TABLET_LOCATION));
+ final Path[] tableMetadataTabletDirs = paths(ServerConstants.prefix(ServerConstants.getMetadataTableDirs(), TABLE_TABLETS_TABLET_DIR));
final Path[] defaultMetadataTabletDirs = paths(ServerConstants.prefix(ServerConstants.getMetadataTableDirs(), Constants.DEFAULT_TABLET_LOCATION));
fs.mkdirs(new Path(ServerConstants.getDataVersionLocation(), "" + ServerConstants.DATA_VERSION));
@@ -279,35 +283,40 @@ public class Initialize {
FileSKVWriter mfw = FileOperations.getInstance().openWriter(initRootTabFile, ns, ns.getConf(), AccumuloConfiguration.getDefaultConfiguration());
mfw.startDefaultLocalityGroup();
- // ----------] table tablet info
- Text tableExtent = new Text(KeyExtent.getMetadataEntry(new Text(MetadataTable.ID), MetadataTable.RESERVED_RANGE_START_KEY.getRow()));
+ Text tableExtent = new Text(KeyExtent.getMetadataEntry(new Text(MetadataTable.ID), MetadataSchema.TabletsSection.getRange().getEndKey().getRow()));
// table tablet's directory
- Key tableDirKey = new Key(tableExtent, MetadataTable.DIRECTORY_COLUMN.getColumnFamily(), MetadataTable.DIRECTORY_COLUMN.getColumnQualifier(), 0);
- mfw.append(tableDirKey, new Value(MetadataTable.TABLE_TABLET_LOCATION.getBytes()));
+ Key tableDirKey = new Key(tableExtent, TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.getColumnFamily(),
+ TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.getColumnQualifier(), 0);
+ mfw.append(tableDirKey, new Value(TABLE_TABLETS_TABLET_DIR.getBytes()));
// table tablet time
- Key tableTimeKey = new Key(tableExtent, MetadataTable.TIME_COLUMN.getColumnFamily(), MetadataTable.TIME_COLUMN.getColumnQualifier(), 0);
+ Key tableTimeKey = new Key(tableExtent, TabletsSection.ServerColumnFamily.TIME_COLUMN.getColumnFamily(),
+ TabletsSection.ServerColumnFamily.TIME_COLUMN.getColumnQualifier(), 0);
mfw.append(tableTimeKey, new Value((TabletTime.LOGICAL_TIME_ID + "0").getBytes()));
// table tablet's prevrow
- Key tablePrevRowKey = new Key(tableExtent, MetadataTable.PREV_ROW_COLUMN.getColumnFamily(), MetadataTable.PREV_ROW_COLUMN.getColumnQualifier(), 0);
+ Key tablePrevRowKey = new Key(tableExtent, TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.getColumnFamily(),
+ TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.getColumnQualifier(), 0);
mfw.append(tablePrevRowKey, KeyExtent.encodePrevEndRow(null));
// ----------] default tablet info
Text defaultExtent = new Text(KeyExtent.getMetadataEntry(new Text(MetadataTable.ID), null));
// default's directory
- Key defaultDirKey = new Key(defaultExtent, MetadataTable.DIRECTORY_COLUMN.getColumnFamily(), MetadataTable.DIRECTORY_COLUMN.getColumnQualifier(), 0);
+ Key defaultDirKey = new Key(defaultExtent, TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.getColumnFamily(),
+ TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.getColumnQualifier(), 0);
mfw.append(defaultDirKey, new Value(Constants.DEFAULT_TABLET_LOCATION.getBytes()));
// default's time
- Key defaultTimeKey = new Key(defaultExtent, MetadataTable.TIME_COLUMN.getColumnFamily(), MetadataTable.TIME_COLUMN.getColumnQualifier(), 0);
+ Key defaultTimeKey = new Key(defaultExtent, TabletsSection.ServerColumnFamily.TIME_COLUMN.getColumnFamily(),
+ TabletsSection.ServerColumnFamily.TIME_COLUMN.getColumnQualifier(), 0);
mfw.append(defaultTimeKey, new Value((TabletTime.LOGICAL_TIME_ID + "0").getBytes()));
// default's prevrow
- Key defaultPrevRowKey = new Key(defaultExtent, MetadataTable.PREV_ROW_COLUMN.getColumnFamily(), MetadataTable.PREV_ROW_COLUMN.getColumnQualifier(), 0);
- mfw.append(defaultPrevRowKey, KeyExtent.encodePrevEndRow(MetadataTable.RESERVED_RANGE_START_KEY.getRow()));
+ Key defaultPrevRowKey = new Key(defaultExtent, TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.getColumnFamily(),
+ TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.getColumnQualifier(), 0);
+ mfw.append(defaultPrevRowKey, KeyExtent.encodePrevEndRow(MetadataSchema.TabletsSection.getRange().getEndKey().getRow()));
mfw.close();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/server/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java b/server/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java
index 5c9bd64..f7a57d8 100644
--- a/server/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java
@@ -26,8 +26,11 @@ import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.util.MetadataTable;
import org.apache.accumulo.server.cli.ClientOpts;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.fs.VolumeManagerImpl;
@@ -45,9 +48,9 @@ public class LocalityCheck {
VolumeManager fs = VolumeManagerImpl.get();
Connector connector = opts.getConnector();
Scanner scanner = connector.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
- scanner.fetchColumnFamily(MetadataTable.CURRENT_LOCATION_COLUMN_FAMILY);
- scanner.fetchColumnFamily(MetadataTable.DATAFILE_COLUMN_FAMILY);
- scanner.setRange(MetadataTable.KEYSPACE);
+ scanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME);
+ scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+ scanner.setRange(MetadataSchema.TabletsSection.getRange());
Map<String,Long> totalBlocks = new HashMap<String,Long>();
Map<String,Long> localBlocks = new HashMap<String,Long>();
@@ -55,13 +58,13 @@ public class LocalityCheck {
for (Entry<Key,Value> entry : scanner) {
Key key = entry.getKey();
- if (key.compareColumnFamily(MetadataTable.CURRENT_LOCATION_COLUMN_FAMILY) == 0) {
+ if (key.compareColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME) == 0) {
String location = entry.getValue().toString();
String[] parts = location.split(":");
String host = parts[0];
addBlocks(fs, host, files, totalBlocks, localBlocks);
files.clear();
- } else if (key.compareColumnFamily(MetadataTable.DATAFILE_COLUMN_FAMILY) == 0) {
+ } else if (key.compareColumnFamily(DataFileColumnFamily.NAME) == 0) {
files.add(fs.getFullPath(key).toString());
}