You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2014/08/20 07:44:15 UTC
[1/7] HBASE-11742 (Backport HBASE-7987 and HBASE-11185 to 0.98)
(Esteban Gutierrez and Matteo Bertozzi)
Repository: hbase
Updated Branches:
refs/heads/0.98 e26411fc7 -> 2e978a165
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotTask.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotTask.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotTask.java
deleted file mode 100644
index 760ac14..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotTask.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.snapshot;
-
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-
-import org.apache.hadoop.hbase.SmallTests;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.errorhandling.ForeignException;
-import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
-import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import org.mockito.Mockito;
-
-@Category(SmallTests.class)
-public class TestSnapshotTask {
-
- /**
- * Check that errors from running the task get propagated back to the error listener.
- */
- @Test
- public void testErrorPropagation() throws Exception {
- ForeignExceptionDispatcher error = mock(ForeignExceptionDispatcher.class);
- SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("snapshot")
- .setTable("table").build();
- final Exception thrown = new Exception("Failed!");
- SnapshotTask fail = new SnapshotTask(snapshot, error) {
- @Override
- public Void call() {
- snapshotFailure("Injected failure", thrown);
- return null;
- }
- };
- fail.call();
-
- verify(error, Mockito.times(1)).receive(any(ForeignException.class));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestWALReferenceTask.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestWALReferenceTask.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestWALReferenceTask.java
deleted file mode 100644
index c71333b..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestWALReferenceTask.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.snapshot;
-
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.SmallTests;
-import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
-import org.apache.hadoop.hbase.snapshot.ReferenceServerWALsTask;
-import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
-import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import org.mockito.Mockito;
-
-/**
- * Test that the WAL reference task works as expected
- */
-@Category(SmallTests.class)
-public class TestWALReferenceTask {
-
- private static final Log LOG = LogFactory.getLog(TestWALReferenceTask.class);
- private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
-
- @Test
- public void testRun() throws IOException {
- Configuration conf = UTIL.getConfiguration();
- FileSystem fs = UTIL.getTestFileSystem();
- // setup the log dir
- Path testDir = UTIL.getDataTestDir();
- Set<String> servers = new HashSet<String>();
- Path logDir = new Path(testDir, HConstants.HREGION_LOGDIR_NAME);
- Path server1Dir = new Path(logDir, "Server1");
- servers.add(server1Dir.getName());
- Path server2Dir = new Path(logDir, "me.hbase.com,56073,1348618509968");
- servers.add(server2Dir.getName());
- // logs under server 1
- Path log1_1 = new Path(server1Dir, "me.hbase.com%2C56073%2C1348618509968.1348618520536");
- Path log1_2 = new Path(server1Dir, "me.hbase.com%2C56073%2C1348618509968.1234567890123");
- // logs under server 2
- Path log2_1 = new Path(server2Dir, "me.hbase.com%2C56074%2C1348618509998.1348618515589");
- Path log2_2 = new Path(server2Dir, "me.hbase.com%2C56073%2C1348618509968.1234567890123");
-
- // create all the log files
- fs.createNewFile(log1_1);
- fs.createNewFile(log1_2);
- fs.createNewFile(log2_1);
- fs.createNewFile(log2_2);
-
- FSUtils.logFileSystemState(fs, testDir, LOG);
- FSUtils.setRootDir(conf, testDir);
- SnapshotDescription snapshot = SnapshotDescription.newBuilder()
- .setName("testWALReferenceSnapshot").build();
- ForeignExceptionDispatcher listener = Mockito.mock(ForeignExceptionDispatcher.class);
-
- // reference all the files in the first server directory
- ReferenceServerWALsTask task = new ReferenceServerWALsTask(snapshot, listener, server1Dir,
- conf, fs);
- task.call();
-
- // reference all the files in the first server directory
- task = new ReferenceServerWALsTask(snapshot, listener, server2Dir, conf, fs);
- task.call();
-
- // verify that we got everything
- FSUtils.logFileSystemState(fs, testDir, LOG);
- Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, testDir);
- Path snapshotLogDir = new Path(workingDir, HConstants.HREGION_LOGDIR_NAME);
-
- // make sure we reference the all the wal files
- TakeSnapshotUtils.verifyAllLogsGotReferenced(fs, logDir, servers, snapshot, snapshotLogDir);
-
- // make sure we never got an error
- Mockito.verify(listener, Mockito.atLeastOnce()).rethrowException();
- Mockito.verifyNoMoreInteractions(listener);
- }
-}
\ No newline at end of file
[3/7] HBASE-11742 (Backport HBASE-7987 and HBASE-11185 to 0.98)
(Esteban Gutierrez and Matteo Bertozzi)
Posted by ap...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
new file mode 100644
index 0000000..47c6ebf
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
@@ -0,0 +1,468 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.snapshot;
+
+import java.io.IOException;
+import java.io.FileNotFoundException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionSnare;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotDataManifest;
+import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.hadoop.hbase.regionserver.Store;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.apache.hadoop.hbase.util.Threads;
+
+/**
+ * Utility class to help read/write the Snapshot Manifest.
+ *
+ * The snapshot format is transparent for the users of this class,
+ * once the snapshot is written, it will never be modified.
+ * On open() the snapshot will be loaded to the current in-memory format.
+ */
+@InterfaceAudience.Private
+public class SnapshotManifest {
+ private static final Log LOG = LogFactory.getLog(SnapshotManifest.class);
+
+ private static final String DATA_MANIFEST_NAME = "data.manifest";
+
+ private List<SnapshotRegionManifest> regionManifests;
+ private SnapshotDescription desc;
+ private HTableDescriptor htd;
+
+ private final ForeignExceptionSnare monitor;
+ private final Configuration conf;
+ private final Path workingDir;
+ private final FileSystem fs;
+
+ private SnapshotManifest(final Configuration conf, final FileSystem fs,
+ final Path workingDir, final SnapshotDescription desc,
+ final ForeignExceptionSnare monitor) {
+ this.monitor = monitor;
+ this.desc = desc;
+ this.workingDir = workingDir;
+ this.conf = conf;
+ this.fs = fs;
+ }
+
+ /**
+ * Return a SnapshotManifest instance, used for writing a snapshot.
+ *
+ * There are two usage pattern:
+ * - The Master will create a manifest, add the descriptor, offline regions
+ * and consolidate the snapshot by writing all the pending stuff on-disk.
+ * manifest = SnapshotManifest.create(...)
+ * manifest.addRegion(tableDir, hri)
+ * manifest.consolidate()
+ * - The RegionServer will create a single region manifest
+ * manifest = SnapshotManifest.create(...)
+ * manifest.addRegion(region)
+ */
+ public static SnapshotManifest create(final Configuration conf, final FileSystem fs,
+ final Path workingDir, final SnapshotDescription desc,
+ final ForeignExceptionSnare monitor) {
+ return new SnapshotManifest(conf, fs, workingDir, desc, monitor);
+ }
+
+ /**
+ * Return a SnapshotManifest instance with the information already loaded in-memory.
+ * SnapshotManifest manifest = SnapshotManifest.open(...)
+ * HTableDescriptor htd = manifest.getTableDescriptor()
+ * for (SnapshotRegionManifest regionManifest: manifest.getRegionManifests())
+ * hri = regionManifest.getRegionInfo()
+ * for (regionManifest.getFamilyFiles())
+ * ...
+ */
+ public static SnapshotManifest open(final Configuration conf, final FileSystem fs,
+ final Path workingDir, final SnapshotDescription desc) throws IOException {
+ SnapshotManifest manifest = new SnapshotManifest(conf, fs, workingDir, desc, null);
+ manifest.load();
+ return manifest;
+ }
+
+
+ /**
+ * Add the table descriptor to the snapshot manifest
+ */
+ public void addTableDescriptor(final HTableDescriptor htd) throws IOException {
+ this.htd = htd;
+ }
+
+ interface RegionVisitor<TRegion, TFamily> {
+ TRegion regionOpen(final HRegionInfo regionInfo) throws IOException;
+ void regionClose(final TRegion region) throws IOException;
+
+ TFamily familyOpen(final TRegion region, final byte[] familyName) throws IOException;
+ void familyClose(final TRegion region, final TFamily family) throws IOException;
+
+ void storeFile(final TRegion region, final TFamily family, final StoreFileInfo storeFile)
+ throws IOException;
+ }
+
+ private RegionVisitor createRegionVisitor(final SnapshotDescription desc) throws IOException {
+ switch (getSnapshotFormat(desc)) {
+ case SnapshotManifestV1.DESCRIPTOR_VERSION:
+ return new SnapshotManifestV1.ManifestBuilder(conf, fs, workingDir);
+ case SnapshotManifestV2.DESCRIPTOR_VERSION:
+ return new SnapshotManifestV2.ManifestBuilder(conf, fs, workingDir);
+ default:
+ throw new CorruptedSnapshotException("Invalid Snapshot version: "+ desc.getVersion(), desc);
+ }
+ }
+
+ /**
+ * Creates a 'manifest' for the specified region, by reading directly from the HRegion object.
+ * This is used by the "online snapshot" when the table is enabled.
+ */
+ public void addRegion(final HRegion region) throws IOException {
+ // 0. Get the ManifestBuilder/RegionVisitor
+ RegionVisitor visitor = createRegionVisitor(desc);
+
+ // 1. dump region meta info into the snapshot directory
+ LOG.debug("Storing '" + region + "' region-info for snapshot.");
+ Object regionData = visitor.regionOpen(region.getRegionInfo());
+ monitor.rethrowException();
+
+ // 2. iterate through all the stores in the region
+ LOG.debug("Creating references for hfiles");
+
+ for (Store store : region.getStores().values()) {
+ // 2.1. build the snapshot reference for the store
+ Object familyData = visitor.familyOpen(regionData, store.getFamily().getName());
+ monitor.rethrowException();
+
+ List<StoreFile> storeFiles = new ArrayList<StoreFile>(store.getStorefiles());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Adding snapshot references for " + storeFiles + " hfiles");
+ }
+
+ // 2.2. iterate through all the store's files and create "references".
+ for (int i = 0, sz = storeFiles.size(); i < sz; i++) {
+ StoreFile storeFile = storeFiles.get(i);
+ monitor.rethrowException();
+
+ // create "reference" to this store file.
+ LOG.debug("Adding reference for file (" + (i+1) + "/" + sz + "): " + storeFile.getPath());
+ visitor.storeFile(regionData, familyData, storeFile.getFileInfo());
+ }
+ visitor.familyClose(regionData, familyData);
+ }
+ visitor.regionClose(regionData);
+ }
+
+ /**
+ * Creates a 'manifest' for the specified region, by reading directly from the disk.
+ * This is used by the "offline snapshot" when the table is disabled.
+ */
+ public void addRegion(final Path tableDir, final HRegionInfo regionInfo) throws IOException {
+ // 0. Get the ManifestBuilder/RegionVisitor
+ RegionVisitor visitor = createRegionVisitor(desc);
+
+ // Open the RegionFS
+ HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, fs,
+ tableDir, regionInfo, true);
+ monitor.rethrowException();
+
+ // 1. dump region meta info into the snapshot directory
+ LOG.debug("Storing region-info for snapshot.");
+ Object regionData = visitor.regionOpen(regionInfo);
+ monitor.rethrowException();
+
+ // 2. iterate through all the stores in the region
+ LOG.debug("Creating references for hfiles");
+
+ // This ensures that we have an atomic view of the directory as long as we have < ls limit
+ // (batch size of the files in a directory) on the namenode. Otherwise, we get back the files in
+ // batches and may miss files being added/deleted. This could be more robust (iteratively
+ // checking to see if we have all the files until we are sure), but the limit is currently 1000
+ // files/batch, far more than the number of store files under a single column family.
+ Collection<String> familyNames = regionFs.getFamilies();
+ if (familyNames != null) {
+ for (String familyName: familyNames) {
+ Object familyData = visitor.familyOpen(regionData, Bytes.toBytes(familyName));
+ monitor.rethrowException();
+
+ Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(familyName);
+ if (storeFiles == null) {
+ LOG.debug("No files under family: " + familyName);
+ continue;
+ }
+
+ // 2.1. build the snapshot reference for the store
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Adding snapshot references for " + storeFiles + " hfiles");
+ }
+
+ // 2.2. iterate through all the store's files and create "references".
+ int i = 0;
+ int sz = storeFiles.size();
+ for (StoreFileInfo storeFile: storeFiles) {
+ monitor.rethrowException();
+
+ // create "reference" to this store file.
+ LOG.debug("Adding reference for file ("+ (++i) +"/" + sz + "): " + storeFile.getPath());
+ visitor.storeFile(regionData, familyData, storeFile);
+ }
+ visitor.familyClose(regionData, familyData);
+ }
+ }
+ visitor.regionClose(regionData);
+ }
+
+ /**
+ * Load the information in the SnapshotManifest. Called by SnapshotManifest.open()
+ *
+ * If the format is v2 and there is no data-manifest, means that we are loading an
+ * in-progress snapshot. Since we support rolling-upgrades, we loook for v1 and v2
+ * regions format.
+ */
+ private void load() throws IOException {
+ switch (getSnapshotFormat(desc)) {
+ case SnapshotManifestV1.DESCRIPTOR_VERSION: {
+ this.htd = FSTableDescriptors.getTableDescriptorFromFs(fs, workingDir);
+ ThreadPoolExecutor tpool = createExecutor("SnapshotManifestLoader");
+ try {
+ this.regionManifests =
+ SnapshotManifestV1.loadRegionManifests(conf, tpool, fs, workingDir, desc);
+ } finally {
+ tpool.shutdown();
+ }
+ break;
+ }
+ case SnapshotManifestV2.DESCRIPTOR_VERSION: {
+ SnapshotDataManifest dataManifest = readDataManifest();
+ if (dataManifest != null) {
+ htd = HTableDescriptor.convert(dataManifest.getTableSchema());
+ regionManifests = dataManifest.getRegionManifestsList();
+ } else {
+ // Compatibility, load the v1 regions
+ // This happens only when the snapshot is in-progress and the cache wants to refresh.
+ List<SnapshotRegionManifest> v1Regions, v2Regions;
+ ThreadPoolExecutor tpool = createExecutor("SnapshotManifestLoader");
+ try {
+ v1Regions = SnapshotManifestV1.loadRegionManifests(conf, tpool, fs, workingDir, desc);
+ v2Regions = SnapshotManifestV2.loadRegionManifests(conf, tpool, fs, workingDir, desc);
+ } finally {
+ tpool.shutdown();
+ }
+ if (v1Regions != null && v2Regions != null) {
+ regionManifests =
+ new ArrayList<SnapshotRegionManifest>(v1Regions.size() + v2Regions.size());
+ regionManifests.addAll(v1Regions);
+ regionManifests.addAll(v2Regions);
+ } else if (v1Regions != null) {
+ regionManifests = v1Regions;
+ } else /* if (v2Regions != null) */ {
+ regionManifests = v2Regions;
+ }
+ }
+ break;
+ }
+ default:
+ throw new CorruptedSnapshotException("Invalid Snapshot version: "+ desc.getVersion(), desc);
+ }
+ }
+
+ /**
+ * Get the current snapshot working dir
+ */
+ public Path getSnapshotDir() {
+ return this.workingDir;
+ }
+
+ /**
+ * Get the SnapshotDescription
+ */
+ public SnapshotDescription getSnapshotDescription() {
+ return this.desc;
+ }
+
+ /**
+ * Get the table descriptor from the Snapshot
+ */
+ public HTableDescriptor getTableDescriptor() {
+ return this.htd;
+ }
+
+ /**
+ * Get all the Region Manifest from the snapshot
+ */
+ public List<SnapshotRegionManifest> getRegionManifests() {
+ return this.regionManifests;
+ }
+
+ /**
+ * Get all the Region Manifest from the snapshot.
+ * This is an helper to get a map with the region encoded name
+ */
+ public Map<String, SnapshotRegionManifest> getRegionManifestsMap() {
+ if (regionManifests == null || regionManifests.size() == 0) return null;
+
+ HashMap<String, SnapshotRegionManifest> regionsMap =
+ new HashMap<String, SnapshotRegionManifest>(regionManifests.size());
+ for (SnapshotRegionManifest manifest: regionManifests) {
+ String regionName = getRegionNameFromManifest(manifest);
+ regionsMap.put(regionName, manifest);
+ }
+ return regionsMap;
+ }
+
+ public void consolidate() throws IOException {
+ if (getSnapshotFormat(desc) == SnapshotManifestV1.DESCRIPTOR_VERSION) {
+ Path rootDir = FSUtils.getRootDir(conf);
+ LOG.info("Using old Snapshot Format");
+ // write a copy of descriptor to the snapshot directory
+ new FSTableDescriptors(fs, rootDir)
+ .createTableDescriptorForTableDirectory(workingDir, htd, false);
+ } else {
+ LOG.debug("Convert to Single Snapshot Manifest");
+ convertToV2SingleManifest();
+ }
+ }
+
+ /*
+ * In case of rolling-upgrade, we try to read all the formats and build
+ * the snapshot with the latest format.
+ */
+ private void convertToV2SingleManifest() throws IOException {
+ // Try to load v1 and v2 regions
+ List<SnapshotRegionManifest> v1Regions, v2Regions;
+ ThreadPoolExecutor tpool = createExecutor("SnapshotManifestLoader");
+ try {
+ v1Regions = SnapshotManifestV1.loadRegionManifests(conf, tpool, fs, workingDir, desc);
+ v2Regions = SnapshotManifestV2.loadRegionManifests(conf, tpool, fs, workingDir, desc);
+ } finally {
+ tpool.shutdown();
+ }
+
+ SnapshotDataManifest.Builder dataManifestBuilder = SnapshotDataManifest.newBuilder();
+ dataManifestBuilder.setTableSchema(htd.convert());
+
+ if (v1Regions != null && v1Regions.size() > 0) {
+ dataManifestBuilder.addAllRegionManifests(v1Regions);
+ }
+ if (v2Regions != null && v2Regions.size() > 0) {
+ dataManifestBuilder.addAllRegionManifests(v2Regions);
+ }
+
+ // Write the v2 Data Manifest.
+ // Once the data-manifest is written, the snapshot can be considered complete.
+ // Currently snapshots are written in a "temporary" directory and later
+ // moved to the "complated" snapshot directory.
+ SnapshotDataManifest dataManifest = dataManifestBuilder.build();
+ writeDataManifest(dataManifest);
+ this.regionManifests = dataManifest.getRegionManifestsList();
+
+ // Remove the region manifests. Everything is now in the data-manifest.
+ // The delete operation is "relaxed", unless we get an exception we keep going.
+ // The extra files in the snapshot directory will not give any problem,
+ // since they have the same content as the data manifest, and even by re-reading
+ // them we will get the same information.
+ if (v1Regions != null && v1Regions.size() > 0) {
+ for (SnapshotRegionManifest regionManifest: v1Regions) {
+ SnapshotManifestV1.deleteRegionManifest(fs, workingDir, regionManifest);
+ }
+ }
+ if (v2Regions != null && v2Regions.size() > 0) {
+ for (SnapshotRegionManifest regionManifest: v2Regions) {
+ SnapshotManifestV2.deleteRegionManifest(fs, workingDir, regionManifest);
+ }
+ }
+ }
+
+ /*
+ * Write the SnapshotDataManifest file
+ */
+ private void writeDataManifest(final SnapshotDataManifest manifest)
+ throws IOException {
+ FSDataOutputStream stream = fs.create(new Path(workingDir, DATA_MANIFEST_NAME));
+ try {
+ manifest.writeTo(stream);
+ } finally {
+ stream.close();
+ }
+ }
+
+ /*
+ * Read the SnapshotDataManifest file
+ */
+ private SnapshotDataManifest readDataManifest() throws IOException {
+ FSDataInputStream in = null;
+ try {
+ in = fs.open(new Path(workingDir, DATA_MANIFEST_NAME));
+ return SnapshotDataManifest.parseFrom(in);
+ } catch (FileNotFoundException e) {
+ return null;
+ } finally {
+ if (in != null) in.close();
+ }
+ }
+
+ private ThreadPoolExecutor createExecutor(final String name) {
+ return createExecutor(conf, name);
+ }
+
+ public static ThreadPoolExecutor createExecutor(final Configuration conf, final String name) {
+ int maxThreads = conf.getInt("hbase.snapshot.thread.pool.max", 8);
+ return Threads.getBoundedCachedThreadPool(maxThreads, 30L, TimeUnit.SECONDS,
+ Threads.getNamedThreadFactory(name));
+ }
+
+ /**
+ * Extract the region encoded name from the region manifest
+ */
+ static String getRegionNameFromManifest(final SnapshotRegionManifest manifest) {
+ byte[] regionName = HRegionInfo.createRegionName(
+ ProtobufUtil.toTableName(manifest.getRegionInfo().getTableName()),
+ manifest.getRegionInfo().getStartKey().toByteArray(),
+ manifest.getRegionInfo().getRegionId(), true);
+ return HRegionInfo.encodeRegionName(regionName);
+ }
+
+ /*
+ * Return the snapshot format
+ */
+ private static int getSnapshotFormat(final SnapshotDescription desc) {
+ return desc.hasVersion() ? desc.getVersion() : SnapshotManifestV1.DESCRIPTOR_VERSION;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java
new file mode 100644
index 0000000..0da0367
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java
@@ -0,0 +1,210 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.snapshot;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
+import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+import com.google.protobuf.HBaseZeroCopyByteString;
+
+/**
+ * DO NOT USE DIRECTLY. USE {@link SnapshotManifest}.
+ *
+ * Snapshot v1 layout format
+ * - Each region in the table is represented by a directory with the .hregioninfo file
+ * /snapshotName/regionName/.hregioninfo
+ * - Each file present in the table is represented by an empty file
+ * /snapshotName/regionName/familyName/fileName
+ */
+@InterfaceAudience.Private
+public class SnapshotManifestV1 {
+ private static final Log LOG = LogFactory.getLog(SnapshotManifestV1.class);
+
+ public static final int DESCRIPTOR_VERSION = 0;
+
+ private SnapshotManifestV1() {
+ }
+
+ static class ManifestBuilder implements SnapshotManifest.RegionVisitor<
+ HRegionFileSystem, Path> {
+ private final Configuration conf;
+ private final Path snapshotDir;
+ private final FileSystem fs;
+
+ public ManifestBuilder(final Configuration conf, final FileSystem fs, final Path snapshotDir) {
+ this.snapshotDir = snapshotDir;
+ this.conf = conf;
+ this.fs = fs;
+ }
+
+ public HRegionFileSystem regionOpen(final HRegionInfo regionInfo) throws IOException {
+ HRegionFileSystem snapshotRegionFs = HRegionFileSystem.createRegionOnFileSystem(conf,
+ fs, snapshotDir, regionInfo);
+ return snapshotRegionFs;
+ }
+
+ public void regionClose(final HRegionFileSystem region) {
+ }
+
+ public Path familyOpen(final HRegionFileSystem snapshotRegionFs, final byte[] familyName) {
+ Path familyDir = snapshotRegionFs.getStoreDir(Bytes.toString(familyName));
+ return familyDir;
+ }
+
+ public void familyClose(final HRegionFileSystem region, final Path family) {
+ }
+
+ public void storeFile(final HRegionFileSystem region, final Path familyDir,
+ final StoreFileInfo storeFile) throws IOException {
+ Path referenceFile = new Path(familyDir, storeFile.getPath().getName());
+ boolean success = true;
+ if (storeFile.isReference()) {
+ // write the Reference object to the snapshot
+ storeFile.getReference().write(fs, referenceFile);
+ } else {
+ // create "reference" to this store file. It is intentionally an empty file -- all
+ // necessary information is captured by its fs location and filename. This allows us to
+ // only figure out what needs to be done via a single nn operation (instead of having to
+ // open and read the files as well).
+ success = fs.createNewFile(referenceFile);
+ }
+ if (!success) {
+ throw new IOException("Failed to create reference file:" + referenceFile);
+ }
+ }
+ }
+
+ static List<SnapshotRegionManifest> loadRegionManifests(final Configuration conf,
+ final Executor executor,final FileSystem fs, final Path snapshotDir,
+ final SnapshotDescription desc) throws IOException {
+ FileStatus[] regions = FSUtils.listStatus(fs, snapshotDir, new FSUtils.RegionDirFilter(fs));
+ if (regions == null) {
+ LOG.info("No regions under directory:" + snapshotDir);
+ return null;
+ }
+
+ final ExecutorCompletionService<SnapshotRegionManifest> completionService =
+ new ExecutorCompletionService<SnapshotRegionManifest>(executor);
+ for (final FileStatus region: regions) {
+ completionService.submit(new Callable<SnapshotRegionManifest>() {
+ @Override
+ public SnapshotRegionManifest call() throws IOException {
+ HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, region.getPath());
+ return buildManifestFromDisk(conf, fs, snapshotDir, hri);
+ }
+ });
+ }
+
+ ArrayList<SnapshotRegionManifest> regionsManifest =
+ new ArrayList<SnapshotRegionManifest>(regions.length);
+ try {
+ for (int i = 0; i < regions.length; ++i) {
+ regionsManifest.add(completionService.take().get());
+ }
+ } catch (InterruptedException e) {
+ throw new InterruptedIOException(e.getMessage());
+ } catch (ExecutionException e) {
+ IOException ex = new IOException();
+ ex.initCause(e.getCause());
+ throw ex;
+ }
+ return regionsManifest;
+ }
+
+ static void deleteRegionManifest(final FileSystem fs, final Path snapshotDir,
+ final SnapshotRegionManifest manifest) throws IOException {
+ String regionName = SnapshotManifest.getRegionNameFromManifest(manifest);
+ fs.delete(new Path(snapshotDir, regionName), true);
+ }
+
+ static SnapshotRegionManifest buildManifestFromDisk (final Configuration conf,
+ final FileSystem fs, final Path tableDir, final HRegionInfo regionInfo) throws IOException {
+ HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, fs,
+ tableDir, regionInfo, true);
+ SnapshotRegionManifest.Builder manifest = SnapshotRegionManifest.newBuilder();
+
+ // 1. dump region meta info into the snapshot directory
+ LOG.debug("Storing region-info for snapshot.");
+ manifest.setRegionInfo(HRegionInfo.convert(regionInfo));
+
+ // 2. iterate through all the stores in the region
+ LOG.debug("Creating references for hfiles");
+
+ // This ensures that we have an atomic view of the directory as long as we have < ls limit
+ // (batch size of the files in a directory) on the namenode. Otherwise, we get back the files in
+ // batches and may miss files being added/deleted. This could be more robust (iteratively
+ // checking to see if we have all the files until we are sure), but the limit is currently 1000
+ // files/batch, far more than the number of store files under a single column family.
+ Collection<String> familyNames = regionFs.getFamilies();
+ if (familyNames != null) {
+ for (String familyName: familyNames) {
+ Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(familyName, false);
+ if (storeFiles == null) {
+ LOG.debug("No files under family: " + familyName);
+ continue;
+ }
+
+ // 2.1. build the snapshot reference for the store
+ SnapshotRegionManifest.FamilyFiles.Builder family =
+ SnapshotRegionManifest.FamilyFiles.newBuilder();
+ family.setFamilyName(HBaseZeroCopyByteString.wrap(Bytes.toBytes(familyName)));
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Adding snapshot references for " + storeFiles + " hfiles");
+ }
+
+ // 2.2. iterate through all the store's files and create "references".
+ int i = 0;
+ int sz = storeFiles.size();
+ for (StoreFileInfo storeFile: storeFiles) {
+ // create "reference" to this store file.
+ LOG.debug("Adding reference for file ("+ (++i) +"/" + sz + "): " + storeFile.getPath());
+ SnapshotRegionManifest.StoreFile.Builder sfManifest =
+ SnapshotRegionManifest.StoreFile.newBuilder();
+ sfManifest.setName(storeFile.getPath().getName());
+ family.addStoreFiles(sfManifest.build());
+ }
+ manifest.addFamilyFiles(family.build());
+ }
+ }
+ return manifest.build();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV2.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV2.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV2.java
new file mode 100644
index 0000000..585c454
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV2.java
@@ -0,0 +1,173 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.snapshot;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+import com.google.protobuf.HBaseZeroCopyByteString;
+
+/**
+ * DO NOT USE DIRECTLY. USE {@link SnapshotManifest}.
+ *
+ * Snapshot v2 layout format
+ * - Single Manifest file containing all the information of regions
+ * - In the online-snapshot case each region will write a "region manifest"
+ * /snapshotName/manifest.regionName
+ */
+@InterfaceAudience.Private
+public class SnapshotManifestV2 {
+ private static final Log LOG = LogFactory.getLog(SnapshotManifestV2.class);
+
+ public static final int DESCRIPTOR_VERSION = 2;
+
+ private static final String SNAPSHOT_MANIFEST_PREFIX = "region-manifest.";
+
+ static class ManifestBuilder implements SnapshotManifest.RegionVisitor<
+ SnapshotRegionManifest.Builder, SnapshotRegionManifest.FamilyFiles.Builder> {
+ private final Configuration conf;
+ private final Path snapshotDir;
+ private final FileSystem fs;
+
+ public ManifestBuilder(final Configuration conf, final FileSystem fs, final Path snapshotDir) {
+ this.snapshotDir = snapshotDir;
+ this.conf = conf;
+ this.fs = fs;
+ }
+
+ public SnapshotRegionManifest.Builder regionOpen(final HRegionInfo regionInfo) {
+ SnapshotRegionManifest.Builder manifest = SnapshotRegionManifest.newBuilder();
+ manifest.setRegionInfo(HRegionInfo.convert(regionInfo));
+ return manifest;
+ }
+
+ public void regionClose(final SnapshotRegionManifest.Builder region) throws IOException {
+ SnapshotRegionManifest manifest = region.build();
+ FSDataOutputStream stream = fs.create(getRegionManifestPath(snapshotDir, manifest));
+ try {
+ manifest.writeTo(stream);
+ } finally {
+ stream.close();
+ }
+ }
+
+ public SnapshotRegionManifest.FamilyFiles.Builder familyOpen(
+ final SnapshotRegionManifest.Builder region, final byte[] familyName) {
+ SnapshotRegionManifest.FamilyFiles.Builder family =
+ SnapshotRegionManifest.FamilyFiles.newBuilder();
+ family.setFamilyName(HBaseZeroCopyByteString.wrap(familyName));
+ return family;
+ }
+
+ public void familyClose(final SnapshotRegionManifest.Builder region,
+ final SnapshotRegionManifest.FamilyFiles.Builder family) {
+ region.addFamilyFiles(family.build());
+ }
+
+ public void storeFile(final SnapshotRegionManifest.Builder region,
+ final SnapshotRegionManifest.FamilyFiles.Builder family, final StoreFileInfo storeFile)
+ throws IOException {
+ SnapshotRegionManifest.StoreFile.Builder sfManifest =
+ SnapshotRegionManifest.StoreFile.newBuilder();
+ sfManifest.setName(storeFile.getPath().getName());
+ if (storeFile.isReference()) {
+ sfManifest.setReference(storeFile.getReference().convert());
+ }
+ sfManifest.setFileSize(storeFile.getReferencedFileStatus(fs).getLen());
+ family.addStoreFiles(sfManifest.build());
+ }
+ }
+
+ static List<SnapshotRegionManifest> loadRegionManifests(final Configuration conf,
+ final Executor executor,final FileSystem fs, final Path snapshotDir,
+ final SnapshotDescription desc) throws IOException {
+ FileStatus[] manifestFiles = FSUtils.listStatus(fs, snapshotDir, new PathFilter() {
+ @Override
+ public boolean accept(Path path) {
+ return path.getName().startsWith(SNAPSHOT_MANIFEST_PREFIX);
+ }
+ });
+
+ if (manifestFiles == null || manifestFiles.length == 0) return null;
+
+ final ExecutorCompletionService<SnapshotRegionManifest> completionService =
+ new ExecutorCompletionService<SnapshotRegionManifest>(executor);
+ for (final FileStatus st: manifestFiles) {
+ completionService.submit(new Callable<SnapshotRegionManifest>() {
+ @Override
+ public SnapshotRegionManifest call() throws IOException {
+ FSDataInputStream stream = fs.open(st.getPath());
+ try {
+ return SnapshotRegionManifest.parseFrom(stream);
+ } finally {
+ stream.close();
+ }
+ }
+ });
+ }
+
+ ArrayList<SnapshotRegionManifest> regionsManifest =
+ new ArrayList<SnapshotRegionManifest>(manifestFiles.length);
+ try {
+ for (int i = 0; i < manifestFiles.length; ++i) {
+ regionsManifest.add(completionService.take().get());
+ }
+ } catch (InterruptedException e) {
+ throw new InterruptedIOException(e.getMessage());
+ } catch (ExecutionException e) {
+ IOException ex = new IOException();
+ ex.initCause(e.getCause());
+ throw ex;
+ }
+ return regionsManifest;
+ }
+
+ static void deleteRegionManifest(final FileSystem fs, final Path snapshotDir,
+ final SnapshotRegionManifest manifest) throws IOException {
+ fs.delete(getRegionManifestPath(snapshotDir, manifest), true);
+ }
+
+ private static Path getRegionManifestPath(final Path snapshotDir,
+ final SnapshotRegionManifest manifest) {
+ String regionName = SnapshotManifest.getRegionNameFromManifest(manifest);
+ return new Path(snapshotDir, SNAPSHOT_MANIFEST_PREFIX + regionName);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java
index dc977bb..c8d0e5c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java
@@ -18,25 +18,31 @@
package org.apache.hadoop.hbase.snapshot;
-import java.io.IOException;
import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InterruptedIOException;
import java.util.HashSet;
-import java.util.LinkedList;
import java.util.List;
-import java.util.Map;
import java.util.Set;
-import java.util.TreeMap;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.HFileLink;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
-import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
import org.apache.hadoop.hbase.util.FSVisitor;
/**
@@ -44,8 +50,15 @@ import org.apache.hadoop.hbase.util.FSVisitor;
*/
@InterfaceAudience.Private
public final class SnapshotReferenceUtil {
- public interface FileVisitor extends FSVisitor.StoreFileVisitor,
- FSVisitor.RecoveredEditsVisitor, FSVisitor.LogFileVisitor {
+ public static final Log LOG = LogFactory.getLog(SnapshotReferenceUtil.class);
+
+ public interface StoreFileVisitor {
+ void storeFile(final HRegionInfo regionInfo, final String familyName,
+ final SnapshotRegionManifest.StoreFile storeFile) throws IOException;
+ }
+
+ public interface SnapshotVisitor extends StoreFileVisitor,
+ FSVisitor.LogFileVisitor {
}
private SnapshotReferenceUtil() {
@@ -64,80 +77,79 @@ public final class SnapshotReferenceUtil {
}
/**
- * Get the snapshotted recovered.edits dir for the specified region.
- *
- * @param snapshotDir directory where the specific snapshot is stored
- * @param regionName name of the region
- * @return path to the recovered.edits directory for the specified region files.
- */
- public static Path getRecoveredEditsDir(Path snapshotDir, String regionName) {
- return HLogUtil.getRegionDirRecoveredEditsDir(new Path(snapshotDir, regionName));
- }
-
- /**
- * Get the snapshot recovered.edits file
+ * Iterate over the snapshot store files, restored.edits and logs
*
- * @param snapshotDir directory where the specific snapshot is stored
- * @param regionName name of the region
- * @param logfile name of the edit file
- * @return full path of the log file for the specified region files.
+ * @param conf The current {@link Configuration} instance.
+ * @param fs {@link FileSystem}
+ * @param snapshotDir {@link Path} to the Snapshot directory
+ * @param visitor callback object to get the referenced files
+ * @throws IOException if an error occurred while scanning the directory
*/
- public static Path getRecoveredEdits(Path snapshotDir, String regionName, String logfile) {
- return new Path(getRecoveredEditsDir(snapshotDir, regionName), logfile);
+ public static void visitReferencedFiles(final Configuration conf, final FileSystem fs,
+ final Path snapshotDir, final SnapshotVisitor visitor)
+ throws IOException {
+ SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
+ visitReferencedFiles(conf, fs, snapshotDir, desc, visitor);
}
/**
* Iterate over the snapshot store files, restored.edits and logs
*
+ * @param conf The current {@link Configuration} instance.
* @param fs {@link FileSystem}
* @param snapshotDir {@link Path} to the Snapshot directory
+ * @param desc the {@link SnapshotDescription} of the snapshot to verify
* @param visitor callback object to get the referenced files
* @throws IOException if an error occurred while scanning the directory
*/
- public static void visitReferencedFiles(final FileSystem fs, final Path snapshotDir,
- final FileVisitor visitor) throws IOException {
- visitTableStoreFiles(fs, snapshotDir, visitor);
- visitRecoveredEdits(fs, snapshotDir, visitor);
+ public static void visitReferencedFiles(final Configuration conf, final FileSystem fs,
+ final Path snapshotDir, final SnapshotDescription desc, final SnapshotVisitor visitor)
+ throws IOException {
+ visitTableStoreFiles(conf, fs, snapshotDir, desc, visitor);
visitLogFiles(fs, snapshotDir, visitor);
}
- /**
+ /**©
* Iterate over the snapshot store files
*
+ * @param conf The current {@link Configuration} instance.
* @param fs {@link FileSystem}
* @param snapshotDir {@link Path} to the Snapshot directory
+ * @param desc the {@link SnapshotDescription} of the snapshot to verify
* @param visitor callback object to get the store files
* @throws IOException if an error occurred while scanning the directory
*/
- public static void visitTableStoreFiles(final FileSystem fs, final Path snapshotDir,
- final FSVisitor.StoreFileVisitor visitor) throws IOException {
- FSVisitor.visitTableStoreFiles(fs, snapshotDir, visitor);
+ static void visitTableStoreFiles(final Configuration conf, final FileSystem fs,
+ final Path snapshotDir, final SnapshotDescription desc, final StoreFileVisitor visitor)
+ throws IOException {
+ SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, desc);
+ List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests();
+ if (regionManifests == null || regionManifests.size() == 0) {
+ LOG.debug("No manifest files present: " + snapshotDir);
+ return;
+ }
+
+ for (SnapshotRegionManifest regionManifest: regionManifests) {
+ visitRegionStoreFiles(regionManifest, visitor);
+ }
}
/**
* Iterate over the snapshot store files in the specified region
*
- * @param fs {@link FileSystem}
- * @param regionDir {@link Path} to the Snapshot region directory
+ * @param manifest snapshot manifest to inspect
* @param visitor callback object to get the store files
* @throws IOException if an error occurred while scanning the directory
*/
- public static void visitRegionStoreFiles(final FileSystem fs, final Path regionDir,
- final FSVisitor.StoreFileVisitor visitor) throws IOException {
- FSVisitor.visitRegionStoreFiles(fs, regionDir, visitor);
- }
-
- /**
- * Iterate over the snapshot recovered.edits
- *
- * @param fs {@link FileSystem}
- * @param snapshotDir {@link Path} to the Snapshot directory
- * @param visitor callback object to get the recovered.edits files
- * @throws IOException if an error occurred while scanning the directory
- */
- public static void visitRecoveredEdits(final FileSystem fs, final Path snapshotDir,
- final FSVisitor.RecoveredEditsVisitor visitor) throws IOException {
- FSVisitor.visitTableRecoveredEdits(fs, snapshotDir, visitor);
+ static void visitRegionStoreFiles(final SnapshotRegionManifest manifest,
+ final StoreFileVisitor visitor) throws IOException {
+ HRegionInfo regionInfo = HRegionInfo.convert(manifest.getRegionInfo());
+ for (SnapshotRegionManifest.FamilyFiles familyFiles: manifest.getFamilyFilesList()) {
+ String familyName = familyFiles.getFamilyName().toStringUtf8();
+ for (SnapshotRegionManifest.StoreFile storeFile: familyFiles.getStoreFilesList()) {
+ visitor.storeFile(regionInfo, familyName, storeFile);
+ }
+ }
}
/**
@@ -165,85 +177,177 @@ public final class SnapshotReferenceUtil {
*/
public static void verifySnapshot(final Configuration conf, final FileSystem fs,
final Path snapshotDir, final SnapshotDescription snapshotDesc) throws IOException {
- final TableName table = TableName.valueOf(snapshotDesc.getTable());
- visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() {
- public void storeFile (final String region, final String family, final String hfile)
- throws IOException {
- HFileLink link = HFileLink.create(conf, table, region, family, hfile);
- try {
- link.getFileStatus(fs);
- } catch (FileNotFoundException e) {
- throw new CorruptedSnapshotException("Corrupted snapshot '" + snapshotDesc + "'", e);
- }
- }
- });
+ SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
+ verifySnapshot(conf, fs, manifest);
}
/**
- * Returns the set of region names available in the snapshot.
+ * Verify the validity of the snapshot
*
+ * @param conf The current {@link Configuration} instance.
* @param fs {@link FileSystem}
- * @param snapshotDir {@link Path} to the Snapshot directory
+ * @param manifest snapshot manifest to inspect
+ * @throws CorruptedSnapshotException if the snapshot is corrupted
* @throws IOException if an error occurred while scanning the directory
- * @return the set of the regions contained in the snapshot
*/
- public static Set<String> getSnapshotRegionNames(final FileSystem fs, final Path snapshotDir)
- throws IOException {
- FileStatus[] regionDirs = FSUtils.listStatus(fs, snapshotDir, new FSUtils.RegionDirFilter(fs));
- if (regionDirs == null) return null;
+ public static void verifySnapshot(final Configuration conf, final FileSystem fs,
+ final SnapshotManifest manifest) throws IOException {
+ final SnapshotDescription snapshotDesc = manifest.getSnapshotDescription();
+ final Path snapshotDir = manifest.getSnapshotDir();
+ concurrentVisitReferencedFiles(conf, fs, manifest, new StoreFileVisitor() {
+ @Override
+ public void storeFile(final HRegionInfo regionInfo, final String family,
+ final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+ verifyStoreFile(conf, fs, snapshotDir, snapshotDesc, regionInfo, family, storeFile);
+ }
+ });
+ }
+
+ public static void concurrentVisitReferencedFiles(final Configuration conf, final FileSystem fs,
+ final SnapshotManifest manifest, final StoreFileVisitor visitor) throws IOException {
+ final SnapshotDescription snapshotDesc = manifest.getSnapshotDescription();
+ final Path snapshotDir = manifest.getSnapshotDir();
- Set<String> regions = new HashSet<String>();
- for (FileStatus regionDir: regionDirs) {
- regions.add(regionDir.getPath().getName());
+ List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests();
+ if (regionManifests == null || regionManifests.size() == 0) {
+ LOG.debug("No manifest files present: " + snapshotDir);
+ return;
+ }
+
+ ExecutorService exec = SnapshotManifest.createExecutor(conf, "VerifySnapshot");
+ final ExecutorCompletionService<Void> completionService =
+ new ExecutorCompletionService<Void>(exec);
+ try {
+ for (final SnapshotRegionManifest regionManifest: regionManifests) {
+ completionService.submit(new Callable<Void>() {
+ @Override
+ public Void call() throws IOException {
+ visitRegionStoreFiles(regionManifest, visitor);
+ return null;
+ }
+ });
+ }
+ try {
+ for (int i = 0; i < regionManifests.size(); ++i) {
+ completionService.take().get();
+ }
+ } catch (InterruptedException e) {
+ throw new InterruptedIOException(e.getMessage());
+ } catch (ExecutionException e) {
+ if (e.getCause() instanceof CorruptedSnapshotException) {
+ throw new CorruptedSnapshotException(e.getCause().getMessage(), snapshotDesc);
+ } else {
+ IOException ex = new IOException();
+ ex.initCause(e.getCause());
+ throw ex;
+ }
+ }
+ } finally {
+ exec.shutdown();
}
- return regions;
}
/**
- * Get the list of hfiles for the specified snapshot region.
- * NOTE: The current implementation keeps one empty file per HFile in the region.
- * The file name matches the one in the original table, and by reconstructing
- * the path you can quickly jump to the referenced file.
+ * Verify the validity of the snapshot store file
*
+ * @param conf The current {@link Configuration} instance.
* @param fs {@link FileSystem}
- * @param snapshotRegionDir {@link Path} to the Snapshot region directory
- * @return Map of hfiles per family, the key is the family name and values are hfile names
+ * @param snapshotDir {@link Path} to the Snapshot directory of the snapshot to verify
+ * @param snapshot the {@link SnapshotDescription} of the snapshot to verify
+ * @param regionInfo {@link HRegionInfo} of the region that contains the store file
+ * @param family family that contains the store file
+ * @param storeFile the store file to verify
+ * @throws CorruptedSnapshotException if the snapshot is corrupted
* @throws IOException if an error occurred while scanning the directory
*/
- public static Map<String, List<String>> getRegionHFileReferences(final FileSystem fs,
- final Path snapshotRegionDir) throws IOException {
- final Map<String, List<String>> familyFiles = new TreeMap<String, List<String>>();
+ private static void verifyStoreFile(final Configuration conf, final FileSystem fs,
+ final Path snapshotDir, final SnapshotDescription snapshot, final HRegionInfo regionInfo,
+ final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+ TableName table = TableName.valueOf(snapshot.getTable());
+ String fileName = storeFile.getName();
- visitRegionStoreFiles(fs, snapshotRegionDir,
- new FSVisitor.StoreFileVisitor() {
- public void storeFile (final String region, final String family, final String hfile)
- throws IOException {
- List<String> hfiles = familyFiles.get(family);
- if (hfiles == null) {
- hfiles = new LinkedList<String>();
- familyFiles.put(family, hfiles);
- }
- hfiles.add(hfile);
- }
- });
+ Path refPath = null;
+ if (StoreFileInfo.isReference(fileName)) {
+ // If is a reference file check if the parent file is present in the snapshot
+ refPath = new Path(new Path(regionInfo.getEncodedName(), family), fileName);
+ refPath = StoreFileInfo.getReferredToFile(refPath);
+ String refRegion = refPath.getParent().getParent().getName();
+ refPath = HFileLink.createPath(table, refRegion, family, refPath.getName());
+ if (!new HFileLink(conf, refPath).exists(fs)) {
+ throw new CorruptedSnapshotException("Missing parent hfile for: " + fileName +
+ " path=" + refPath, snapshot);
+ }
+
+ if (storeFile.hasReference()) {
+ // We don't really need to look for the file on-disk
+ // we already have the Reference information embedded here.
+ return;
+ }
+ }
- return familyFiles;
+ Path linkPath;
+ if (refPath != null && HFileLink.isHFileLink(refPath)) {
+ linkPath = new Path(family, refPath.getName());
+ } else if (HFileLink.isHFileLink(fileName)) {
+ linkPath = new Path(family, fileName);
+ } else {
+ linkPath = new Path(family, HFileLink.createHFileLinkName(
+ table, regionInfo.getEncodedName(), fileName));
+ }
+
+ // check if the linked file exists (in the archive, or in the table dir)
+ HFileLink link = new HFileLink(conf, linkPath);
+ try {
+ FileStatus fstat = link.getFileStatus(fs);
+ if (storeFile.hasFileSize() && storeFile.getFileSize() != fstat.getLen()) {
+ String msg = "hfile: " + fileName + " size does not match with the expected one. " +
+ " found=" + fstat.getLen() + " expected=" + storeFile.getFileSize();
+ LOG.error(msg);
+ throw new CorruptedSnapshotException(msg, snapshot);
+ }
+ } catch (FileNotFoundException e) {
+ String msg = "Can't find hfile: " + fileName + " in the real (" +
+ link.getOriginPath() + ") or archive (" + link.getArchivePath()
+ + ") directory for the primary table.";
+ LOG.error(msg);
+ throw new CorruptedSnapshotException(msg, snapshot);
+ }
}
/**
* Returns the store file names in the snapshot.
*
+ * @param conf The current {@link Configuration} instance.
+ * @param fs {@link FileSystem}
+ * @param snapshotDir {@link Path} to the Snapshot directory
+ * @throws IOException if an error occurred while scanning the directory
+ * @return the names of hfiles in the specified snaphot
+ */
+ public static Set<String> getHFileNames(final Configuration conf, final FileSystem fs,
+ final Path snapshotDir) throws IOException {
+ SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
+ return getHFileNames(conf, fs, snapshotDir, desc);
+ }
+
+ /**
+ * Returns the store file names in the snapshot.
+ *
+ * @param conf The current {@link Configuration} instance.
* @param fs {@link FileSystem}
* @param snapshotDir {@link Path} to the Snapshot directory
+ * @param snapshotDesc the {@link SnapshotDescription} of the snapshot to inspect
* @throws IOException if an error occurred while scanning the directory
* @return the names of hfiles in the specified snaphot
*/
- public static Set<String> getHFileNames(final FileSystem fs, final Path snapshotDir)
+ private static Set<String> getHFileNames(final Configuration conf, final FileSystem fs,
+ final Path snapshotDir, final SnapshotDescription snapshotDesc)
throws IOException {
final Set<String> names = new HashSet<String>();
- visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() {
- public void storeFile (final String region, final String family, final String hfile)
- throws IOException {
+ visitTableStoreFiles(conf, fs, snapshotDir, snapshotDesc, new StoreFileVisitor() {
+ @Override
+ public void storeFile(final HRegionInfo regionInfo, final String family,
+ final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+ String hfile = storeFile.getName();
if (HFileLink.isHFileLink(hfile)) {
names.add(HFileLink.getReferencedHFileName(hfile));
} else {
@@ -266,6 +370,7 @@ public final class SnapshotReferenceUtil {
throws IOException {
final Set<String> names = new HashSet<String>();
visitLogFiles(fs, snapshotDir, new FSVisitor.LogFileVisitor() {
+ @Override
public void logFile (final String server, final String logfile) throws IOException {
names.add(logfile);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotTask.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotTask.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotTask.java
deleted file mode 100644
index ede2d85..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotTask.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.snapshot;
-
-import java.util.concurrent.Callable;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.errorhandling.ForeignException;
-import org.apache.hadoop.hbase.errorhandling.ForeignExceptionSnare;
-import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
-
-/**
- * General snapshot operation taken on a regionserver
- */
-@InterfaceAudience.Private
-public abstract class SnapshotTask implements ForeignExceptionSnare, Callable<Void>{
-
- protected final SnapshotDescription snapshot;
- protected final ForeignExceptionDispatcher errorMonitor;
-
- /**
- * @param snapshot Description of the snapshot we are going to operate on
- * @param monitor listener interested in failures to the snapshot caused by this operation
- */
- public SnapshotTask(SnapshotDescription snapshot, ForeignExceptionDispatcher monitor) {
- assert monitor != null : "ForeignExceptionDispatcher must not be null!";
- assert snapshot != null : "SnapshotDescription must not be null!";
- this.snapshot = snapshot;
- this.errorMonitor = monitor;
- }
-
- public void snapshotFailure(String message, Exception e) {
- ForeignException ee = new ForeignException(message, e);
- errorMonitor.receive(ee);
- }
-
- @Override
- public void rethrowException() throws ForeignException {
- this.errorMonitor.rethrowException();
- }
-
- @Override
- public boolean hasException() {
- return this.errorMonitor.hasException();
- }
-
- @Override
- public ForeignException getException() {
- return this.errorMonitor.getException();
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/TableInfoCopyTask.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/TableInfoCopyTask.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/TableInfoCopyTask.java
deleted file mode 100644
index ec50b71..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/TableInfoCopyTask.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.snapshot;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
-import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
-import org.apache.hadoop.hbase.util.FSTableDescriptors;
-
-/**
- * Copy the table info into the snapshot directory
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class TableInfoCopyTask extends SnapshotTask {
-
- public static final Log LOG = LogFactory.getLog(TableInfoCopyTask.class);
- private final FileSystem fs;
- private final Path rootDir;
-
- /**
- * Copy the table info for the given table into the snapshot
- * @param monitor listen for errors while running the snapshot
- * @param snapshot snapshot for which we are copying the table info
- * @param fs {@link FileSystem} where the tableinfo is stored (and where the copy will be written)
- * @param rootDir root of the {@link FileSystem} where the tableinfo is stored
- */
- public TableInfoCopyTask(ForeignExceptionDispatcher monitor,
- SnapshotDescription snapshot, FileSystem fs, Path rootDir) {
- super(snapshot, monitor);
- this.rootDir = rootDir;
- this.fs = fs;
- }
-
- @Override
- public Void call() throws Exception {
- LOG.debug("Running table info copy.");
- this.rethrowException();
- LOG.debug("Attempting to copy table info for snapshot:"
- + ClientSnapshotDescriptionUtils.toString(this.snapshot));
- // get the HTable descriptor
- HTableDescriptor orig = FSTableDescriptors.getTableDescriptorFromFs(fs, rootDir,
- TableName.valueOf(this.snapshot.getTable()));
- this.rethrowException();
- // write a copy of descriptor to the snapshot directory
- Path snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
- new FSTableDescriptors(fs, rootDir)
- .createTableDescriptorForTableDirectory(snapshotDir, orig, false);
- LOG.debug("Finished copying tableinfo.");
- return null;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/TakeSnapshotUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/TakeSnapshotUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/TakeSnapshotUtils.java
deleted file mode 100644
index fdc1834..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/TakeSnapshotUtils.java
+++ /dev/null
@@ -1,309 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.snapshot;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map.Entry;
-import java.util.Set;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.errorhandling.ForeignExceptionListener;
-import org.apache.hadoop.hbase.errorhandling.TimeoutExceptionInjector;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.HStore;
-import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
-
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Multimap;
-
-/**
- * Utilities for useful when taking a snapshot
- */
-public class TakeSnapshotUtils {
-
- private static final Log LOG = LogFactory.getLog(TakeSnapshotUtils.class);
-
- private TakeSnapshotUtils() {
- // private constructor for util class
- }
-
- /**
- * Get the per-region snapshot description location.
- * <p>
- * Under the per-snapshot directory, specific files per-region are kept in a similar layout as per
- * the current directory layout.
- * @param desc description of the snapshot
- * @param rootDir root directory for the hbase installation
- * @param regionName encoded name of the region (see {@link HRegionInfo#encodeRegionName(byte[])})
- * @return path to the per-region directory for the snapshot
- */
- public static Path getRegionSnapshotDirectory(SnapshotDescription desc, Path rootDir,
- String regionName) {
- Path snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
- return HRegion.getRegionDir(snapshotDir, regionName);
- }
-
- /**
- * Get the snapshot directory for each family to be added to the the snapshot
- * @param snapshot description of the snapshot being take
- * @param snapshotRegionDir directory in the snapshot where the region directory information
- * should be stored
- * @param families families to be added (can be null)
- * @return paths to the snapshot directory for each family, in the same order as the families
- * passed in
- */
- public static List<Path> getFamilySnapshotDirectories(SnapshotDescription snapshot,
- Path snapshotRegionDir, FileStatus[] families) {
- if (families == null || families.length == 0) return Collections.emptyList();
-
- List<Path> familyDirs = new ArrayList<Path>(families.length);
- for (FileStatus family : families) {
- // build the reference directory name
- familyDirs.add(new Path(snapshotRegionDir, family.getPath().getName()));
- }
- return familyDirs;
- }
-
- /**
- * Create a snapshot timer for the master which notifies the monitor when an error occurs
- * @param snapshot snapshot to monitor
- * @param conf configuration to use when getting the max snapshot life
- * @param monitor monitor to notify when the snapshot life expires
- * @return the timer to use update to signal the start and end of the snapshot
- */
- public static TimeoutExceptionInjector getMasterTimerAndBindToMonitor(SnapshotDescription snapshot,
- Configuration conf, ForeignExceptionListener monitor) {
- long maxTime = SnapshotDescriptionUtils.getMaxMasterTimeout(conf, snapshot.getType(),
- SnapshotDescriptionUtils.DEFAULT_MAX_WAIT_TIME);
- return new TimeoutExceptionInjector(monitor, maxTime);
- }
-
- /**
- * Verify that all the expected logs got referenced
- * @param fs filesystem where the logs live
- * @param logsDir original logs directory
- * @param serverNames names of the servers that involved in the snapshot
- * @param snapshot description of the snapshot being taken
- * @param snapshotLogDir directory for logs in the snapshot
- * @throws IOException
- */
- public static void verifyAllLogsGotReferenced(FileSystem fs, Path logsDir,
- Set<String> serverNames, SnapshotDescription snapshot, Path snapshotLogDir)
- throws IOException {
- assertTrue(snapshot, "Logs directory doesn't exist in snapshot", fs.exists(logsDir));
- // for each of the server log dirs, make sure it matches the main directory
- Multimap<String, String> snapshotLogs = getMapOfServersAndLogs(fs, snapshotLogDir, serverNames);
- Multimap<String, String> realLogs = getMapOfServersAndLogs(fs, logsDir, serverNames);
- if (realLogs != null) {
- assertNotNull(snapshot, "No server logs added to snapshot", snapshotLogs);
- } else {
- assertNull(snapshot, "Snapshotted server logs that don't exist", snapshotLogs);
- }
-
- // check the number of servers
- Set<Entry<String, Collection<String>>> serverEntries = realLogs.asMap().entrySet();
- Set<Entry<String, Collection<String>>> snapshotEntries = snapshotLogs.asMap().entrySet();
- assertEquals(snapshot, "Not the same number of snapshot and original server logs directories",
- serverEntries.size(), snapshotEntries.size());
-
- // verify we snapshotted each of the log files
- for (Entry<String, Collection<String>> serverLogs : serverEntries) {
- // if the server is not the snapshot, skip checking its logs
- if (!serverNames.contains(serverLogs.getKey())) continue;
- Collection<String> snapshotServerLogs = snapshotLogs.get(serverLogs.getKey());
- assertNotNull(snapshot, "Snapshots missing logs for server:" + serverLogs.getKey(),
- snapshotServerLogs);
-
- // check each of the log files
- assertEquals(snapshot,
- "Didn't reference all the log files for server:" + serverLogs.getKey(), serverLogs
- .getValue().size(), snapshotServerLogs.size());
- for (String log : serverLogs.getValue()) {
- assertTrue(snapshot, "Snapshot logs didn't include " + log,
- snapshotServerLogs.contains(log));
- }
- }
- }
-
- /**
- * Verify one of a snapshot's region's recovered.edits, has been at the surface (file names,
- * length), match the original directory.
- * @param fs filesystem on which the snapshot had been taken
- * @param rootDir full path to the root hbase directory
- * @param regionInfo info for the region
- * @param snapshot description of the snapshot that was taken
- * @throws IOException if there is an unexpected error talking to the filesystem
- */
- public static void verifyRecoveredEdits(FileSystem fs, Path rootDir, HRegionInfo regionInfo,
- SnapshotDescription snapshot) throws IOException {
- Path regionDir = HRegion.getRegionDir(rootDir, regionInfo);
- Path editsDir = HLogUtil.getRegionDirRecoveredEditsDir(regionDir);
- Path snapshotRegionDir = TakeSnapshotUtils.getRegionSnapshotDirectory(snapshot, rootDir,
- regionInfo.getEncodedName());
- Path snapshotEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(snapshotRegionDir);
-
- FileStatus[] edits = FSUtils.listStatus(fs, editsDir);
- FileStatus[] snapshotEdits = FSUtils.listStatus(fs, snapshotEditsDir);
- if (edits == null) {
- assertNull(snapshot, "Snapshot has edits but table doesn't", snapshotEdits);
- return;
- }
-
- assertNotNull(snapshot, "Table has edits, but snapshot doesn't", snapshotEdits);
-
- // check each of the files
- assertEquals(snapshot, "Not same number of edits in snapshot as table", edits.length,
- snapshotEdits.length);
-
- // make sure we have a file with the same name as the original
- // it would be really expensive to verify the content matches the original
- for (FileStatus edit : edits) {
- for (FileStatus sEdit : snapshotEdits) {
- if (sEdit.getPath().equals(edit.getPath())) {
- assertEquals(snapshot, "Snapshot file" + sEdit.getPath()
- + " length not equal to the original: " + edit.getPath(), edit.getLen(),
- sEdit.getLen());
- break;
- }
- }
- assertTrue(snapshot, "No edit in snapshot with name:" + edit.getPath(), false);
- }
- }
-
- private static void assertNull(SnapshotDescription snapshot, String msg, Object isNull)
- throws CorruptedSnapshotException {
- if (isNull != null) {
- throw new CorruptedSnapshotException(msg + ", Expected " + isNull + " to be null.", snapshot);
- }
- }
-
- private static void assertNotNull(SnapshotDescription snapshot, String msg, Object notNull)
- throws CorruptedSnapshotException {
- if (notNull == null) {
- throw new CorruptedSnapshotException(msg + ", Expected object to not be null, but was null.",
- snapshot);
- }
- }
-
- private static void assertTrue(SnapshotDescription snapshot, String msg, boolean isTrue)
- throws CorruptedSnapshotException {
- if (!isTrue) {
- throw new CorruptedSnapshotException(msg + ", Expected true, but was false", snapshot);
- }
- }
-
- /**
- * Assert that the expect matches the gotten amount
- * @param msg message to add the to exception
- * @param expected
- * @param gotten
- * @throws CorruptedSnapshotException thrown if the two elements don't match
- */
- private static void assertEquals(SnapshotDescription snapshot, String msg, int expected,
- int gotten) throws CorruptedSnapshotException {
- if (expected != gotten) {
- throw new CorruptedSnapshotException(msg + ". Expected:" + expected + ", got:" + gotten,
- snapshot);
- }
- }
-
- /**
- * Assert that the expect matches the gotten amount
- * @param msg message to add the to exception
- * @param expected
- * @param gotten
- * @throws CorruptedSnapshotException thrown if the two elements don't match
- */
- private static void assertEquals(SnapshotDescription snapshot, String msg, long expected,
- long gotten) throws CorruptedSnapshotException {
- if (expected != gotten) {
- throw new CorruptedSnapshotException(msg + ". Expected:" + expected + ", got:" + gotten,
- snapshot);
- }
- }
-
- /**
- * @param logdir
- * @param toInclude list of servers to include. If empty or null, returns all servers
- * @return maps of servers to all their log files. If there is no log directory, returns
- * <tt>null</tt>
- */
- private static Multimap<String, String> getMapOfServersAndLogs(FileSystem fs, Path logdir,
- Collection<String> toInclude) throws IOException {
- // create a path filter based on the passed directories to include
- PathFilter filter = toInclude == null || toInclude.size() == 0 ? null
- : new MatchesDirectoryNames(toInclude);
-
- // get all the expected directories
- FileStatus[] serverLogDirs = FSUtils.listStatus(fs, logdir, filter);
- if (serverLogDirs == null) return null;
-
- // map those into a multimap of servername -> [log files]
- Multimap<String, String> map = HashMultimap.create();
- for (FileStatus server : serverLogDirs) {
- FileStatus[] serverLogs = FSUtils.listStatus(fs, server.getPath(), null);
- if (serverLogs == null) continue;
- for (FileStatus log : serverLogs) {
- map.put(server.getPath().getName(), log.getPath().getName());
- }
- }
- return map;
- }
-
- /**
- * Path filter that only accepts paths where that have a {@link Path#getName()} that is contained
- * in the specified collection.
- */
- private static class MatchesDirectoryNames implements PathFilter {
-
- Collection<String> paths;
-
- public MatchesDirectoryNames(Collection<String> dirNames) {
- this.paths = dirNames;
- }
-
- @Override
- public boolean accept(Path path) {
- return paths.contains(path.getName());
- }
- }
-
- /**
- * Get the log directory for a specific snapshot
- * @param snapshotDir directory where the specific snapshot will be store
- * @param serverName name of the parent regionserver for the log files
- * @return path to the log home directory for the archive files.
- */
- public static Path getSnapshotHLogsDir(Path snapshotDir, String serverName) {
- return new Path(snapshotDir, HLogUtil.getHLogDirectoryName(serverName));
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java
index 75729b6..c771144 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java
@@ -150,7 +150,7 @@ public final class FSVisitor {
final FSVisitor.RecoveredEditsVisitor visitor) throws IOException {
FileStatus[] regions = FSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs));
if (regions == null) {
- LOG.info("No regions under directory:" + tableDir);
+ LOG.info("No recoveredEdits regions under directory:" + tableDir);
return;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
index b9084b9..0451121 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
@@ -22,6 +22,7 @@ package org.apache.hadoop.hbase.util;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletionService;
@@ -55,6 +56,10 @@ public abstract class ModifyRegionUtils {
void fillRegion(final HRegion region) throws IOException;
}
+ public interface RegionEditTask {
+ void editRegion(final HRegionInfo region) throws IOException;
+ }
+
/**
* Create new set of regions on the specified file-system.
* NOTE: that you should add the regions to hbase:meta after this operation.
@@ -106,10 +111,36 @@ public abstract class ModifyRegionUtils {
final RegionFillTask task) throws IOException {
if (newRegions == null) return null;
int regionNumber = newRegions.length;
- ThreadPoolExecutor regionOpenAndInitThreadPool = getRegionOpenAndInitThreadPool(conf,
+ ThreadPoolExecutor exec = getRegionOpenAndInitThreadPool(conf,
"RegionOpenAndInitThread-" + hTableDescriptor.getTableName(), regionNumber);
- CompletionService<HRegionInfo> completionService = new ExecutorCompletionService<HRegionInfo>(
- regionOpenAndInitThreadPool);
+ try {
+ return createRegions(exec, conf, rootDir, tableDir, hTableDescriptor, newRegions, task);
+ } finally {
+ exec.shutdownNow();
+ }
+ }
+
+ /**
+ * Create new set of regions on the specified file-system.
+ * NOTE: that you should add the regions to hbase:meta after this operation.
+ *
+ * @param exec Thread Pool Executor
+ * @param conf {@link Configuration}
+ * @param rootDir Root directory for HBase instance
+ * @param tableDir table directory
+ * @param hTableDescriptor description of the table
+ * @param newRegions {@link HRegionInfo} that describes the regions to create
+ * @param task {@link RegionFillTask} custom code to populate region after creation
+ * @throws IOException
+ */
+ public static List<HRegionInfo> createRegions(final ThreadPoolExecutor exec,
+ final Configuration conf, final Path rootDir, final Path tableDir,
+ final HTableDescriptor hTableDescriptor, final HRegionInfo[] newRegions,
+ final RegionFillTask task) throws IOException {
+ if (newRegions == null) return null;
+ int regionNumber = newRegions.length;
+ CompletionService<HRegionInfo> completionService =
+ new ExecutorCompletionService<HRegionInfo>(exec);
List<HRegionInfo> regionInfos = new ArrayList<HRegionInfo>();
for (final HRegionInfo newRegion : newRegions) {
completionService.submit(new Callable<HRegionInfo>() {
@@ -131,8 +162,6 @@ public abstract class ModifyRegionUtils {
throw new InterruptedIOException(e.getMessage());
} catch (ExecutionException e) {
throw new IOException(e);
- } finally {
- regionOpenAndInitThreadPool.shutdownNow();
}
return regionInfos;
}
@@ -166,6 +195,41 @@ public abstract class ModifyRegionUtils {
return region.getRegionInfo();
}
+ /**
+ * Execute the task on the specified set of regions.
+ *
+ * @param exec Thread Pool Executor
+ * @param regions {@link HRegionInfo} that describes the regions to edit
+ * @param task {@link RegionFillTask} custom code to edit the region
+ * @throws IOException
+ */
+ public static void editRegions(final ThreadPoolExecutor exec,
+ final Collection<HRegionInfo> regions, final RegionEditTask task) throws IOException {
+ final ExecutorCompletionService<Void> completionService =
+ new ExecutorCompletionService<Void>(exec);
+ for (final HRegionInfo hri: regions) {
+ completionService.submit(new Callable<Void>() {
+ @Override
+ public Void call() throws IOException {
+ task.editRegion(hri);
+ return null;
+ }
+ });
+ }
+
+ try {
+ for (HRegionInfo hri: regions) {
+ completionService.take().get();
+ }
+ } catch (InterruptedException e) {
+ throw new InterruptedIOException(e.getMessage());
+ } catch (ExecutionException e) {
+ IOException ex = new IOException();
+ ex.initCause(e.getCause());
+ throw ex;
+ }
+ }
+
/*
* used by createRegions() to get the thread pool executor based on the
* "hbase.hregion.open.and.init.threads.max" property.
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
index ef53ad2..75f1c10 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
@@ -20,9 +20,7 @@ package org.apache.hadoop.hbase.client;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
-import java.util.HashSet;
import java.util.List;
-import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -39,9 +37,9 @@ import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescriptio
import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotManifestV1;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -182,15 +180,6 @@ public class TestSnapshotFromClient {
HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
UTIL.loadTable(table, TEST_FAM, false);
- // get the name of all the regionservers hosting the snapshotted table
- Set<String> snapshotServers = new HashSet<String>();
- List<RegionServerThread> servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
- for (RegionServerThread server : servers) {
- if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) {
- snapshotServers.add(server.getRegionServer().getServerName().toString());
- }
- }
-
LOG.debug("FS state before disable:");
FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
@@ -203,8 +192,16 @@ public class TestSnapshotFromClient {
FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
// take a snapshot of the disabled table
- byte[] snapshot = Bytes.toBytes("offlineTableSnapshot");
- admin.snapshot(snapshot, TABLE_NAME);
+ final String SNAPSHOT_NAME = "offlineTableSnapshot";
+ byte[] snapshot = Bytes.toBytes(SNAPSHOT_NAME);
+
+ SnapshotDescription desc = SnapshotDescription.newBuilder()
+ .setType(SnapshotDescription.Type.DISABLED)
+ .setTable(STRING_TABLE_NAME)
+ .setName(SNAPSHOT_NAME)
+ .setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION)
+ .build();
+ admin.snapshot(desc);
LOG.debug("Snapshot completed.");
// make sure we have the snapshot
@@ -219,7 +216,7 @@ public class TestSnapshotFromClient {
FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
- admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers);
+ admin, fs);
admin.deleteSnapshot(snapshot);
snapshots = admin.listSnapshots();
@@ -263,15 +260,6 @@ public class TestSnapshotFromClient {
// make sure we don't fail on listing snapshots
SnapshotTestingUtils.assertNoSnapshots(admin);
- // get the name of all the regionservers hosting the snapshotted table
- Set<String> snapshotServers = new HashSet<String>();
- List<RegionServerThread> servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
- for (RegionServerThread server : servers) {
- if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) {
- snapshotServers.add(server.getRegionServer().getServerName().toString());
- }
- }
-
LOG.debug("FS state before disable:");
FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
@@ -299,8 +287,8 @@ public class TestSnapshotFromClient {
List<byte[]> emptyCfs = Lists.newArrayList(TEST_FAM); // no file in the region
List<byte[]> nonEmptyCfs = Lists.newArrayList();
- SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, nonEmptyCfs, emptyCfs, rootDir,
- admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers);
+ SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, nonEmptyCfs, emptyCfs,
+ rootDir, admin, fs);
admin.deleteSnapshot(snapshot);
snapshots = admin.listSnapshots();
[2/7] HBASE-11742 (Backport HBASE-7987 and HBASE-11185 to 0.98)
(Esteban Gutierrez and Matteo Bertozzi)
Posted by ap...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java
index 34edae2..8b2dd1d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java
@@ -29,13 +29,13 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.LargeTests;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.mapreduce.TestTableSnapshotInputFormat;
import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Test;
@@ -94,7 +94,7 @@ public class TestTableSnapshotScanner {
HTable table = new HTable(util.getConfiguration(), tableName);
util.loadTable(table, FAMILIES);
- Path rootDir = new Path(util.getConfiguration().get(HConstants.HBASE_DIR));
+ Path rootDir = FSUtils.getRootDir(util.getConfiguration());
FileSystem fs = rootDir.getFileSystem(util.getConfiguration());
SnapshotTestingUtils.createSnapshotAndValidate(admin, tableName,
@@ -124,8 +124,8 @@ public class TestTableSnapshotScanner {
testScanner(UTIL, "testWithMultiRegion", 20, true);
}
- private void testScanner(HBaseTestingUtility util, String snapshotName, int numRegions, boolean shutdownCluster)
- throws Exception {
+ private void testScanner(HBaseTestingUtility util, String snapshotName, int numRegions,
+ boolean shutdownCluster) throws Exception {
setupCluster();
TableName tableName = TableName.valueOf("testScanner");
try {
@@ -138,7 +138,8 @@ public class TestTableSnapshotScanner {
Path restoreDir = util.getDataTestDirOnTestFS(snapshotName);
Scan scan = new Scan(bbb, yyy); // limit the scan
- TableSnapshotScanner scanner = new TableSnapshotScanner(UTIL.getConfiguration(), restoreDir, snapshotName, scan);
+ TableSnapshotScanner scanner = new TableSnapshotScanner(UTIL.getConfiguration(), restoreDir,
+ snapshotName, scan);
verifyScanner(scanner, bbb, yyy);
scanner.close();
@@ -154,7 +155,8 @@ public class TestTableSnapshotScanner {
private void verifyScanner(ResultScanner scanner, byte[] startRow, byte[] stopRow)
throws IOException, InterruptedException {
- HBaseTestingUtility.SeenRowTracker rowTracker = new HBaseTestingUtility.SeenRowTracker(startRow, stopRow);
+ HBaseTestingUtility.SeenRowTracker rowTracker =
+ new HBaseTestingUtility.SeenRowTracker(startRow, stopRow);
while (true) {
Result result = scanner.next();
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java
index ac86e18..a6a10fd 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableSnapshotInputFormat.TableSnapshotRegionSplit;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.Job;
@@ -90,15 +91,18 @@ public class TestTableSnapshotInputFormat extends TableSnapshotInputFormatTestBa
Assert.assertEquals(Lists.newArrayList("h1"), tsif.getBestLocations(conf, blockDistribution));
blockDistribution.addHostsAndBlockWeight(new String[] {"h2"}, 2);
- Assert.assertEquals(Lists.newArrayList("h1", "h2"), tsif.getBestLocations(conf, blockDistribution));
+ Assert.assertEquals(Lists.newArrayList("h1", "h2"),
+ tsif.getBestLocations(conf, blockDistribution));
blockDistribution.addHostsAndBlockWeight(new String[] {"h2"}, 3);
- Assert.assertEquals(Lists.newArrayList("h2", "h1"), tsif.getBestLocations(conf, blockDistribution));
+ Assert.assertEquals(Lists.newArrayList("h2", "h1"),
+ tsif.getBestLocations(conf, blockDistribution));
blockDistribution.addHostsAndBlockWeight(new String[] {"h3"}, 6);
blockDistribution.addHostsAndBlockWeight(new String[] {"h4"}, 9);
- Assert.assertEquals(Lists.newArrayList("h2", "h3", "h4", "h1"), tsif.getBestLocations(conf, blockDistribution));
+ Assert.assertEquals(Lists.newArrayList("h2", "h3", "h4", "h1"),
+ tsif.getBestLocations(conf, blockDistribution));
}
public static enum TestTableSnapshotCounters {
@@ -118,7 +122,8 @@ public class TestTableSnapshotInputFormat extends TableSnapshotInputFormatTestBa
public static class TestTableSnapshotReducer
extends Reducer<ImmutableBytesWritable, NullWritable, NullWritable, NullWritable> {
- HBaseTestingUtility.SeenRowTracker rowTracker = new HBaseTestingUtility.SeenRowTracker(bbb, yyy);
+ HBaseTestingUtility.SeenRowTracker rowTracker =
+ new HBaseTestingUtility.SeenRowTracker(bbb, yyy);
@Override
protected void reduce(ImmutableBytesWritable key, Iterable<NullWritable> values,
Context context) throws IOException, InterruptedException {
@@ -199,7 +204,8 @@ public class TestTableSnapshotInputFormat extends TableSnapshotInputFormatTestBa
Assert.assertEquals(expectedNumSplits, splits.size());
- HBaseTestingUtility.SeenRowTracker rowTracker = new HBaseTestingUtility.SeenRowTracker(startRow, stopRow);
+ HBaseTestingUtility.SeenRowTracker rowTracker =
+ new HBaseTestingUtility.SeenRowTracker(startRow, stopRow);
for (int i = 0; i < splits.size(); i++) {
// validate input split
@@ -209,7 +215,8 @@ public class TestTableSnapshotInputFormat extends TableSnapshotInputFormatTestBa
// validate record reader
TaskAttemptContext taskAttemptContext = mock(TaskAttemptContext.class);
when(taskAttemptContext.getConfiguration()).thenReturn(job.getConfiguration());
- RecordReader<ImmutableBytesWritable, Result> rr = tsif.createRecordReader(split, taskAttemptContext);
+ RecordReader<ImmutableBytesWritable, Result> rr =
+ tsif.createRecordReader(split, taskAttemptContext);
rr.initialize(split, taskAttemptContext);
// validate we can read all the data back
@@ -252,7 +259,8 @@ public class TestTableSnapshotInputFormat extends TableSnapshotInputFormatTestBa
Scan scan = new Scan(startRow, endRow); // limit the scan
job.setJarByClass(util.getClass());
- TableMapReduceUtil.addDependencyJars(job.getConfiguration(), TestTableSnapshotInputFormat.class);
+ TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
+ TestTableSnapshotInputFormat.class);
TableMapReduceUtil.initTableSnapshotMapperJob(snapshotName,
scan, TestTableSnapshotMapper.class, ImmutableBytesWritable.class,
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
index 74ec2cb..85260b4 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
@@ -26,6 +26,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
+import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -53,6 +54,7 @@ import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneRes
import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
import org.apache.hadoop.hbase.util.Bytes;
@@ -331,19 +333,20 @@ public class TestSnapshotFromMaster {
// get the snapshot files for the table
Path snapshotTable = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
- Path[] snapshotHFiles = SnapshotTestingUtils.listHFiles(fs, snapshotTable);
+ Set<String> snapshotHFiles = SnapshotReferenceUtil.getHFileNames(
+ UTIL.getConfiguration(), fs, snapshotTable);
// check that the files in the archive contain the ones that we need for the snapshot
LOG.debug("Have snapshot hfiles:");
- for (Path file : snapshotHFiles) {
- LOG.debug(file);
+ for (String fileName : snapshotHFiles) {
+ LOG.debug(fileName);
}
// get the archived files for the table
Collection<String> files = getArchivedHFiles(archiveDir, rootDir, fs, TABLE_NAME);
// and make sure that there is a proper subset
- for (Path file : snapshotHFiles) {
- assertTrue("Archived hfiles " + files + " is missing snapshot file:" + file,
- files.contains(file.getName()));
+ for (String fileName : snapshotHFiles) {
+ assertTrue("Archived hfiles " + files + " is missing snapshot file:" + fileName,
+ files.contains(fileName));
}
// delete the existing snapshot
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java
index 3063b8e..e0b3782 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java
@@ -17,27 +17,26 @@
*/
package org.apache.hadoop.hbase.master.snapshot;
-import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
-import java.util.*;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.Collection;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.ObjectArrays;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.MediumTests;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
-import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock;
import org.apache.hadoop.hbase.util.FSUtils;
import org.junit.After;
import org.junit.AfterClass;
@@ -53,6 +52,7 @@ public class TestSnapshotFileCache {
private static final Log LOG = LogFactory.getLog(TestSnapshotFileCache.class);
private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+ private static long sequenceId = 0;
private static FileSystem fs;
private static Path rootDir;
@@ -76,151 +76,17 @@ public class TestSnapshotFileCache {
}
@Test(timeout = 10000000)
- public void testLoadAndDelete() throws Exception {
+ public void testLoadAndDelete() throws IOException {
// don't refresh the cache unless we tell it to
long period = Long.MAX_VALUE;
- Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
- SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
- "test-snapshot-file-cache-refresh", new SnapshotFiles());
-
- Path snapshot = new Path(snapshotDir, "snapshot");
- Path region = new Path(snapshot, "7e91021");
- Path family = new Path(region, "fam");
- Path file1 = new Path(family, "file1");
- Path file2 = new Path(family, "file2");
-
- // create two hfiles under the snapshot
- fs.createNewFile(file1);
- fs.createNewFile(file2);
-
- FSUtils.logFileSystemState(fs, rootDir, LOG);
-
- // then make sure the cache finds them
- Iterable<FileStatus> nonSnapshotFiles = cache.getUnreferencedFiles(
- Arrays.asList(FSUtils.listStatus(fs, family))
- );
- assertFalse("Cache didn't find:" + file1, Iterables.contains(nonSnapshotFiles, file1));
- assertFalse("Cache didn't find:" + file2, Iterables.contains(nonSnapshotFiles, file2));
- String not = "file-shouldn't-be-found";
- assertFalse("Cache found '" + not + "', but it shouldn't have.", Iterables.contains(nonSnapshotFiles, not));
-
- // make sure we get a little bit of separation in the modification times
- // its okay if we sleep a little longer (b/c of GC pause), as long as we sleep a little
- Thread.sleep(10);
-
- LOG.debug("Deleting snapshot.");
- // then delete the snapshot and make sure that we can still find the files
- if (!fs.delete(snapshot, true)) {
- throw new IOException("Couldn't delete " + snapshot + " for an unknown reason.");
- }
- FSUtils.logFileSystemState(fs, rootDir, LOG);
-
-
- LOG.debug("Checking to see if file is deleted.");
- nonSnapshotFiles = cache.getUnreferencedFiles(
- nonSnapshotFiles
- );
-
- assertFalse("Cache didn't find:" + file1, Iterables.contains(nonSnapshotFiles, file1));
- assertFalse("Cache didn't find:" + file2, Iterables.contains(nonSnapshotFiles, file2));
-
- // then trigger a refresh
- cache.triggerCacheRefreshForTesting();
-
- nonSnapshotFiles = cache.getUnreferencedFiles(
- nonSnapshotFiles
- );
- // and not it shouldn't find those files
- assertFalse("Cache found '" + file1 + "', but it shouldn't have.",
- Iterables.contains(nonSnapshotFiles, file1));
- assertFalse("Cache found '" + file2 + "', but it shouldn't have.",
- Iterables.contains(nonSnapshotFiles, file2));
-
- fs.delete(snapshotDir, true);
- }
-
- @Test
- public void testWeNeverCacheTmpDirAndLoadIt() throws Exception {
-
- final AtomicInteger count = new AtomicInteger(0);
- // don't refresh the cache unless we tell it to
- long period = Long.MAX_VALUE;
- Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
- SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
- "test-snapshot-file-cache-refresh", new SnapshotFiles()) {
- @Override
- List<String> getSnapshotsInProgress() throws IOException {
- List<String> result = super.getSnapshotsInProgress();
- count.incrementAndGet();
- return result;
- }
- };
-
- // create a file in a 'completed' snapshot
- Path snapshot = new Path(snapshotDir, "snapshot");
- Path region = new Path(snapshot, "7e91021");
- Path family = new Path(region, "fam");
- Path file1 = new Path(family, "file1");
- fs.createNewFile(file1);
-
- FileStatus[] completedFiles = FSUtils.listStatus(fs, family);
-
- // create an 'in progress' snapshot
- SnapshotDescription desc = SnapshotDescription.newBuilder().setName("working").build();
- snapshot = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
- region = new Path(snapshot, "7e91021");
- family = new Path(region, "fam");
- Path file2 = new Path(family, "file2");
- fs.createNewFile(file2);
- cache.triggerCacheRefreshForTesting();
-
- Iterable<FileStatus> deletableFiles = cache.getUnreferencedFiles(Arrays.asList(
- ObjectArrays.concat(completedFiles, FSUtils.listStatus(fs, family), FileStatus.class))
- );
- assertTrue(Iterables.isEmpty(deletableFiles));
- assertEquals(1, count.get()); // we check the tmp directory
-
- Path file3 = new Path(family, "file3");
- fs.create(file3);
- deletableFiles = cache.getUnreferencedFiles(Arrays.asList(
- ObjectArrays.concat(completedFiles, FSUtils.listStatus(fs, family), FileStatus.class))
- );
- assertTrue(Iterables.isEmpty(deletableFiles));
- assertEquals(2, count.get()); // we check the tmp directory
-
- }
-
- @Test
- public void testLoadsTmpDir() throws Exception {
- // don't refresh the cache unless we tell it to
- long period = Long.MAX_VALUE;
- Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
"test-snapshot-file-cache-refresh", new SnapshotFiles());
- // create a file in a 'completed' snapshot
- Path snapshot = new Path(snapshotDir, "snapshot");
- Path region = new Path(snapshot, "7e91021");
- Path family = new Path(region, "fam");
- Path file1 = new Path(family, "file1");
- fs.createNewFile(file1);
-
- // create an 'in progress' snapshot
- SnapshotDescription desc = SnapshotDescription.newBuilder().setName("working").build();
- snapshot = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
- region = new Path(snapshot, "7e91021");
- family = new Path(region, "fam");
- Path file2 = new Path(family, "file2");
- fs.createNewFile(file2);
-
- FSUtils.logFileSystemState(fs, rootDir, LOG);
+ createAndTestSnapshotV1(cache, "snapshot1a", false, true);
+ createAndTestSnapshotV1(cache, "snapshot1b", true, true);
- // then make sure the cache finds both files
- Iterable<FileStatus> nonSnapshotFiles = cache.getUnreferencedFiles(
- Arrays.asList(FSUtils.listStatus(fs, family))
- );
- assertFalse("Cache didn't find:" + file1, Iterables.contains(nonSnapshotFiles, file1));
- assertFalse("Cache didn't find:" + file2, Iterables.contains(nonSnapshotFiles, file2));
+ createAndTestSnapshotV2(cache, "snapshot2a", false, true);
+ createAndTestSnapshotV2(cache, "snapshot2b", true, true);
}
@Test
@@ -237,92 +103,136 @@ public class TestSnapshotFileCache {
});
// create a file in a 'completed' snapshot
- Path snapshot = new Path(snapshotDir, "snapshot");
- Path region = new Path(snapshot, "7e91021");
- Path family = new Path(region, "fam");
- Path file1 = new Path(family, "file1");
+ SnapshotDescription desc = SnapshotDescription.newBuilder().setName("snapshot").build();
+ Path snapshot = SnapshotDescriptionUtils.getCompletedSnapshotDir(desc, rootDir);
+ SnapshotDescriptionUtils.writeSnapshotInfo(desc, snapshot, fs);
+ Path file1 = new Path(new Path(new Path(snapshot, "7e91021"), "fam"), "file1");
fs.createNewFile(file1);
// and another file in the logs directory
- Path logs = TakeSnapshotUtils.getSnapshotHLogsDir(snapshot, "server");
+ Path logs = getSnapshotHLogsDir(snapshot, "server");
Path log = new Path(logs, "me.hbase.com%2C58939%2C1350424310315.1350424315552");
fs.createNewFile(log);
FSUtils.logFileSystemState(fs, rootDir, LOG);
- Iterable<FileStatus> nonSnapshotFiles = cache.getUnreferencedFiles(
- Arrays.asList(FSUtils.listStatus(fs, family))
- );
// then make sure the cache only finds the log files
assertFalse("Cache found '" + file1 + "', but it shouldn't have.",
- Iterables.contains(nonSnapshotFiles, file1));
- assertFalse("Cache didn't find:" + log, Iterables.contains(nonSnapshotFiles, log));
+ cache.contains(file1.getName()));
+ assertTrue("Cache didn't find:" + log, cache.contains(log.getName()));
+ }
+
+ /**
+ * Get the log directory for a specific snapshot
+ * @param snapshotDir directory where the specific snapshot will be store
+ * @param serverName name of the parent regionserver for the log files
+ * @return path to the log home directory for the archive files.
+ */
+ public static Path getSnapshotHLogsDir(Path snapshotDir, String serverName) {
+ return new Path(snapshotDir, HLogUtil.getHLogDirectoryName(serverName));
}
@Test
public void testReloadModifiedDirectory() throws IOException {
// don't refresh the cache unless we tell it to
long period = Long.MAX_VALUE;
- Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
"test-snapshot-file-cache-refresh", new SnapshotFiles());
- Path snapshot = new Path(snapshotDir, "snapshot");
- Path region = new Path(snapshot, "7e91021");
- Path family = new Path(region, "fam");
- Path file1 = new Path(family, "file1");
- Path file2 = new Path(family, "file2");
-
- // create two hfiles under the snapshot
- fs.createNewFile(file1);
- fs.createNewFile(file2);
-
- FSUtils.logFileSystemState(fs, rootDir, LOG);
-
- Iterable<FileStatus> nonSnapshotFiles = cache.getUnreferencedFiles(
- Arrays.asList(FSUtils.listStatus(fs, family))
- );
- assertFalse("Cache didn't find " + file1, Iterables.contains(nonSnapshotFiles, file1));
-
+ createAndTestSnapshotV1(cache, "snapshot1", false, true);
// now delete the snapshot and add a file with a different name
- fs.delete(snapshot, true);
- Path file3 = new Path(family, "new_file");
- fs.createNewFile(file3);
+ createAndTestSnapshotV1(cache, "snapshot1", false, false);
- FSUtils.logFileSystemState(fs, rootDir, LOG);
- nonSnapshotFiles = cache.getUnreferencedFiles(
- Arrays.asList(FSUtils.listStatus(fs, family))
- );
- assertFalse("Cache didn't find new file:" + file3, Iterables.contains(nonSnapshotFiles, file3));
+ createAndTestSnapshotV2(cache, "snapshot2", false, true);
+ // now delete the snapshot and add a file with a different name
+ createAndTestSnapshotV2(cache, "snapshot2", false, false);
}
@Test
public void testSnapshotTempDirReload() throws IOException {
long period = Long.MAX_VALUE;
- Path snapshotDir = new Path(SnapshotDescriptionUtils.getSnapshotsDir(rootDir),
- SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME);
+ // This doesn't refresh cache until we invoke it explicitly
SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
"test-snapshot-file-cache-refresh", new SnapshotFiles());
- // Add a new snapshot
- Path snapshot1 = new Path(snapshotDir, "snapshot1");
- Path file1 = new Path(new Path(new Path(snapshot1, "7e91021"), "fam"), "file1");
- fs.createNewFile(file1);
- assertTrue(cache.getSnapshotsInProgress().contains(file1.getName()));
+ // Add a new non-tmp snapshot
+ createAndTestSnapshotV1(cache, "snapshot0v1", false, false);
+ createAndTestSnapshotV1(cache, "snapshot0v2", false, false);
+
+ // Add a new tmp snapshot
+ createAndTestSnapshotV2(cache, "snapshot1", true, false);
- // Add another snapshot
- Path snapshot2 = new Path(snapshotDir, "snapshot2");
- Path file2 = new Path(new Path(new Path(snapshot2, "7e91021"), "fam2"), "file2");
- fs.createNewFile(file2);
- assertTrue(cache.getSnapshotsInProgress().contains((file2.getName())));
+ // Add another tmp snapshot
+ createAndTestSnapshotV2(cache, "snapshot2", true, false);
}
class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector {
public Collection<String> filesUnderSnapshot(final Path snapshotDir) throws IOException {
Collection<String> files = new HashSet<String>();
files.addAll(SnapshotReferenceUtil.getHLogNames(fs, snapshotDir));
- files.addAll(SnapshotReferenceUtil.getHFileNames(fs, snapshotDir));
+ files.addAll(SnapshotReferenceUtil.getHFileNames(UTIL.getConfiguration(), fs, snapshotDir));
return files;
}
};
+
+ private void createAndTestSnapshotV1(final SnapshotFileCache cache, final String name,
+ final boolean tmp, final boolean removeOnExit) throws IOException {
+ SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir);
+ SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV1(name);
+ createAndTestSnapshot(cache, builder, tmp, removeOnExit);
+ }
+
+ private void createAndTestSnapshotV2(final SnapshotFileCache cache, final String name,
+ final boolean tmp, final boolean removeOnExit) throws IOException {
+ SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir);
+ SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2(name);
+ createAndTestSnapshot(cache, builder, tmp, removeOnExit);
+ }
+
+ private void createAndTestSnapshot(final SnapshotFileCache cache,
+ final SnapshotMock.SnapshotBuilder builder,
+ final boolean tmp, final boolean removeOnExit) throws IOException {
+ List<String> files = new ArrayList<String>();
+ for (int i = 0; i < 3; ++i) {
+ for (Path filePath: builder.addRegion()) {
+ String fileName = filePath.getName();
+ if (tmp) {
+ // We should be able to find all the files while the snapshot creation is in-progress
+ FSUtils.logFileSystemState(fs, rootDir, LOG);
+ assertTrue("Cache didn't find " + fileName, cache.contains(fileName));
+ }
+ files.add(fileName);
+ }
+ }
+
+ // Finalize the snapshot
+ if (!tmp) {
+ builder.commit();
+ }
+
+ // Make sure that all files are still present
+ for (String fileName: files) {
+ assertTrue("Cache didn't find " + fileName, cache.contains(fileName));
+ }
+
+ FSUtils.logFileSystemState(fs, rootDir, LOG);
+ if (removeOnExit) {
+ LOG.debug("Deleting snapshot.");
+ fs.delete(builder.getSnapshotsDir(), true);
+ FSUtils.logFileSystemState(fs, rootDir, LOG);
+
+ // The files should be in cache until next refresh
+ for (String fileName: files) {
+ assertTrue("Cache didn't find " + fileName, cache.contains(fileName));
+ }
+
+ // then trigger a refresh
+ cache.triggerCacheRefreshForTesting();
+ // and not it shouldn't find those files
+ for (String fileName: files) {
+ assertFalse("Cache found '" + fileName + "', but it shouldn't have.",
+ cache.contains(fileName));
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
index 476da37..f3a399e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
@@ -22,17 +22,20 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
+import java.util.Arrays;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.HashSet;
import java.util.TreeSet;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
@@ -47,11 +50,13 @@ import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
import org.apache.hadoop.hbase.io.HFileLink;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.MasterFileSystem;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneResponse;
import org.apache.hadoop.hbase.regionserver.HRegion;
@@ -62,6 +67,7 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSTableDescriptors;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.FSVisitor;
+import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.MD5Hash;
import org.junit.Assert;
@@ -150,14 +156,12 @@ public class SnapshotTestingUtils {
*/
public static void confirmSnapshotValid(
SnapshotDescription snapshotDescriptor, TableName tableName,
- byte[] testFamily, Path rootDir, HBaseAdmin admin, FileSystem fs,
- boolean requireLogs, Path logsDir, Set<String> snapshotServers)
+ byte[] testFamily, Path rootDir, HBaseAdmin admin, FileSystem fs)
throws IOException {
ArrayList nonEmptyTestFamilies = new ArrayList(1);
nonEmptyTestFamilies.add(testFamily);
confirmSnapshotValid(snapshotDescriptor, tableName,
- nonEmptyTestFamilies, null, rootDir, admin, fs, requireLogs,
- logsDir, snapshotServers);
+ nonEmptyTestFamilies, null, rootDir, admin, fs);
}
/**
@@ -165,14 +169,12 @@ public class SnapshotTestingUtils {
*/
public static void confirmEmptySnapshotValid(
SnapshotDescription snapshotDescriptor, TableName tableName,
- byte[] testFamily, Path rootDir, HBaseAdmin admin, FileSystem fs,
- boolean requireLogs, Path logsDir, Set<String> snapshotServers)
+ byte[] testFamily, Path rootDir, HBaseAdmin admin, FileSystem fs)
throws IOException {
ArrayList emptyTestFamilies = new ArrayList(1);
emptyTestFamilies.add(testFamily);
confirmSnapshotValid(snapshotDescriptor, tableName,
- null, emptyTestFamilies, rootDir, admin, fs, requireLogs,
- logsDir, snapshotServers);
+ null, emptyTestFamilies, rootDir, admin, fs);
}
/**
@@ -184,45 +186,31 @@ public class SnapshotTestingUtils {
public static void confirmSnapshotValid(
SnapshotDescription snapshotDescriptor, TableName tableName,
List<byte[]> nonEmptyTestFamilies, List<byte[]> emptyTestFamilies,
- Path rootDir, HBaseAdmin admin, FileSystem fs, boolean requireLogs,
- Path logsDir, Set<String> snapshotServers) throws IOException {
+ Path rootDir, HBaseAdmin admin, FileSystem fs) throws IOException {
+ final Configuration conf = admin.getConfiguration();
+
// check snapshot dir
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
snapshotDescriptor, rootDir);
assertTrue(fs.exists(snapshotDir));
- // check snapshot info
- Path snapshotinfo = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
- assertTrue(fs.exists(snapshotinfo));
-
- // check the logs dir
- if (requireLogs) {
- TakeSnapshotUtils.verifyAllLogsGotReferenced(fs, logsDir,
- snapshotServers, snapshotDescriptor, new Path(snapshotDir,
- HConstants.HREGION_LOGDIR_NAME));
- }
-
- // check the table info
- HTableDescriptor desc = FSTableDescriptors.getTableDescriptorFromFs(fs, rootDir, tableName);
- HTableDescriptor snapshotDesc = FSTableDescriptors.getTableDescriptorFromFs(fs, snapshotDir);
- assertEquals(desc, snapshotDesc);
+ SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
// Extract regions and families with store files
- final Set<String> snapshotRegions = new HashSet<String>();
final Set<byte[]> snapshotFamilies = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
- FSVisitor.visitRegions(fs, snapshotDir, new FSVisitor.RegionVisitor() {
- @Override
- public void region(String region) throws IOException {
- snapshotRegions.add(region);
- }
- });
- FSVisitor.visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() {
- @Override
- public void storeFile(final String region, final String family, final String hfileName)
- throws IOException {
- snapshotFamilies.add(Bytes.toBytes(family));
- }
- });
+
+ SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, desc);
+ Map<String, SnapshotRegionManifest> regionManifests = manifest.getRegionManifestsMap();
+ for (SnapshotRegionManifest regionManifest: regionManifests.values()) {
+ SnapshotReferenceUtil.visitRegionStoreFiles(regionManifest,
+ new SnapshotReferenceUtil.StoreFileVisitor() {
+ @Override
+ public void storeFile(final HRegionInfo regionInfo, final String family,
+ final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+ snapshotFamilies.add(Bytes.toBytes(family));
+ }
+ });
+ }
// Verify that there are store files in the specified families
if (nonEmptyTestFamilies != null) {
@@ -240,16 +228,12 @@ public class SnapshotTestingUtils {
// check the region snapshot for all the regions
List<HRegionInfo> regions = admin.getTableRegions(tableName);
- assertEquals(regions.size(), snapshotRegions.size());
+ assertEquals(regions.size(), regionManifests.size());
- // Verify Regions
+ // Verify Regions (redundant check, see MasterSnapshotVerifier)
for (HRegionInfo info : regions) {
String regionName = info.getEncodedName();
- assertTrue(snapshotRegions.contains(regionName));
-
- Path regionDir = new Path(snapshotDir, regionName);
- HRegionInfo snapshotRegionInfo = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
- assertEquals(info, snapshotRegionInfo);
+ assertTrue(regionManifests.containsKey(regionName));
}
}
@@ -357,6 +341,17 @@ public class SnapshotTestingUtils {
return hfiles.toArray(new Path[hfiles.size()]);
}
+ public static String[] listHFileNames(final FileSystem fs, final Path tableDir)
+ throws IOException {
+ Path[] files = listHFiles(fs, tableDir);
+ String[] names = new String[files.length];
+ for (int i = 0; i < files.length; ++i) {
+ names[i] = files[i].getName();
+ }
+ Arrays.sort(names);
+ return names;
+ }
+
/**
* Take a snapshot of the specified table and verify that the given family is
* not empty. Note that this will leave the table disabled
@@ -397,8 +392,7 @@ public class SnapshotTestingUtils {
}
SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), tableName, nonEmptyFamilyNames,
- emptyFamilyNames, rootDir, admin, fs, false,
- new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), null);
+ emptyFamilyNames, rootDir, admin, fs);
}
/**
@@ -420,11 +414,15 @@ public class SnapshotTestingUtils {
final TableName table = TableName.valueOf(snapshotDesc.getTable());
final ArrayList corruptedFiles = new ArrayList();
- SnapshotReferenceUtil.visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() {
+ final Configuration conf = util.getConfiguration();
+ SnapshotReferenceUtil.visitTableStoreFiles(conf, fs, snapshotDir, snapshotDesc,
+ new SnapshotReferenceUtil.StoreFileVisitor() {
@Override
- public void storeFile (final String region, final String family, final String hfile)
- throws IOException {
- HFileLink link = HFileLink.create(util.getConfiguration(), table, region, family, hfile);
+ public void storeFile(final HRegionInfo regionInfo, final String family,
+ final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+ String region = regionInfo.getEncodedName();
+ String hfile = storeFile.getName();
+ HFileLink link = HFileLink.create(conf, table, region, family, hfile);
if (corruptedFiles.size() % 2 == 0) {
fs.delete(link.getAvailablePath(fs));
corruptedFiles.add(hfile);
@@ -437,6 +435,191 @@ public class SnapshotTestingUtils {
}
// ==========================================================================
+ // Snapshot Mock
+ // ==========================================================================
+ public static class SnapshotMock {
+ private final static String TEST_FAMILY = "cf";
+ public final static int TEST_NUM_REGIONS = 4;
+
+ private final Configuration conf;
+ private final FileSystem fs;
+ private final Path rootDir;
+
+ static class RegionData {
+ public HRegionInfo hri;
+ public Path tableDir;
+ public Path[] files;
+
+ public RegionData(final Path tableDir, final HRegionInfo hri, final int nfiles) {
+ this.tableDir = tableDir;
+ this.hri = hri;
+ this.files = new Path[nfiles];
+ }
+ }
+
+ public static class SnapshotBuilder {
+ private final RegionData[] tableRegions;
+ private final SnapshotDescription desc;
+ private final HTableDescriptor htd;
+ private final Configuration conf;
+ private final FileSystem fs;
+ private final Path rootDir;
+ private Path snapshotDir;
+ private int snapshotted = 0;
+
+ public SnapshotBuilder(final Configuration conf, final FileSystem fs,
+ final Path rootDir, final HTableDescriptor htd,
+ final SnapshotDescription desc, final RegionData[] tableRegions)
+ throws IOException {
+ this.fs = fs;
+ this.conf = conf;
+ this.rootDir = rootDir;
+ this.htd = htd;
+ this.desc = desc;
+ this.tableRegions = tableRegions;
+ this.snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
+ new FSTableDescriptors(conf)
+ .createTableDescriptorForTableDirectory(snapshotDir, htd, false);
+ }
+
+ public HTableDescriptor getTableDescriptor() {
+ return this.htd;
+ }
+
+ public SnapshotDescription getSnapshotDescription() {
+ return this.desc;
+ }
+
+ public Path getSnapshotsDir() {
+ return this.snapshotDir;
+ }
+
+ public Path[] addRegion() throws IOException {
+ return addRegion(desc);
+ }
+
+ public Path[] addRegionV1() throws IOException {
+ return addRegion(desc.toBuilder()
+ .setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION)
+ .build());
+ }
+
+ public Path[] addRegionV2() throws IOException {
+ return addRegion(desc.toBuilder()
+ .setVersion(SnapshotManifestV2.DESCRIPTOR_VERSION)
+ .build());
+ }
+
+ private Path[] addRegion(final SnapshotDescription desc) throws IOException {
+ if (this.snapshotted == tableRegions.length) {
+ throw new UnsupportedOperationException("No more regions in the table");
+ }
+
+ RegionData regionData = tableRegions[this.snapshotted++];
+ ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
+ SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
+ manifest.addRegion(regionData.tableDir, regionData.hri);
+ return regionData.files;
+ }
+
+ public Path commit() throws IOException {
+ ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
+ SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
+ manifest.addTableDescriptor(htd);
+ manifest.consolidate();
+ SnapshotDescriptionUtils.completeSnapshot(desc, rootDir, snapshotDir, fs);
+ snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(desc, rootDir);
+ return snapshotDir;
+ }
+ }
+
+ public SnapshotMock(final Configuration conf, final FileSystem fs, final Path rootDir) {
+ this.fs = fs;
+ this.conf = conf;
+ this.rootDir = rootDir;
+ }
+
+ public SnapshotBuilder createSnapshotV1(final String snapshotName) throws IOException {
+ return createSnapshot(snapshotName, SnapshotManifestV1.DESCRIPTOR_VERSION);
+ }
+
+ public SnapshotBuilder createSnapshotV2(final String snapshotName) throws IOException {
+ return createSnapshot(snapshotName, SnapshotManifestV2.DESCRIPTOR_VERSION);
+ }
+
+ private SnapshotBuilder createSnapshot(final String snapshotName, final int version)
+ throws IOException {
+ HTableDescriptor htd = createHtd(snapshotName);
+ htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
+
+ RegionData[] regions = createTable(htd, TEST_NUM_REGIONS);
+
+ SnapshotDescription desc = SnapshotDescription.newBuilder()
+ .setTable(htd.getNameAsString())
+ .setName(snapshotName)
+ .setVersion(version)
+ .build();
+
+ Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
+ SnapshotDescriptionUtils.writeSnapshotInfo(desc, workingDir, fs);
+ return new SnapshotBuilder(conf, fs, rootDir, htd, desc, regions);
+ }
+
+ public HTableDescriptor createHtd(final String tableName) {
+ HTableDescriptor htd = new HTableDescriptor(tableName);
+ htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
+ return htd;
+ }
+
+ private RegionData[] createTable(final HTableDescriptor htd, final int nregions)
+ throws IOException {
+ Path tableDir = FSUtils.getTableDir(rootDir, htd.getTableName());
+ new FSTableDescriptors(conf).createTableDescriptorForTableDirectory(tableDir, htd, false);
+
+ assertTrue(nregions % 2 == 0);
+ RegionData[] regions = new RegionData[nregions];
+ for (int i = 0; i < regions.length; i += 2) {
+ byte[] startKey = Bytes.toBytes(0 + i * 2);
+ byte[] endKey = Bytes.toBytes(1 + i * 2);
+
+ // First region, simple with one plain hfile.
+ HRegionInfo hri = new HRegionInfo(htd.getTableName(), startKey, endKey);
+ HRegionFileSystem rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
+ regions[i] = new RegionData(tableDir, hri, 3);
+ for (int j = 0; j < regions[i].files.length; ++j) {
+ Path storeFile = createStoreFile(rfs.createTempName());
+ regions[i].files[j] = rfs.commitStoreFile(TEST_FAMILY, storeFile);
+ }
+
+ // Second region, used to test the split case.
+ // This region contains a reference to the hfile in the first region.
+ startKey = Bytes.toBytes(2 + i * 2);
+ endKey = Bytes.toBytes(3 + i * 2);
+ hri = new HRegionInfo(htd.getTableName());
+ rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
+ regions[i+1] = new RegionData(tableDir, hri, regions[i].files.length);
+ for (int j = 0; j < regions[i].files.length; ++j) {
+ String refName = regions[i].files[j].getName() + '.' + regions[i].hri.getEncodedName();
+ Path refFile = createStoreFile(new Path(rootDir, refName));
+ regions[i+1].files[j] = rfs.commitStoreFile(TEST_FAMILY, refFile);
+ }
+ }
+ return regions;
+ }
+
+ private Path createStoreFile(final Path storeFile)
+ throws IOException {
+ FSDataOutputStream out = fs.create(storeFile);
+ try {
+ out.write(Bytes.toBytes(storeFile.toString()));
+ } finally {
+ out.close();
+ }
+ return storeFile;
+ }
+ }
+
+ // ==========================================================================
// Table Helpers
// ==========================================================================
public static void waitForTableToBeOnline(final HBaseTestingUtility util,
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestCopyRecoveredEditsTask.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestCopyRecoveredEditsTask.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestCopyRecoveredEditsTask.java
deleted file mode 100644
index 51e934a..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestCopyRecoveredEditsTask.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.snapshot;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.SmallTests;
-import org.apache.hadoop.hbase.errorhandling.ForeignException;
-import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
-import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
-import org.apache.hadoop.hbase.snapshot.CopyRecoveredEditsTask;
-import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import org.mockito.Mockito;
-
-/**
- * Test that we correctly copy the recovered edits from a directory
- */
-@Category(SmallTests.class)
-public class TestCopyRecoveredEditsTask {
-
- private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
-
- @Test
- public void testCopyFiles() throws Exception {
-
- SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("snapshot").build();
- ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class);
- FileSystem fs = UTIL.getTestFileSystem();
- Path root = UTIL.getDataTestDir();
- String regionName = "regionA";
- Path regionDir = new Path(root, regionName);
- Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, root);
-
- try {
- // doesn't really matter where the region's snapshot directory is, but this is pretty close
- Path snapshotRegionDir = new Path(workingDir, regionName);
- fs.mkdirs(snapshotRegionDir);
-
- // put some stuff in the recovered.edits directory
- Path edits = HLogUtil.getRegionDirRecoveredEditsDir(regionDir);
- fs.mkdirs(edits);
- // make a file with some data
- Path file1 = new Path(edits, "0000000000000002352");
- FSDataOutputStream out = fs.create(file1);
- byte[] data = new byte[] { 1, 2, 3, 4 };
- out.write(data);
- out.close();
- // make an empty file
- Path empty = new Path(edits, "empty");
- fs.createNewFile(empty);
-
- CopyRecoveredEditsTask task = new CopyRecoveredEditsTask(snapshot, monitor, fs, regionDir,
- snapshotRegionDir);
- CopyRecoveredEditsTask taskSpy = Mockito.spy(task);
- taskSpy.call();
-
- Path snapshotEdits = HLogUtil.getRegionDirRecoveredEditsDir(snapshotRegionDir);
- FileStatus[] snapshotEditFiles = FSUtils.listStatus(fs, snapshotEdits);
- assertEquals("Got wrong number of files in the snapshot edits", 1, snapshotEditFiles.length);
- FileStatus file = snapshotEditFiles[0];
- assertEquals("Didn't copy expected file", file1.getName(), file.getPath().getName());
-
- Mockito.verify(monitor, Mockito.never()).receive(Mockito.any(ForeignException.class));
- Mockito.verify(taskSpy, Mockito.never()).snapshotFailure(Mockito.anyString(),
- Mockito.any(Exception.class));
-
- } finally {
- // cleanup the working directory
- FSUtils.delete(fs, regionDir, true);
- FSUtils.delete(fs, workingDir, true);
- }
- }
-
- /**
- * Check that we don't get an exception if there is no recovered edits directory to copy
- * @throws Exception on failure
- */
- @Test
- public void testNoEditsDir() throws Exception {
- SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("snapshot").build();
- ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class);
- FileSystem fs = UTIL.getTestFileSystem();
- Path root = UTIL.getDataTestDir();
- String regionName = "regionA";
- Path regionDir = new Path(root, regionName);
- Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, root);
- try {
- // doesn't really matter where the region's snapshot directory is, but this is pretty close
- Path snapshotRegionDir = new Path(workingDir, regionName);
- fs.mkdirs(snapshotRegionDir);
- Path regionEdits = HLogUtil.getRegionDirRecoveredEditsDir(regionDir);
- assertFalse("Edits dir exists already - it shouldn't", fs.exists(regionEdits));
-
- CopyRecoveredEditsTask task = new CopyRecoveredEditsTask(snapshot, monitor, fs, regionDir,
- snapshotRegionDir);
- task.call();
- } finally {
- // cleanup the working directory
- FSUtils.delete(fs, regionDir, true);
- FSUtils.delete(fs, workingDir, true);
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
index e0f2d37..ce74d91 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
@@ -24,7 +24,6 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -48,9 +47,12 @@ import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo;
+import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.FSTableDescriptors;
import org.apache.hadoop.hbase.util.Pair;
import org.junit.After;
import org.junit.AfterClass;
@@ -144,9 +146,13 @@ public class TestExportSnapshot {
@Test
public void testBalanceSplit() throws Exception {
// Create a list of files
- List<Pair<Path, Long>> files = new ArrayList<Pair<Path, Long>>();
+ List<Pair<SnapshotFileInfo, Long>> files = new ArrayList<Pair<SnapshotFileInfo, Long>>();
for (long i = 0; i <= 20; i++) {
- files.add(new Pair<Path, Long>(new Path("file-" + i), i));
+ SnapshotFileInfo fileInfo = SnapshotFileInfo.newBuilder()
+ .setType(SnapshotFileInfo.Type.HFILE)
+ .setHfile("file-" + i)
+ .build();
+ files.add(new Pair<SnapshotFileInfo, Long>(fileInfo, i));
}
// Create 5 groups (total size 210)
@@ -155,18 +161,26 @@ public class TestExportSnapshot {
// group 2: 18, 13, 8, 3 (total size: 42)
// group 3: 17, 12, 7, 4 (total size: 42)
// group 4: 16, 11, 6, 5 (total size: 42)
- List<List<Path>> splits = ExportSnapshot.getBalancedSplits(files, 5);
+ List<List<SnapshotFileInfo>> splits = ExportSnapshot.getBalancedSplits(files, 5);
assertEquals(5, splits.size());
- assertEquals(Arrays.asList(new Path("file-20"), new Path("file-11"),
- new Path("file-10"), new Path("file-1"), new Path("file-0")), splits.get(0));
- assertEquals(Arrays.asList(new Path("file-19"), new Path("file-12"),
- new Path("file-9"), new Path("file-2")), splits.get(1));
- assertEquals(Arrays.asList(new Path("file-18"), new Path("file-13"),
- new Path("file-8"), new Path("file-3")), splits.get(2));
- assertEquals(Arrays.asList(new Path("file-17"), new Path("file-14"),
- new Path("file-7"), new Path("file-4")), splits.get(3));
- assertEquals(Arrays.asList(new Path("file-16"), new Path("file-15"),
- new Path("file-6"), new Path("file-5")), splits.get(4));
+
+ String[] split0 = new String[] {"file-20", "file-11", "file-10", "file-1", "file-0"};
+ verifyBalanceSplit(splits.get(0), split0);
+ String[] split1 = new String[] {"file-19", "file-12", "file-9", "file-2"};
+ verifyBalanceSplit(splits.get(1), split1);
+ String[] split2 = new String[] {"file-18", "file-13", "file-8", "file-3"};
+ verifyBalanceSplit(splits.get(2), split2);
+ String[] split3 = new String[] {"file-17", "file-14", "file-7", "file-4"};
+ verifyBalanceSplit(splits.get(3), split3);
+ String[] split4 = new String[] {"file-16", "file-15", "file-6", "file-5"};
+ verifyBalanceSplit(splits.get(4), split4);
+ }
+
+ private void verifyBalanceSplit(final List<SnapshotFileInfo> split, final String[] expected) {
+ assertEquals(expected.length, split.size());
+ for (int i = 0; i < expected.length; ++i) {
+ assertEquals(expected[i], split.get(i).getHfile());
+ }
}
/**
@@ -246,6 +260,11 @@ public class TestExportSnapshot {
r1fs.commitStoreFile(TEST_FAMILY, storeFile);
Path tableDir = FSUtils.getTableDir(archiveDir, tableWithRefsName);
+ HTableDescriptor htd = new HTableDescriptor(tableWithRefsName);
+ htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
+ new FSTableDescriptors(fs, rootDir)
+ .createTableDescriptorForTableDirectory(tableDir, htd, false);
+
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
FileUtil.copy(fs, tableDir, fs, snapshotDir, false, conf);
SnapshotDescriptionUtils.writeSnapshotInfo(sd, snapshotDir, fs);
@@ -365,13 +384,14 @@ public class TestExportSnapshot {
new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName));
final Path exportedArchive = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
LOG.debug(listFiles(fs, exportedArchive, exportedArchive));
- SnapshotReferenceUtil.visitReferencedFiles(fs, exportedSnapshot,
- new SnapshotReferenceUtil.FileVisitor() {
- public void storeFile (final String region, final String family, final String hfile)
- throws IOException {
+ SnapshotReferenceUtil.visitReferencedFiles(TEST_UTIL.getConfiguration(), fs, exportedSnapshot,
+ new SnapshotReferenceUtil.SnapshotVisitor() {
+ public void storeFile(final HRegionInfo regionInfo, final String family,
+ final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+ String hfile = storeFile.getName();
verifyNonEmptyFile(new Path(exportedArchive,
new Path(FSUtils.getTableDir(new Path("./"), tableName),
- new Path(region, new Path(family, hfile)))));
+ new Path(regionInfo.getEncodedName(), new Path(family, hfile)))));
}
public void recoveredEdits (final String region, final String logfile)
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
index 72a6b51..13c711a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
@@ -40,7 +40,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.LargeTests;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
@@ -53,10 +52,8 @@ import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSTableDescriptors;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
import org.apache.log4j.Level;
@@ -153,15 +150,6 @@ public class TestFlushSnapshotFromClient {
HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
SnapshotTestingUtils.loadData(UTIL, table, DEFAULT_NUM_ROWS, TEST_FAM);
- // get the name of all the regionservers hosting the snapshotted table
- Set<String> snapshotServers = new HashSet<String>();
- List<RegionServerThread> servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
- for (RegionServerThread server : servers) {
- if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) {
- snapshotServers.add(server.getRegionServer().getServerName().toString());
- }
- }
-
LOG.debug("FS state before snapshot:");
FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
@@ -184,7 +172,7 @@ public class TestFlushSnapshotFromClient {
FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
- admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers);
+ admin, fs);
}
/**
@@ -201,15 +189,6 @@ public class TestFlushSnapshotFromClient {
HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
UTIL.loadTable(table, TEST_FAM);
- // get the name of all the regionservers hosting the snapshotted table
- Set<String> snapshotServers = new HashSet<String>();
- List<RegionServerThread> servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
- for (RegionServerThread server : servers) {
- if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) {
- snapshotServers.add(server.getRegionServer().getServerName().toString());
- }
- }
-
LOG.debug("FS state before snapshot:");
FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
@@ -232,7 +211,7 @@ public class TestFlushSnapshotFromClient {
FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
- admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers);
+ admin, fs);
admin.deleteSnapshot(snapshot);
snapshots = admin.listSnapshots();
@@ -254,15 +233,6 @@ public class TestFlushSnapshotFromClient {
HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
SnapshotTestingUtils.loadData(UTIL, table, DEFAULT_NUM_ROWS, TEST_FAM);
- // get the name of all the regionservers hosting the snapshotted table
- Set<String> snapshotServers = new HashSet<String>();
- List<RegionServerThread> servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
- for (RegionServerThread server : servers) {
- if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) {
- snapshotServers.add(server.getRegionServer().getServerName().toString());
- }
- }
-
LOG.debug("FS state before snapshot:");
FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
@@ -290,7 +260,7 @@ public class TestFlushSnapshotFromClient {
FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
- admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers);
+ admin, fs);
}
@Test (timeout=300000)
@@ -451,46 +421,11 @@ public class TestFlushSnapshotFromClient {
SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
String snapshotName = "flushSnapshotCreateListDestroy";
- // test creating the snapshot
- admin.snapshot(snapshotName, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH);
- logFSTree(FSUtils.getRootDir(UTIL.getConfiguration()));
-
- // make sure we only have 1 matching snapshot
- List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
- snapshotName, TABLE_NAME);
-
- // check the directory structure
FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
- Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshots.get(0), rootDir);
- assertTrue(fs.exists(snapshotDir));
- FSUtils.logFileSystemState(UTIL.getTestFileSystem(), snapshotDir, LOG);
- Path snapshotinfo = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
- assertTrue(fs.exists(snapshotinfo));
-
- // check the table info
- HTableDescriptor desc = FSTableDescriptors.getTableDescriptorFromFs(fs,
- rootDir, TABLE_NAME);
- HTableDescriptor snapshotDesc = FSTableDescriptors.getTableDescriptorFromFs(fs,
- new Path(SnapshotDescriptionUtils.getSnapshotsDir(rootDir), snapshotName));
- assertEquals(desc, snapshotDesc);
-
- // check the region snapshot for all the regions
- List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
- assertTrue(regions.size() > 1);
- for (HRegionInfo info : regions) {
- String regionName = info.getEncodedName();
- Path regionDir = new Path(snapshotDir, regionName);
- HRegionInfo snapshotRegionInfo = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
- assertEquals(info, snapshotRegionInfo);
- // check to make sure we have the family
- Path familyDir = new Path(regionDir, Bytes.toString(TEST_FAM));
- assertTrue("Missing region " + Bytes.toString(snapshotRegionInfo.getStartKey()),
- fs.exists(familyDir));
-
- // make sure we have some file references
- assertTrue(fs.listStatus(familyDir).length > 0);
- }
+ SnapshotTestingUtils.createSnapshotAndValidate(admin,
+ TableName.valueOf(STRING_TABLE_NAME), Bytes.toString(TEST_FAM),
+ snapshotName, rootDir, fs, true);
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestReferenceRegionHFilesTask.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestReferenceRegionHFilesTask.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestReferenceRegionHFilesTask.java
deleted file mode 100644
index f11ea8a..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestReferenceRegionHFilesTask.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.snapshot;
-
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.SmallTests;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
-import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import org.mockito.Mockito;
-
-@Category(SmallTests.class)
-public class TestReferenceRegionHFilesTask {
- private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
-
- @Test
- public void testRun() throws IOException {
- FileSystem fs = UTIL.getTestFileSystem();
- // setup the region internals
- Path testdir = UTIL.getDataTestDir();
- Path regionDir = new Path(testdir, "region");
- Path family1 = new Path(regionDir, "fam1");
- // make an empty family
- Path family2 = new Path(regionDir, "fam2");
- fs.mkdirs(family2);
-
- // add some files to family 1
- Path file1 = new Path(family1, "05f99689ae254693836613d1884c6b63");
- fs.createNewFile(file1);
- Path file2 = new Path(family1, "7ac9898bf41d445aa0003e3d699d5d26");
- fs.createNewFile(file2);
-
- // create the snapshot directory
- Path snapshotRegionDir = new Path(testdir, HConstants.SNAPSHOT_DIR_NAME);
- fs.mkdirs(snapshotRegionDir);
-
- SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("name")
- .setTable("table").build();
- ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class);
- ReferenceRegionHFilesTask task = new ReferenceRegionHFilesTask(snapshot, monitor, regionDir,
- fs, snapshotRegionDir);
- ReferenceRegionHFilesTask taskSpy = Mockito.spy(task);
- task.call();
-
- // make sure we never get an error
- Mockito.verify(taskSpy, Mockito.never()).snapshotFailure(Mockito.anyString(),
- Mockito.any(Exception.class));
-
- // verify that all the hfiles get referenced
- List<String> hfiles = new ArrayList<String>(2);
- FileStatus[] regions = FSUtils.listStatus(fs, snapshotRegionDir);
- for (FileStatus region : regions) {
- FileStatus[] fams = FSUtils.listStatus(fs, region.getPath());
- for (FileStatus fam : fams) {
- FileStatus[] files = FSUtils.listStatus(fs, fam.getPath());
- for (FileStatus file : files) {
- hfiles.add(file.getPath().getName());
- }
- }
- }
- assertTrue("Didn't reference :" + file1, hfiles.contains(file1.getName()));
- assertTrue("Didn't reference :" + file1, hfiles.contains(file2.getName()));
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java
index e2bf7b8..040a466 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java
@@ -21,31 +21,23 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.SmallTests;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.catalog.CatalogTracker;
import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
import org.apache.hadoop.hbase.io.HFileLink;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock;
import org.apache.hadoop.hbase.util.FSTableDescriptors;
import org.apache.hadoop.hbase.util.FSUtils;
import org.junit.After;
@@ -62,7 +54,6 @@ public class TestRestoreSnapshotHelper {
final Log LOG = LogFactory.getLog(getClass());
private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
- private final static String TEST_FAMILY = "cf";
private final static String TEST_HFILE = "abc";
private Configuration conf;
@@ -86,49 +77,65 @@ public class TestRestoreSnapshotHelper {
@Test
public void testRestore() throws IOException {
- HTableDescriptor htd = createTableDescriptor("testtb");
-
- Path snapshotDir = new Path(rootDir, "snapshot");
- createSnapshot(rootDir, snapshotDir, htd);
+ // Test Rolling-Upgrade like Snapshot.
+ // half machines writing using v1 and the others using v2 format.
+ SnapshotMock snapshotMock = new SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir);
+ SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2("snapshot");
+ builder.addRegionV1();
+ builder.addRegionV2();
+ builder.addRegionV2();
+ builder.addRegionV1();
+ Path snapshotDir = builder.commit();
+ HTableDescriptor htd = builder.getTableDescriptor();
+ SnapshotDescription desc = builder.getSnapshotDescription();
// Test clone a snapshot
- HTableDescriptor htdClone = createTableDescriptor("testtb-clone");
- testRestore(snapshotDir, htd.getTableName().getNameAsString(), htdClone);
+ HTableDescriptor htdClone = snapshotMock.createHtd("testtb-clone");
+ testRestore(snapshotDir, desc, htdClone);
verifyRestore(rootDir, htd, htdClone);
// Test clone a clone ("link to link")
+ SnapshotDescription cloneDesc = SnapshotDescription.newBuilder()
+ .setName("cloneSnapshot")
+ .setTable("testtb-clone")
+ .build();
Path cloneDir = FSUtils.getTableDir(rootDir, htdClone.getTableName());
- HTableDescriptor htdClone2 = createTableDescriptor("testtb-clone2");
- testRestore(cloneDir, htdClone.getTableName().getNameAsString(), htdClone2);
+ HTableDescriptor htdClone2 = snapshotMock.createHtd("testtb-clone2");
+ testRestore(cloneDir, cloneDesc, htdClone2);
verifyRestore(rootDir, htd, htdClone2);
}
private void verifyRestore(final Path rootDir, final HTableDescriptor sourceHtd,
final HTableDescriptor htdClone) throws IOException {
- String[] files = getHFiles(FSUtils.getTableDir(rootDir, htdClone.getTableName()));
- assertEquals(2, files.length);
- assertTrue(files[0] + " should be a HFileLink", HFileLink.isHFileLink(files[0]));
- assertTrue(files[1] + " should be a Referene", StoreFileInfo.isReference(files[1]));
- assertEquals(sourceHtd.getTableName(), HFileLink.getReferencedTableName(files[0]));
- assertEquals(TEST_HFILE, HFileLink.getReferencedHFileName(files[0]));
- Path refPath = getReferredToFile(files[1]);
- assertTrue(refPath.getName() + " should be a HFileLink", HFileLink.isHFileLink(refPath.getName()));
- assertEquals(files[0], refPath.getName());
+ String[] files = SnapshotTestingUtils.listHFileNames(fs,
+ FSUtils.getTableDir(rootDir, htdClone.getTableName()));
+ assertEquals(12, files.length);
+ for (int i = 0; i < files.length; i += 2) {
+ String linkFile = files[i];
+ String refFile = files[i+1];
+ assertTrue(linkFile + " should be a HFileLink", HFileLink.isHFileLink(linkFile));
+ assertTrue(refFile + " should be a Referene", StoreFileInfo.isReference(refFile));
+ assertEquals(sourceHtd.getTableName(), HFileLink.getReferencedTableName(linkFile));
+ Path refPath = getReferredToFile(refFile);
+ LOG.debug("get reference name for file " + refFile + " = " + refPath);
+ assertTrue(refPath.getName() + " should be a HFileLink", HFileLink.isHFileLink(refPath.getName()));
+ assertEquals(linkFile, refPath.getName());
+ }
}
/**
* Execute the restore operation
* @param snapshotDir The snapshot directory to use as "restore source"
- * @param sourceTableName The name of the snapshotted table
+ * @param sd The snapshot descriptor
* @param htdClone The HTableDescriptor of the table to restore/clone.
*/
- public void testRestore(final Path snapshotDir, final String sourceTableName,
+ public void testRestore(final Path snapshotDir, final SnapshotDescription sd,
final HTableDescriptor htdClone) throws IOException {
LOG.debug("pre-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir);
FSUtils.logFileSystemState(fs, rootDir, LOG);
new FSTableDescriptors(conf).createTableDescriptor(htdClone);
- RestoreSnapshotHelper helper = getRestoreHelper(rootDir, snapshotDir, sourceTableName, htdClone);
+ RestoreSnapshotHelper helper = getRestoreHelper(rootDir, snapshotDir, sd, htdClone);
helper.restoreHdfsRegions();
LOG.debug("post-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir);
@@ -139,65 +146,19 @@ public class TestRestoreSnapshotHelper {
* Initialize the restore helper, based on the snapshot and table information provided.
*/
private RestoreSnapshotHelper getRestoreHelper(final Path rootDir, final Path snapshotDir,
- final String sourceTableName, final HTableDescriptor htdClone) throws IOException {
+ final SnapshotDescription sd, final HTableDescriptor htdClone) throws IOException {
CatalogTracker catalogTracker = Mockito.mock(CatalogTracker.class);
HTableDescriptor tableDescriptor = Mockito.mock(HTableDescriptor.class);
ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class);
MonitoredTask status = Mockito.mock(MonitoredTask.class);
- SnapshotDescription sd = SnapshotDescription.newBuilder()
- .setName("snapshot")
- .setTable(sourceTableName)
- .build();
-
- return new RestoreSnapshotHelper(conf, fs, sd, snapshotDir,
+ SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd);
+ return new RestoreSnapshotHelper(conf, fs, manifest,
htdClone, rootDir, monitor, status);
}
- private void createSnapshot(final Path rootDir, final Path snapshotDir, final HTableDescriptor htd)
- throws IOException {
- // First region, simple with one plain hfile.
- HRegionInfo hri = new HRegionInfo(htd.getTableName());
- HRegionFileSystem r0fs = HRegionFileSystem.createRegionOnFileSystem(conf,
- fs, FSUtils.getTableDir(archiveDir, hri.getTable()), hri);
- Path storeFile = new Path(rootDir, TEST_HFILE);
- fs.createNewFile(storeFile);
- r0fs.commitStoreFile(TEST_FAMILY, storeFile);
-
- // Second region, used to test the split case.
- // This region contains a reference to the hfile in the first region.
- hri = new HRegionInfo(htd.getTableName());
- HRegionFileSystem r1fs = HRegionFileSystem.createRegionOnFileSystem(conf,
- fs, FSUtils.getTableDir(archiveDir, hri.getTable()), hri);
- storeFile = new Path(rootDir, TEST_HFILE + '.' + r0fs.getRegionInfo().getEncodedName());
- fs.createNewFile(storeFile);
- r1fs.commitStoreFile(TEST_FAMILY, storeFile);
-
- Path tableDir = FSUtils.getTableDir(archiveDir, htd.getTableName());
- FileUtil.copy(fs, tableDir, fs, snapshotDir, false, conf);
- }
-
- private HTableDescriptor createTableDescriptor(final String tableName) {
- HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
- htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
- return htd;
- }
-
private Path getReferredToFile(final String referenceName) {
Path fakeBasePath = new Path(new Path("table", "region"), "cf");
return StoreFileInfo.getReferredToFile(new Path(fakeBasePath, referenceName));
}
-
- private String[] getHFiles(final Path tableDir) throws IOException {
- List<String> files = new ArrayList<String>();
- for (Path regionDir: FSUtils.getRegionDirs(fs, tableDir)) {
- for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) {
- for (FileStatus file: FSUtils.listStatus(fs, familyDir)) {
- files.add(file.getPath().getName());
- }
- }
- }
- Collections.sort(files);
- return files.toArray(new String[files.size()]);
- }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotDescriptionUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotDescriptionUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotDescriptionUtils.java
index 0a32ca3..4e01ec6 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotDescriptionUtils.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotDescriptionUtils.java
@@ -32,8 +32,6 @@ import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.MediumTests;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type;
-import org.apache.hadoop.hbase.util.EnvironmentEdge;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
import org.junit.After;
import org.junit.BeforeClass;
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotLogSplitter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotLogSplitter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotLogSplitter.java
deleted file mode 100644
index d66623c..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotLogSplitter.java
+++ /dev/null
@@ -1,179 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.snapshot;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.TreeMap;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.SmallTests;
-import org.apache.hadoop.hbase.regionserver.wal.HLog;
-import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
-import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
-import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
-import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.junit.*;
-import org.junit.experimental.categories.Category;
-
-/**
- * Test snapshot log splitter
- */
-@Category(SmallTests.class)
-public class TestSnapshotLogSplitter {
- final Log LOG = LogFactory.getLog(getClass());
-
- private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-
- private byte[] TEST_QUALIFIER = Bytes.toBytes("q");
- private byte[] TEST_FAMILY = Bytes.toBytes("f");
-
- private Configuration conf;
- private FileSystem fs;
- private Path logFile;
-
- @Before
- public void setup() throws Exception {
- conf = TEST_UTIL.getConfiguration();
- fs = FileSystem.get(conf);
- logFile = new Path(TEST_UTIL.getDataTestDir(), "test.log");
- writeTestLog(logFile);
- }
-
- @After
- public void tearDown() throws Exception {
- fs.delete(logFile, false);
- }
-
- @Test
- public void testSplitLogs() throws IOException {
- Map<byte[], byte[]> regionsMap = new TreeMap<byte[], byte[]>(Bytes.BYTES_COMPARATOR);
- splitTestLogs(getTableName(5), regionsMap);
- }
-
- @Test
- public void testSplitLogsOnDifferentTable() throws IOException {
- TableName tableName = getTableName(1);
- Map<byte[], byte[]> regionsMap = new TreeMap<byte[], byte[]>(Bytes.BYTES_COMPARATOR);
- for (int j = 0; j < 10; ++j) {
- byte[] regionName = getRegionName(tableName, j);
- byte[] newRegionName = getNewRegionName(tableName, j);
- regionsMap.put(regionName, newRegionName);
- }
- splitTestLogs(tableName, regionsMap);
- }
-
- /*
- * Split and verify test logs for the specified table
- */
- private void splitTestLogs(final TableName tableName,
- final Map<byte[], byte[]> regionsMap) throws IOException {
- Path tableDir = FSUtils.getTableDir(TEST_UTIL.getDataTestDir(), tableName);
- SnapshotLogSplitter logSplitter = new SnapshotLogSplitter(conf, fs, tableDir,
- tableName, regionsMap);
- try {
- logSplitter.splitLog(logFile);
- } finally {
- logSplitter.close();
- }
- verifyRecoverEdits(tableDir, tableName, regionsMap);
- }
-
- /*
- * Verify that every logs in the table directory has just the specified table and regions.
- */
- private void verifyRecoverEdits(final Path tableDir, final TableName tableName,
- final Map<byte[], byte[]> regionsMap) throws IOException {
- for (FileStatus regionStatus: FSUtils.listStatus(fs, tableDir)) {
- assertTrue(regionStatus.getPath().getName().startsWith(tableName.getNameAsString()));
- Path regionEdits = HLogUtil.getRegionDirRecoveredEditsDir(regionStatus.getPath());
- byte[] regionName = Bytes.toBytes(regionStatus.getPath().getName());
- assertFalse(regionsMap.containsKey(regionName));
- for (FileStatus logStatus: FSUtils.listStatus(fs, regionEdits)) {
- HLog.Reader reader = HLogFactory.createReader(fs, logStatus.getPath(), conf);
- try {
- HLog.Entry entry;
- while ((entry = reader.next()) != null) {
- HLogKey key = entry.getKey();
- assertEquals(tableName, key.getTablename());
- assertArrayEquals(regionName, key.getEncodedRegionName());
- }
- } finally {
- reader.close();
- }
- }
- }
- }
-
- /*
- * Write some entries in the log file.
- * 7 different tables with name "testtb-%d"
- * 10 region per table with name "tableName-region-%d"
- * 50 entry with row key "row-%d"
- */
- private void writeTestLog(final Path logFile) throws IOException {
- fs.mkdirs(logFile.getParent());
- HLog.Writer writer = HLogFactory.createWALWriter(fs, logFile, conf);
- try {
- for (int i = 0; i < 7; ++i) {
- TableName tableName = getTableName(i);
- for (int j = 0; j < 10; ++j) {
- byte[] regionName = getRegionName(tableName, j);
- for (int k = 0; k < 50; ++k) {
- byte[] rowkey = Bytes.toBytes("row-" + k);
- HLogKey key = new HLogKey(regionName, tableName, (long)k,
- System.currentTimeMillis(), HConstants.DEFAULT_CLUSTER_ID);
- WALEdit edit = new WALEdit();
- edit.add(new KeyValue(rowkey, TEST_FAMILY, TEST_QUALIFIER, rowkey));
- writer.append(new HLog.Entry(key, edit));
- }
- }
- }
- } finally {
- writer.close();
- }
- }
-
- private TableName getTableName(int tableId) {
- return TableName.valueOf("testtb-" + tableId);
- }
-
- private byte[] getRegionName(final TableName tableName, int regionId) {
- return Bytes.toBytes(tableName + "-region-" + regionId);
- }
-
- private byte[] getNewRegionName(final TableName tableName, int regionId) {
- return Bytes.toBytes(tableName + "-new-region-" + regionId);
- }
-}
[7/7] git commit: HBASE-11742 (Backport HBASE-7987 and HBASE-11185 to
0.98) (Esteban Gutierrez and Matteo Bertozzi)
Posted by ap...@apache.org.
HBASE-11742 (Backport HBASE-7987 and HBASE-11185 to 0.98) (Esteban Gutierrez and Matteo Bertozzi)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2e978a16
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2e978a16
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2e978a16
Branch: refs/heads/0.98
Commit: 2e978a165481c45c6a3d97b185c797a0425ddc44
Parents: e26411f
Author: Andrew Purtell <ap...@apache.org>
Authored: Tue Aug 19 20:05:21 2014 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Tue Aug 19 20:05:21 2014 -0700
----------------------------------------------------------------------
.../src/main/resources/hbase-default.xml | 10 +
.../protobuf/generated/MapReduceProtos.java | 620 ++-
.../protobuf/generated/SnapshotProtos.java | 4787 ++++++++++++++++++
.../src/main/protobuf/MapReduce.proto | 3 +-
hbase-protocol/src/main/protobuf/Snapshot.proto | 64 +
.../hbase/client/TableSnapshotScanner.java | 32 +-
.../org/apache/hadoop/hbase/io/Reference.java | 4 +-
.../hbase/mapred/TableSnapshotInputFormat.java | 7 +-
.../mapreduce/TableSnapshotInputFormat.java | 20 +-
.../mapreduce/TableSnapshotInputFormatImpl.java | 73 +-
.../master/snapshot/CloneSnapshotHandler.java | 4 +-
.../snapshot/DisabledTableSnapshotHandler.java | 58 +-
.../snapshot/EnabledTableSnapshotHandler.java | 10 +-
.../master/snapshot/MasterSnapshotVerifier.java | 131 +-
.../master/snapshot/RestoreSnapshotHandler.java | 7 +-
.../master/snapshot/SnapshotFileCache.java | 110 +-
.../master/snapshot/SnapshotHFileCleaner.java | 19 +-
.../master/snapshot/SnapshotLogCleaner.java | 12 +-
.../hbase/master/snapshot/SnapshotManager.java | 26 +-
.../master/snapshot/TakeSnapshotHandler.java | 53 +-
.../hbase/procedure/ProcedureManager.java | 1 -
.../hadoop/hbase/regionserver/HRegion.java | 78 +-
.../hbase/regionserver/HRegionFileSystem.java | 37 +-
.../hadoop/hbase/regionserver/StoreFile.java | 8 +-
.../hbase/regionserver/StoreFileInfo.java | 19 +-
.../hbase/snapshot/CopyRecoveredEditsTask.java | 90 -
.../hadoop/hbase/snapshot/ExportSnapshot.java | 231 +-
.../snapshot/ReferenceRegionHFilesTask.java | 127 -
.../hbase/snapshot/ReferenceServerWALsTask.java | 108 -
.../hbase/snapshot/RestoreSnapshotHelper.java | 241 +-
.../snapshot/SnapshotDescriptionUtils.java | 20 +-
.../hadoop/hbase/snapshot/SnapshotInfo.java | 154 +-
.../hbase/snapshot/SnapshotLogSplitter.java | 200 -
.../hadoop/hbase/snapshot/SnapshotManifest.java | 468 ++
.../hbase/snapshot/SnapshotManifestV1.java | 210 +
.../hbase/snapshot/SnapshotManifestV2.java | 173 +
.../hbase/snapshot/SnapshotReferenceUtil.java | 305 +-
.../hadoop/hbase/snapshot/SnapshotTask.java | 67 -
.../hbase/snapshot/TableInfoCopyTask.java | 75 -
.../hbase/snapshot/TakeSnapshotUtils.java | 309 --
.../org/apache/hadoop/hbase/util/FSVisitor.java | 2 +-
.../hadoop/hbase/util/ModifyRegionUtils.java | 74 +-
.../hbase/client/TestSnapshotFromClient.java | 40 +-
.../hbase/client/TestTableSnapshotScanner.java | 14 +-
.../mapreduce/TestTableSnapshotInputFormat.java | 22 +-
.../master/cleaner/TestSnapshotFromMaster.java | 15 +-
.../master/snapshot/TestSnapshotFileCache.java | 300 +-
.../hbase/snapshot/SnapshotTestingUtils.java | 289 +-
.../snapshot/TestCopyRecoveredEditsTask.java | 128 -
.../hbase/snapshot/TestExportSnapshot.java | 58 +-
.../snapshot/TestFlushSnapshotFromClient.java | 77 +-
.../snapshot/TestReferenceRegionHFilesTask.java | 93 -
.../snapshot/TestRestoreSnapshotHelper.java | 119 +-
.../snapshot/TestSnapshotDescriptionUtils.java | 2 -
.../hbase/snapshot/TestSnapshotLogSplitter.java | 179 -
.../hadoop/hbase/snapshot/TestSnapshotTask.java | 58 -
.../hbase/snapshot/TestWALReferenceTask.java | 103 -
57 files changed, 7544 insertions(+), 3000 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-common/src/main/resources/hbase-default.xml
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml
index dac527b..463f434 100644
--- a/hbase-common/src/main/resources/hbase-default.xml
+++ b/hbase-common/src/main/resources/hbase-default.xml
@@ -1007,6 +1007,16 @@ possible configurations would overwhelm and obscure the important.
<description>Set to true to allow snapshots to be taken / restored / cloned.</description>
</property>
<property>
+ <name>hbase.snapshot.format.version</name>
+ <value>1</value>
+ <description>The Snapshot format version to use for new snapshots.
+ 1 is the old format used by 0.94, 0.96 and 0.98 as default.
+ 2 is the new format, which should be used if all the clients that are
+ accessing snapshots via MR job are updated with the latest hbase jars.
+ By switching to version 2, the previous snapshots will still be readable.
+ </description>
+ </property>
+ <property>
<name>hbase.snapshot.restore.take.failsafe.snapshot</name>
<value>true</value>
<description>Set to true to take a snapshot before the restore operation.
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MapReduceProtos.java
----------------------------------------------------------------------
diff --git a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MapReduceProtos.java b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MapReduceProtos.java
index 1457fe8..f1243c0 100644
--- a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MapReduceProtos.java
+++ b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MapReduceProtos.java
@@ -720,20 +720,6 @@ public final class MapReduceProtos {
public interface TableSnapshotRegionSplitOrBuilder
extends com.google.protobuf.MessageOrBuilder {
- // optional .RegionSpecifier region = 1;
- /**
- * <code>optional .RegionSpecifier region = 1;</code>
- */
- boolean hasRegion();
- /**
- * <code>optional .RegionSpecifier region = 1;</code>
- */
- org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier getRegion();
- /**
- * <code>optional .RegionSpecifier region = 1;</code>
- */
- org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifierOrBuilder getRegionOrBuilder();
-
// repeated string locations = 2;
/**
* <code>repeated string locations = 2;</code>
@@ -753,6 +739,34 @@ public final class MapReduceProtos {
*/
com.google.protobuf.ByteString
getLocationsBytes(int index);
+
+ // optional .TableSchema table = 3;
+ /**
+ * <code>optional .TableSchema table = 3;</code>
+ */
+ boolean hasTable();
+ /**
+ * <code>optional .TableSchema table = 3;</code>
+ */
+ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema getTable();
+ /**
+ * <code>optional .TableSchema table = 3;</code>
+ */
+ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchemaOrBuilder getTableOrBuilder();
+
+ // optional .RegionInfo region = 4;
+ /**
+ * <code>optional .RegionInfo region = 4;</code>
+ */
+ boolean hasRegion();
+ /**
+ * <code>optional .RegionInfo region = 4;</code>
+ */
+ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo getRegion();
+ /**
+ * <code>optional .RegionInfo region = 4;</code>
+ */
+ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfoOrBuilder getRegionOrBuilder();
}
/**
* Protobuf type {@code TableSnapshotRegionSplit}
@@ -805,25 +819,38 @@ public final class MapReduceProtos {
}
break;
}
- case 10: {
- org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.Builder subBuilder = null;
+ case 18: {
+ if (!((mutable_bitField0_ & 0x00000001) == 0x00000001)) {
+ locations_ = new com.google.protobuf.LazyStringArrayList();
+ mutable_bitField0_ |= 0x00000001;
+ }
+ locations_.add(input.readBytes());
+ break;
+ }
+ case 26: {
+ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema.Builder subBuilder = null;
if (((bitField0_ & 0x00000001) == 0x00000001)) {
- subBuilder = region_.toBuilder();
+ subBuilder = table_.toBuilder();
}
- region_ = input.readMessage(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.PARSER, extensionRegistry);
+ table_ = input.readMessage(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema.PARSER, extensionRegistry);
if (subBuilder != null) {
- subBuilder.mergeFrom(region_);
- region_ = subBuilder.buildPartial();
+ subBuilder.mergeFrom(table_);
+ table_ = subBuilder.buildPartial();
}
bitField0_ |= 0x00000001;
break;
}
- case 18: {
- if (!((mutable_bitField0_ & 0x00000002) == 0x00000002)) {
- locations_ = new com.google.protobuf.LazyStringArrayList();
- mutable_bitField0_ |= 0x00000002;
+ case 34: {
+ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.Builder subBuilder = null;
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ subBuilder = region_.toBuilder();
}
- locations_.add(input.readBytes());
+ region_ = input.readMessage(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.PARSER, extensionRegistry);
+ if (subBuilder != null) {
+ subBuilder.mergeFrom(region_);
+ region_ = subBuilder.buildPartial();
+ }
+ bitField0_ |= 0x00000002;
break;
}
}
@@ -834,7 +861,7 @@ public final class MapReduceProtos {
throw new com.google.protobuf.InvalidProtocolBufferException(
e.getMessage()).setUnfinishedMessage(this);
} finally {
- if (((mutable_bitField0_ & 0x00000002) == 0x00000002)) {
+ if (((mutable_bitField0_ & 0x00000001) == 0x00000001)) {
locations_ = new com.google.protobuf.UnmodifiableLazyStringList(locations_);
}
this.unknownFields = unknownFields.build();
@@ -869,28 +896,6 @@ public final class MapReduceProtos {
}
private int bitField0_;
- // optional .RegionSpecifier region = 1;
- public static final int REGION_FIELD_NUMBER = 1;
- private org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier region_;
- /**
- * <code>optional .RegionSpecifier region = 1;</code>
- */
- public boolean hasRegion() {
- return ((bitField0_ & 0x00000001) == 0x00000001);
- }
- /**
- * <code>optional .RegionSpecifier region = 1;</code>
- */
- public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier getRegion() {
- return region_;
- }
- /**
- * <code>optional .RegionSpecifier region = 1;</code>
- */
- public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifierOrBuilder getRegionOrBuilder() {
- return region_;
- }
-
// repeated string locations = 2;
public static final int LOCATIONS_FIELD_NUMBER = 2;
private com.google.protobuf.LazyStringList locations_;
@@ -921,15 +926,66 @@ public final class MapReduceProtos {
return locations_.getByteString(index);
}
+ // optional .TableSchema table = 3;
+ public static final int TABLE_FIELD_NUMBER = 3;
+ private org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema table_;
+ /**
+ * <code>optional .TableSchema table = 3;</code>
+ */
+ public boolean hasTable() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>optional .TableSchema table = 3;</code>
+ */
+ public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema getTable() {
+ return table_;
+ }
+ /**
+ * <code>optional .TableSchema table = 3;</code>
+ */
+ public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchemaOrBuilder getTableOrBuilder() {
+ return table_;
+ }
+
+ // optional .RegionInfo region = 4;
+ public static final int REGION_FIELD_NUMBER = 4;
+ private org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo region_;
+ /**
+ * <code>optional .RegionInfo region = 4;</code>
+ */
+ public boolean hasRegion() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ /**
+ * <code>optional .RegionInfo region = 4;</code>
+ */
+ public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo getRegion() {
+ return region_;
+ }
+ /**
+ * <code>optional .RegionInfo region = 4;</code>
+ */
+ public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfoOrBuilder getRegionOrBuilder() {
+ return region_;
+ }
+
private void initFields() {
- region_ = org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.getDefaultInstance();
locations_ = com.google.protobuf.LazyStringArrayList.EMPTY;
+ table_ = org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema.getDefaultInstance();
+ region_ = org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.getDefaultInstance();
}
private byte memoizedIsInitialized = -1;
public final boolean isInitialized() {
byte isInitialized = memoizedIsInitialized;
if (isInitialized != -1) return isInitialized == 1;
+ if (hasTable()) {
+ if (!getTable().isInitialized()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ }
if (hasRegion()) {
if (!getRegion().isInitialized()) {
memoizedIsInitialized = 0;
@@ -943,12 +999,15 @@ public final class MapReduceProtos {
public void writeTo(com.google.protobuf.CodedOutputStream output)
throws java.io.IOException {
getSerializedSize();
- if (((bitField0_ & 0x00000001) == 0x00000001)) {
- output.writeMessage(1, region_);
- }
for (int i = 0; i < locations_.size(); i++) {
output.writeBytes(2, locations_.getByteString(i));
}
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ output.writeMessage(3, table_);
+ }
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ output.writeMessage(4, region_);
+ }
getUnknownFields().writeTo(output);
}
@@ -958,10 +1017,6 @@ public final class MapReduceProtos {
if (size != -1) return size;
size = 0;
- if (((bitField0_ & 0x00000001) == 0x00000001)) {
- size += com.google.protobuf.CodedOutputStream
- .computeMessageSize(1, region_);
- }
{
int dataSize = 0;
for (int i = 0; i < locations_.size(); i++) {
@@ -971,6 +1026,14 @@ public final class MapReduceProtos {
size += dataSize;
size += 1 * getLocationsList().size();
}
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeMessageSize(3, table_);
+ }
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeMessageSize(4, region_);
+ }
size += getUnknownFields().getSerializedSize();
memoizedSerializedSize = size;
return size;
@@ -994,13 +1057,18 @@ public final class MapReduceProtos {
org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit other = (org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit) obj;
boolean result = true;
+ result = result && getLocationsList()
+ .equals(other.getLocationsList());
+ result = result && (hasTable() == other.hasTable());
+ if (hasTable()) {
+ result = result && getTable()
+ .equals(other.getTable());
+ }
result = result && (hasRegion() == other.hasRegion());
if (hasRegion()) {
result = result && getRegion()
.equals(other.getRegion());
}
- result = result && getLocationsList()
- .equals(other.getLocationsList());
result = result &&
getUnknownFields().equals(other.getUnknownFields());
return result;
@@ -1014,14 +1082,18 @@ public final class MapReduceProtos {
}
int hash = 41;
hash = (19 * hash) + getDescriptorForType().hashCode();
- if (hasRegion()) {
- hash = (37 * hash) + REGION_FIELD_NUMBER;
- hash = (53 * hash) + getRegion().hashCode();
- }
if (getLocationsCount() > 0) {
hash = (37 * hash) + LOCATIONS_FIELD_NUMBER;
hash = (53 * hash) + getLocationsList().hashCode();
}
+ if (hasTable()) {
+ hash = (37 * hash) + TABLE_FIELD_NUMBER;
+ hash = (53 * hash) + getTable().hashCode();
+ }
+ if (hasRegion()) {
+ hash = (37 * hash) + REGION_FIELD_NUMBER;
+ hash = (53 * hash) + getRegion().hashCode();
+ }
hash = (29 * hash) + getUnknownFields().hashCode();
memoizedHashCode = hash;
return hash;
@@ -1123,6 +1195,7 @@ public final class MapReduceProtos {
}
private void maybeForceBuilderInitialization() {
if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+ getTableFieldBuilder();
getRegionFieldBuilder();
}
}
@@ -1132,14 +1205,20 @@ public final class MapReduceProtos {
public Builder clear() {
super.clear();
+ locations_ = com.google.protobuf.LazyStringArrayList.EMPTY;
+ bitField0_ = (bitField0_ & ~0x00000001);
+ if (tableBuilder_ == null) {
+ table_ = org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema.getDefaultInstance();
+ } else {
+ tableBuilder_.clear();
+ }
+ bitField0_ = (bitField0_ & ~0x00000002);
if (regionBuilder_ == null) {
- region_ = org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.getDefaultInstance();
+ region_ = org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.getDefaultInstance();
} else {
regionBuilder_.clear();
}
- bitField0_ = (bitField0_ & ~0x00000001);
- locations_ = com.google.protobuf.LazyStringArrayList.EMPTY;
- bitField0_ = (bitField0_ & ~0x00000002);
+ bitField0_ = (bitField0_ & ~0x00000004);
return this;
}
@@ -1168,20 +1247,28 @@ public final class MapReduceProtos {
org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit result = new org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit(this);
int from_bitField0_ = bitField0_;
int to_bitField0_ = 0;
- if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ locations_ = new com.google.protobuf.UnmodifiableLazyStringList(
+ locations_);
+ bitField0_ = (bitField0_ & ~0x00000001);
+ }
+ result.locations_ = locations_;
+ if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
to_bitField0_ |= 0x00000001;
}
+ if (tableBuilder_ == null) {
+ result.table_ = table_;
+ } else {
+ result.table_ = tableBuilder_.build();
+ }
+ if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+ to_bitField0_ |= 0x00000002;
+ }
if (regionBuilder_ == null) {
result.region_ = region_;
} else {
result.region_ = regionBuilder_.build();
}
- if (((bitField0_ & 0x00000002) == 0x00000002)) {
- locations_ = new com.google.protobuf.UnmodifiableLazyStringList(
- locations_);
- bitField0_ = (bitField0_ & ~0x00000002);
- }
- result.locations_ = locations_;
result.bitField0_ = to_bitField0_;
onBuilt();
return result;
@@ -1198,24 +1285,33 @@ public final class MapReduceProtos {
public Builder mergeFrom(org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit other) {
if (other == org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit.getDefaultInstance()) return this;
- if (other.hasRegion()) {
- mergeRegion(other.getRegion());
- }
if (!other.locations_.isEmpty()) {
if (locations_.isEmpty()) {
locations_ = other.locations_;
- bitField0_ = (bitField0_ & ~0x00000002);
+ bitField0_ = (bitField0_ & ~0x00000001);
} else {
ensureLocationsIsMutable();
locations_.addAll(other.locations_);
}
onChanged();
}
+ if (other.hasTable()) {
+ mergeTable(other.getTable());
+ }
+ if (other.hasRegion()) {
+ mergeRegion(other.getRegion());
+ }
this.mergeUnknownFields(other.getUnknownFields());
return this;
}
public final boolean isInitialized() {
+ if (hasTable()) {
+ if (!getTable().isInitialized()) {
+
+ return false;
+ }
+ }
if (hasRegion()) {
if (!getRegion().isInitialized()) {
@@ -1244,129 +1340,12 @@ public final class MapReduceProtos {
}
private int bitField0_;
- // optional .RegionSpecifier region = 1;
- private org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier region_ = org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.getDefaultInstance();
- private com.google.protobuf.SingleFieldBuilder<
- org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.Builder, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifierOrBuilder> regionBuilder_;
- /**
- * <code>optional .RegionSpecifier region = 1;</code>
- */
- public boolean hasRegion() {
- return ((bitField0_ & 0x00000001) == 0x00000001);
- }
- /**
- * <code>optional .RegionSpecifier region = 1;</code>
- */
- public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier getRegion() {
- if (regionBuilder_ == null) {
- return region_;
- } else {
- return regionBuilder_.getMessage();
- }
- }
- /**
- * <code>optional .RegionSpecifier region = 1;</code>
- */
- public Builder setRegion(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier value) {
- if (regionBuilder_ == null) {
- if (value == null) {
- throw new NullPointerException();
- }
- region_ = value;
- onChanged();
- } else {
- regionBuilder_.setMessage(value);
- }
- bitField0_ |= 0x00000001;
- return this;
- }
- /**
- * <code>optional .RegionSpecifier region = 1;</code>
- */
- public Builder setRegion(
- org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.Builder builderForValue) {
- if (regionBuilder_ == null) {
- region_ = builderForValue.build();
- onChanged();
- } else {
- regionBuilder_.setMessage(builderForValue.build());
- }
- bitField0_ |= 0x00000001;
- return this;
- }
- /**
- * <code>optional .RegionSpecifier region = 1;</code>
- */
- public Builder mergeRegion(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier value) {
- if (regionBuilder_ == null) {
- if (((bitField0_ & 0x00000001) == 0x00000001) &&
- region_ != org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.getDefaultInstance()) {
- region_ =
- org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.newBuilder(region_).mergeFrom(value).buildPartial();
- } else {
- region_ = value;
- }
- onChanged();
- } else {
- regionBuilder_.mergeFrom(value);
- }
- bitField0_ |= 0x00000001;
- return this;
- }
- /**
- * <code>optional .RegionSpecifier region = 1;</code>
- */
- public Builder clearRegion() {
- if (regionBuilder_ == null) {
- region_ = org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.getDefaultInstance();
- onChanged();
- } else {
- regionBuilder_.clear();
- }
- bitField0_ = (bitField0_ & ~0x00000001);
- return this;
- }
- /**
- * <code>optional .RegionSpecifier region = 1;</code>
- */
- public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.Builder getRegionBuilder() {
- bitField0_ |= 0x00000001;
- onChanged();
- return getRegionFieldBuilder().getBuilder();
- }
- /**
- * <code>optional .RegionSpecifier region = 1;</code>
- */
- public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifierOrBuilder getRegionOrBuilder() {
- if (regionBuilder_ != null) {
- return regionBuilder_.getMessageOrBuilder();
- } else {
- return region_;
- }
- }
- /**
- * <code>optional .RegionSpecifier region = 1;</code>
- */
- private com.google.protobuf.SingleFieldBuilder<
- org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.Builder, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifierOrBuilder>
- getRegionFieldBuilder() {
- if (regionBuilder_ == null) {
- regionBuilder_ = new com.google.protobuf.SingleFieldBuilder<
- org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.Builder, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifierOrBuilder>(
- region_,
- getParentForChildren(),
- isClean());
- region_ = null;
- }
- return regionBuilder_;
- }
-
// repeated string locations = 2;
private com.google.protobuf.LazyStringList locations_ = com.google.protobuf.LazyStringArrayList.EMPTY;
private void ensureLocationsIsMutable() {
- if (!((bitField0_ & 0x00000002) == 0x00000002)) {
+ if (!((bitField0_ & 0x00000001) == 0x00000001)) {
locations_ = new com.google.protobuf.LazyStringArrayList(locations_);
- bitField0_ |= 0x00000002;
+ bitField0_ |= 0x00000001;
}
}
/**
@@ -1436,7 +1415,7 @@ public final class MapReduceProtos {
*/
public Builder clearLocations() {
locations_ = com.google.protobuf.LazyStringArrayList.EMPTY;
- bitField0_ = (bitField0_ & ~0x00000002);
+ bitField0_ = (bitField0_ & ~0x00000001);
onChanged();
return this;
}
@@ -1454,6 +1433,240 @@ public final class MapReduceProtos {
return this;
}
+ // optional .TableSchema table = 3;
+ private org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema table_ = org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema.getDefaultInstance();
+ private com.google.protobuf.SingleFieldBuilder<
+ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema.Builder, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchemaOrBuilder> tableBuilder_;
+ /**
+ * <code>optional .TableSchema table = 3;</code>
+ */
+ public boolean hasTable() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ /**
+ * <code>optional .TableSchema table = 3;</code>
+ */
+ public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema getTable() {
+ if (tableBuilder_ == null) {
+ return table_;
+ } else {
+ return tableBuilder_.getMessage();
+ }
+ }
+ /**
+ * <code>optional .TableSchema table = 3;</code>
+ */
+ public Builder setTable(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema value) {
+ if (tableBuilder_ == null) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ table_ = value;
+ onChanged();
+ } else {
+ tableBuilder_.setMessage(value);
+ }
+ bitField0_ |= 0x00000002;
+ return this;
+ }
+ /**
+ * <code>optional .TableSchema table = 3;</code>
+ */
+ public Builder setTable(
+ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema.Builder builderForValue) {
+ if (tableBuilder_ == null) {
+ table_ = builderForValue.build();
+ onChanged();
+ } else {
+ tableBuilder_.setMessage(builderForValue.build());
+ }
+ bitField0_ |= 0x00000002;
+ return this;
+ }
+ /**
+ * <code>optional .TableSchema table = 3;</code>
+ */
+ public Builder mergeTable(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema value) {
+ if (tableBuilder_ == null) {
+ if (((bitField0_ & 0x00000002) == 0x00000002) &&
+ table_ != org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema.getDefaultInstance()) {
+ table_ =
+ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema.newBuilder(table_).mergeFrom(value).buildPartial();
+ } else {
+ table_ = value;
+ }
+ onChanged();
+ } else {
+ tableBuilder_.mergeFrom(value);
+ }
+ bitField0_ |= 0x00000002;
+ return this;
+ }
+ /**
+ * <code>optional .TableSchema table = 3;</code>
+ */
+ public Builder clearTable() {
+ if (tableBuilder_ == null) {
+ table_ = org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema.getDefaultInstance();
+ onChanged();
+ } else {
+ tableBuilder_.clear();
+ }
+ bitField0_ = (bitField0_ & ~0x00000002);
+ return this;
+ }
+ /**
+ * <code>optional .TableSchema table = 3;</code>
+ */
+ public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema.Builder getTableBuilder() {
+ bitField0_ |= 0x00000002;
+ onChanged();
+ return getTableFieldBuilder().getBuilder();
+ }
+ /**
+ * <code>optional .TableSchema table = 3;</code>
+ */
+ public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchemaOrBuilder getTableOrBuilder() {
+ if (tableBuilder_ != null) {
+ return tableBuilder_.getMessageOrBuilder();
+ } else {
+ return table_;
+ }
+ }
+ /**
+ * <code>optional .TableSchema table = 3;</code>
+ */
+ private com.google.protobuf.SingleFieldBuilder<
+ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema.Builder, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchemaOrBuilder>
+ getTableFieldBuilder() {
+ if (tableBuilder_ == null) {
+ tableBuilder_ = new com.google.protobuf.SingleFieldBuilder<
+ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema.Builder, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchemaOrBuilder>(
+ table_,
+ getParentForChildren(),
+ isClean());
+ table_ = null;
+ }
+ return tableBuilder_;
+ }
+
+ // optional .RegionInfo region = 4;
+ private org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo region_ = org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.getDefaultInstance();
+ private com.google.protobuf.SingleFieldBuilder<
+ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.Builder, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfoOrBuilder> regionBuilder_;
+ /**
+ * <code>optional .RegionInfo region = 4;</code>
+ */
+ public boolean hasRegion() {
+ return ((bitField0_ & 0x00000004) == 0x00000004);
+ }
+ /**
+ * <code>optional .RegionInfo region = 4;</code>
+ */
+ public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo getRegion() {
+ if (regionBuilder_ == null) {
+ return region_;
+ } else {
+ return regionBuilder_.getMessage();
+ }
+ }
+ /**
+ * <code>optional .RegionInfo region = 4;</code>
+ */
+ public Builder setRegion(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo value) {
+ if (regionBuilder_ == null) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ region_ = value;
+ onChanged();
+ } else {
+ regionBuilder_.setMessage(value);
+ }
+ bitField0_ |= 0x00000004;
+ return this;
+ }
+ /**
+ * <code>optional .RegionInfo region = 4;</code>
+ */
+ public Builder setRegion(
+ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.Builder builderForValue) {
+ if (regionBuilder_ == null) {
+ region_ = builderForValue.build();
+ onChanged();
+ } else {
+ regionBuilder_.setMessage(builderForValue.build());
+ }
+ bitField0_ |= 0x00000004;
+ return this;
+ }
+ /**
+ * <code>optional .RegionInfo region = 4;</code>
+ */
+ public Builder mergeRegion(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo value) {
+ if (regionBuilder_ == null) {
+ if (((bitField0_ & 0x00000004) == 0x00000004) &&
+ region_ != org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.getDefaultInstance()) {
+ region_ =
+ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.newBuilder(region_).mergeFrom(value).buildPartial();
+ } else {
+ region_ = value;
+ }
+ onChanged();
+ } else {
+ regionBuilder_.mergeFrom(value);
+ }
+ bitField0_ |= 0x00000004;
+ return this;
+ }
+ /**
+ * <code>optional .RegionInfo region = 4;</code>
+ */
+ public Builder clearRegion() {
+ if (regionBuilder_ == null) {
+ region_ = org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.getDefaultInstance();
+ onChanged();
+ } else {
+ regionBuilder_.clear();
+ }
+ bitField0_ = (bitField0_ & ~0x00000004);
+ return this;
+ }
+ /**
+ * <code>optional .RegionInfo region = 4;</code>
+ */
+ public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.Builder getRegionBuilder() {
+ bitField0_ |= 0x00000004;
+ onChanged();
+ return getRegionFieldBuilder().getBuilder();
+ }
+ /**
+ * <code>optional .RegionInfo region = 4;</code>
+ */
+ public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfoOrBuilder getRegionOrBuilder() {
+ if (regionBuilder_ != null) {
+ return regionBuilder_.getMessageOrBuilder();
+ } else {
+ return region_;
+ }
+ }
+ /**
+ * <code>optional .RegionInfo region = 4;</code>
+ */
+ private com.google.protobuf.SingleFieldBuilder<
+ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.Builder, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfoOrBuilder>
+ getRegionFieldBuilder() {
+ if (regionBuilder_ == null) {
+ regionBuilder_ = new com.google.protobuf.SingleFieldBuilder<
+ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.Builder, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfoOrBuilder>(
+ region_,
+ getParentForChildren(),
+ isClean());
+ region_ = null;
+ }
+ return regionBuilder_;
+ }
+
// @@protoc_insertion_point(builder_scope:TableSnapshotRegionSplit)
}
@@ -1485,11 +1698,12 @@ public final class MapReduceProtos {
static {
java.lang.String[] descriptorData = {
"\n\017MapReduce.proto\032\013HBase.proto\".\n\013ScanMe" +
- "trics\022\037\n\007metrics\030\001 \003(\0132\016.NameInt64Pair\"O" +
- "\n\030TableSnapshotRegionSplit\022 \n\006region\030\001 \001" +
- "(\0132\020.RegionSpecifier\022\021\n\tlocations\030\002 \003(\tB" +
- "B\n*org.apache.hadoop.hbase.protobuf.gene" +
- "ratedB\017MapReduceProtosH\001\240\001\001"
+ "trics\022\037\n\007metrics\030\001 \003(\0132\016.NameInt64Pair\"g" +
+ "\n\030TableSnapshotRegionSplit\022\021\n\tlocations\030" +
+ "\002 \003(\t\022\033\n\005table\030\003 \001(\0132\014.TableSchema\022\033\n\006re" +
+ "gion\030\004 \001(\0132\013.RegionInfoBB\n*org.apache.ha" +
+ "doop.hbase.protobuf.generatedB\017MapReduce" +
+ "ProtosH\001\240\001\001"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -1507,7 +1721,7 @@ public final class MapReduceProtos {
internal_static_TableSnapshotRegionSplit_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_TableSnapshotRegionSplit_descriptor,
- new java.lang.String[] { "Region", "Locations", });
+ new java.lang.String[] { "Locations", "Table", "Region", });
return null;
}
};
[5/7] HBASE-11742 (Backport HBASE-7987 and HBASE-11185 to 0.98)
(Esteban Gutierrez and Matteo Bertozzi)
Posted by ap...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-protocol/src/main/protobuf/MapReduce.proto
----------------------------------------------------------------------
diff --git a/hbase-protocol/src/main/protobuf/MapReduce.proto b/hbase-protocol/src/main/protobuf/MapReduce.proto
index 0c4c29e..eeb1b66 100644
--- a/hbase-protocol/src/main/protobuf/MapReduce.proto
+++ b/hbase-protocol/src/main/protobuf/MapReduce.proto
@@ -30,6 +30,7 @@ message ScanMetrics {
}
message TableSnapshotRegionSplit {
- optional RegionSpecifier region = 1;
repeated string locations = 2;
+ optional TableSchema table = 3;
+ optional RegionInfo region = 4;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-protocol/src/main/protobuf/Snapshot.proto
----------------------------------------------------------------------
diff --git a/hbase-protocol/src/main/protobuf/Snapshot.proto b/hbase-protocol/src/main/protobuf/Snapshot.proto
new file mode 100644
index 0000000..34cb919
--- /dev/null
+++ b/hbase-protocol/src/main/protobuf/Snapshot.proto
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "SnapshotProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "FS.proto";
+import "HBase.proto";
+
+message SnapshotFileInfo {
+ enum Type {
+ HFILE = 1;
+ WAL = 2;
+ }
+
+ required Type type = 1;
+
+ optional string hfile = 3;
+
+ optional string wal_server = 4;
+ optional string wal_name = 5;
+}
+
+message SnapshotRegionManifest {
+ optional int32 version = 1;
+
+ required RegionInfo region_info = 2;
+ repeated FamilyFiles family_files = 3;
+
+ message StoreFile {
+ required string name = 1;
+ optional Reference reference = 2;
+
+ // TODO: Add checksums or other fields to verify the file
+ optional uint64 file_size = 3;
+ }
+
+ message FamilyFiles {
+ required bytes family_name = 1;
+ repeated StoreFile store_files = 2;
+ }
+}
+
+message SnapshotDataManifest {
+ required TableSchema table_schema = 1;
+ repeated SnapshotRegionManifest region_manifests = 2;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/main/java/org/apache/hadoop/hbase/client/TableSnapshotScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/TableSnapshotScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/TableSnapshotScanner.java
index c20e98d..177ff9b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/TableSnapshotScanner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/TableSnapshotScanner.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.hbase.client;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.Set;
+import java.util.List;
import java.util.UUID;
import org.apache.commons.logging.Log;
@@ -32,16 +32,16 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.mapreduce.TableSnapshotInputFormat;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
import org.apache.hadoop.hbase.snapshot.ExportSnapshot;
import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
-import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
-import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
+import org.apache.hadoop.hbase.util.FSUtils;
/**
* A Scanner which performs a scan over snapshot files. Using this class requires copying the
@@ -99,8 +99,7 @@ public class TableSnapshotScanner extends AbstractClientScanner {
*/
public TableSnapshotScanner(Configuration conf, Path restoreDir,
String snapshotName, Scan scan) throws IOException {
- this(conf, new Path(conf.get(HConstants.HBASE_DIR)),
- restoreDir, snapshotName, scan);
+ this(conf, FSUtils.getRootDir(conf), restoreDir, snapshotName, scan);
}
/**
@@ -128,22 +127,21 @@ public class TableSnapshotScanner extends AbstractClientScanner {
private void init() throws IOException {
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
+ SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
+ SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
- //load table descriptor
- htd = FSTableDescriptors.getTableDescriptorFromFs(fs, snapshotDir);
+ // load table descriptor
+ htd = manifest.getTableDescriptor();
- Set<String> snapshotRegionNames
- = SnapshotReferenceUtil.getSnapshotRegionNames(fs, snapshotDir);
- if (snapshotRegionNames == null) {
+ List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests();
+ if (regionManifests == null) {
throw new IllegalArgumentException("Snapshot seems empty");
}
- regions = new ArrayList<HRegionInfo>(snapshotRegionNames.size());
- for (String regionName : snapshotRegionNames) {
+ regions = new ArrayList<HRegionInfo>(regionManifests.size());
+ for (SnapshotRegionManifest regionManifest : regionManifests) {
// load region descriptor
- Path regionDir = new Path(snapshotDir, regionName);
- HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs,
- regionDir);
+ HRegionInfo hri = HRegionInfo.convert(regionManifest.getRegionInfo());
if (CellUtil.overlappingKeys(scan.getStartRow(), scan.getStopRow(),
hri.getStartKey(), hri.getEndKey())) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/main/java/org/apache/hadoop/hbase/io/Reference.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/Reference.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/Reference.java
index de05575..394fad8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/Reference.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/Reference.java
@@ -189,7 +189,7 @@ public class Reference {
}
}
- FSProtos.Reference convert() {
+ public FSProtos.Reference convert() {
FSProtos.Reference.Builder builder = FSProtos.Reference.newBuilder();
builder.setRange(isTopFileRegion(getFileRegion())?
FSProtos.Reference.Range.TOP: FSProtos.Reference.Range.BOTTOM);
@@ -197,7 +197,7 @@ public class Reference {
return builder.build();
}
- static Reference convert(final FSProtos.Reference r) {
+ public static Reference convert(final FSProtos.Reference r) {
Reference result = new Reference();
result.splitkey = r.getSplitkey().toByteArray();
result.region = r.getRange() == FSProtos.Reference.Range.TOP? Range.top: Range.bottom;
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableSnapshotInputFormat.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableSnapshotInputFormat.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableSnapshotInputFormat.java
index b32d2fd..aa7d066 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableSnapshotInputFormat.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableSnapshotInputFormat.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.hbase.mapred;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableSnapshotInputFormatImpl;
@@ -56,8 +58,9 @@ public class TableSnapshotInputFormat implements InputFormat<ImmutableBytesWrita
this.delegate = delegate;
}
- public TableSnapshotRegionSplit(String regionName, List<String> locations) {
- this.delegate = new TableSnapshotInputFormatImpl.InputSplit(regionName, locations);
+ public TableSnapshotRegionSplit(HTableDescriptor htd, HRegionInfo regionInfo,
+ List<String> locations) {
+ this.delegate = new TableSnapshotInputFormatImpl.InputSplit(htd, regionInfo, locations);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormat.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormat.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormat.java
index 80c1ee6..2db4a58 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormat.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormat.java
@@ -30,12 +30,21 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.TableSnapshotScanner;
import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.snapshot.ExportSnapshot;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
+import org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
+import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.InputSplit;
@@ -57,7 +66,8 @@ import com.google.common.annotations.VisibleForTesting;
* while there are jobs reading from snapshot files.
* <p>
* Usage is similar to TableInputFormat, and
- * {@link TableMapReduceUtil#initTableSnapshotMapperJob(String, Scan, Class, Class, Class, Job, boolean, Path)}
+ * {@link TableMapReduceUtil#initTableSnapshotMapperJob(String, Scan, Class, Class, Class, Job,
+ * boolean, Path)}
* can be used to configure the job.
* <pre>{@code
* Job job = new Job(conf);
@@ -100,8 +110,9 @@ public class TableSnapshotInputFormat extends InputFormat<ImmutableBytesWritable
this.delegate = delegate;
}
- public TableSnapshotRegionSplit(String regionName, List<String> locations) {
- this.delegate = new TableSnapshotInputFormatImpl.InputSplit(regionName, locations);
+ public TableSnapshotRegionSplit(HTableDescriptor htd, HRegionInfo regionInfo,
+ List<String> locations) {
+ this.delegate = new TableSnapshotInputFormatImpl.InputSplit(htd, regionInfo, locations);
}
@Override
@@ -135,7 +146,6 @@ public class TableSnapshotInputFormat extends InputFormat<ImmutableBytesWritable
@Override
public void initialize(InputSplit split, TaskAttemptContext context) throws IOException,
InterruptedException {
-
this.context = context;
getCounter = TableRecordReaderImpl.retrieveGetCounterWithStringsParams(context);
delegate.initialize(
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatImpl.java
index 4811d8f..74d6dc8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatImpl.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.hadoop.hbase.mapreduce;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -35,11 +36,14 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
import org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos;
+import org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.TableSnapshotRegionSplit;
+import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos;
+import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
-import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
+import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
import org.apache.hadoop.hbase.util.ByteStringer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSTableDescriptors;
@@ -77,14 +81,16 @@ public class TableSnapshotInputFormatImpl {
* Implementation class for InputSplit logic common between mapred and mapreduce.
*/
public static class InputSplit implements Writable {
- private String regionName;
+ private HTableDescriptor htd;
+ private HRegionInfo regionInfo;
private String[] locations;
// constructor for mapreduce framework / Writable
public InputSplit() { }
- public InputSplit(String regionName, List<String> locations) {
- this.regionName = regionName;
+ public InputSplit(HTableDescriptor htd, HRegionInfo regionInfo, List<String> locations) {
+ this.htd = htd;
+ this.regionInfo = regionInfo;
if (locations == null || locations.isEmpty()) {
this.locations = new String[0];
} else {
@@ -101,15 +107,21 @@ public class TableSnapshotInputFormatImpl {
return locations;
}
+ public HTableDescriptor getTableDescriptor() {
+ return htd;
+ }
+
+ public HRegionInfo getRegionInfo() {
+ return regionInfo;
+ }
+
// TODO: We should have ProtobufSerialization in Hadoop, and directly use PB objects instead of
// doing this wrapping with Writables.
@Override
public void write(DataOutput out) throws IOException {
- MapReduceProtos.TableSnapshotRegionSplit.Builder builder =
- MapReduceProtos.TableSnapshotRegionSplit.newBuilder()
- .setRegion(HBaseProtos.RegionSpecifier.newBuilder()
- .setType(HBaseProtos.RegionSpecifier.RegionSpecifierType.ENCODED_REGION_NAME)
- .setValue(ByteStringer.wrap(Bytes.toBytes(regionName))).build());
+ MapReduceProtos.TableSnapshotRegionSplit.Builder builder = MapReduceProtos.TableSnapshotRegionSplit.newBuilder()
+ .setTable(htd.convert())
+ .setRegion(HRegionInfo.convert(regionInfo));
for (String location : locations) {
builder.addLocations(location);
@@ -130,8 +142,9 @@ public class TableSnapshotInputFormatImpl {
int len = in.readInt();
byte[] buf = new byte[len];
in.readFully(buf);
- MapReduceProtos.TableSnapshotRegionSplit split = MapReduceProtos.TableSnapshotRegionSplit.PARSER.parseFrom(buf);
- this.regionName = Bytes.toString(split.getRegion().getValue().toByteArray());
+ TableSnapshotRegionSplit split = TableSnapshotRegionSplit.PARSER.parseFrom(buf);
+ this.htd = HTableDescriptor.convert(split.getTable());
+ this.regionInfo = HRegionInfo.convert(split.getRegion());
List<String> locationsList = split.getLocationsList();
this.locations = locationsList.toArray(new String[locationsList.size()]);
}
@@ -153,23 +166,13 @@ public class TableSnapshotInputFormatImpl {
public void initialize(InputSplit split, Configuration conf) throws IOException {
this.split = split;
- String regionName = this.split.regionName;
- String snapshotName = getSnapshotName(conf);
- Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
- FileSystem fs = rootDir.getFileSystem(conf);
+ HTableDescriptor htd = split.htd;
+ HRegionInfo hri = this.split.getRegionInfo();
+ FileSystem fs = FSUtils.getCurrentFileSystem(conf);
Path tmpRootDir = new Path(conf.get(RESTORE_DIR_KEY)); // This is the user specified root
// directory where snapshot was restored
- Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
-
- //load table descriptor
- HTableDescriptor htd = FSTableDescriptors.getTableDescriptorFromFs(fs, snapshotDir);
-
- //load region descriptor
- Path regionDir = new Path(snapshotDir, regionName);
- HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
-
// create scan
// TODO: mapred does not support scan as input API. Work around for now.
if (conf.get(TableInputFormat.SCAN) != null) {
@@ -240,16 +243,16 @@ public class TableSnapshotInputFormatImpl {
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
HBaseProtos.SnapshotDescription snapshotDesc =
SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
+ SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
+
+ List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests();
- Set<String> snapshotRegionNames =
- SnapshotReferenceUtil.getSnapshotRegionNames(fs, snapshotDir);
- if (snapshotRegionNames == null) {
- throw new IllegalArgumentException("Snapshot seems empty");
+ if (regionManifests == null) {
+ throw new IllegalArgumentException("Snapshot seems empty");
}
// load table descriptor
- HTableDescriptor htd = FSTableDescriptors.getTableDescriptorFromFs(fs,
- snapshotDir);
+ HTableDescriptor htd = manifest.getTableDescriptor();
// TODO: mapred does not support scan as input API. Work around for now.
Scan scan = null;
@@ -270,11 +273,9 @@ public class TableSnapshotInputFormatImpl {
Path tableDir = FSUtils.getTableDir(restoreDir, htd.getTableName());
List<InputSplit> splits = new ArrayList<InputSplit>();
- for (String regionName : snapshotRegionNames) {
+ for (SnapshotRegionManifest regionManifest : regionManifests) {
// load region descriptor
- Path regionDir = new Path(snapshotDir, regionName);
- HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs,
- regionDir);
+ HRegionInfo hri = HRegionInfo.convert(regionManifest.getRegionInfo());
if (CellUtil.overlappingKeys(scan.getStartRow(), scan.getStopRow(),
hri.getStartKey(), hri.getEndKey())) {
@@ -285,7 +286,7 @@ public class TableSnapshotInputFormatImpl {
int len = Math.min(3, hosts.size());
hosts = hosts.subList(0, len);
- splits.add(new InputSplit(regionName, hosts));
+ splits.add(new InputSplit(htd, hri, hosts));
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java
index 8c5aba7..d902c58 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils;
import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
import com.google.common.base.Preconditions;
@@ -107,8 +108,9 @@ public class CloneSnapshotHandler extends CreateTableHandler implements Snapshot
try {
// 1. Execute the on-disk Clone
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
+ SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshot);
RestoreSnapshotHelper restoreHelper = new RestoreSnapshotHelper(conf, fs,
- snapshot, snapshotDir, hTableDescriptor, tableRootDir, monitor, status);
+ manifest, hTableDescriptor, tableRootDir, monitor, status);
metaChanges = restoreHelper.restoreHdfsRegions();
// Clone operation should not have stuff to restore or remove
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java
index af21904..d7c7956 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java
@@ -21,26 +21,25 @@ import java.io.IOException;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import java.util.concurrent.ThreadPoolExecutor;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionListener;
import org.apache.hadoop.hbase.errorhandling.TimeoutExceptionInjector;
import org.apache.hadoop.hbase.master.MasterServices;
-import org.apache.hadoop.hbase.master.MetricsMaster;
-import org.apache.hadoop.hbase.monitoring.MonitoredTask;
-import org.apache.hadoop.hbase.monitoring.TaskMonitor;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
-import org.apache.hadoop.hbase.snapshot.TableInfoCopyTask;
-import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.ModifyRegionUtils;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.zookeeper.KeeperException;
@@ -64,7 +63,7 @@ public class DisabledTableSnapshotHandler extends TakeSnapshotHandler {
super(snapshot, masterServices);
// setup the timer
- timeoutInjector = TakeSnapshotUtils.getMasterTimerAndBindToMonitor(snapshot, conf, monitor);
+ timeoutInjector = getMasterTimerAndBindToMonitor(snapshot, conf, monitor);
}
@Override
@@ -80,8 +79,6 @@ public class DisabledTableSnapshotHandler extends TakeSnapshotHandler {
try {
timeoutInjector.start();
- Path snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
-
// 1. get all the regions hosting this table.
// extract each pair to separate lists
@@ -95,34 +92,47 @@ public class DisabledTableSnapshotHandler extends TakeSnapshotHandler {
+ ClientSnapshotDescriptionUtils.toString(snapshot);
LOG.info(msg);
status.setStatus(msg);
- for (HRegionInfo regionInfo : regions) {
- snapshotDisabledRegion(regionInfo);
- }
- // 3. write the table info to disk
- LOG.info("Starting to copy tableinfo for offline snapshot: " +
- ClientSnapshotDescriptionUtils.toString(snapshot));
- TableInfoCopyTask tableInfoCopyTask = new TableInfoCopyTask(this.monitor, snapshot, fs,
- FSUtils.getRootDir(conf));
- tableInfoCopyTask.call();
- monitor.rethrowException();
- status.setStatus("Finished copying tableinfo for snapshot of table: " +
- snapshotTable);
+ ThreadPoolExecutor exec = SnapshotManifest.createExecutor(conf, "DisabledTableSnapshot");
+ try {
+ ModifyRegionUtils.editRegions(exec, regions, new ModifyRegionUtils.RegionEditTask() {
+ @Override
+ public void editRegion(final HRegionInfo regionInfo) throws IOException {
+ snapshotManifest.addRegion(FSUtils.getTableDir(rootDir, snapshotTable), regionInfo);
+ }
+ });
+ } finally {
+ exec.shutdown();
+ }
} catch (Exception e) {
// make sure we capture the exception to propagate back to the client later
String reason = "Failed snapshot " + ClientSnapshotDescriptionUtils.toString(snapshot)
+ " due to exception:" + e.getMessage();
ForeignException ee = new ForeignException(reason, e);
monitor.receive(ee);
- status.abort("Snapshot of table: "+ snapshotTable +
- " failed because " + e.getMessage());
+ status.abort("Snapshot of table: "+ snapshotTable + " failed because " + e.getMessage());
} finally {
LOG.debug("Marking snapshot" + ClientSnapshotDescriptionUtils.toString(snapshot)
+ " as finished.");
- // 6. mark the timer as finished - even if we got an exception, we don't need to time the
+ // 3. mark the timer as finished - even if we got an exception, we don't need to time the
// operation any further
timeoutInjector.complete();
}
}
+
+
+ /**
+ * Create a snapshot timer for the master which notifies the monitor when an error occurs
+ * @param snapshot snapshot to monitor
+ * @param conf configuration to use when getting the max snapshot life
+ * @param monitor monitor to notify when the snapshot life expires
+ * @return the timer to use update to signal the start and end of the snapshot
+ */
+ private TimeoutExceptionInjector getMasterTimerAndBindToMonitor(SnapshotDescription snapshot,
+ Configuration conf, ForeignExceptionListener monitor) {
+ long maxTime = SnapshotDescriptionUtils.getMaxMasterTimeout(conf, snapshot.getType(),
+ SnapshotDescriptionUtils.DEFAULT_MAX_WAIT_TIME);
+ return new TimeoutExceptionInjector(monitor, maxTime);
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java
index c024fcf..02027ca 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java
@@ -25,17 +25,14 @@ import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.errorhandling.ForeignException;
import org.apache.hadoop.hbase.master.MasterServices;
-import org.apache.hadoop.hbase.master.MetricsMaster;
import org.apache.hadoop.hbase.procedure.Procedure;
import org.apache.hadoop.hbase.procedure.ProcedureCoordinator;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
-import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
import org.apache.hadoop.hbase.util.Pair;
import com.google.common.collect.Lists;
@@ -99,14 +96,11 @@ public class EnabledTableSnapshotHandler extends TakeSnapshotHandler {
LOG.info("Done waiting - online snapshot for " + this.snapshot.getName());
// Take the offline regions as disabled
- Path snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
for (Pair<HRegionInfo, ServerName> region : regions) {
HRegionInfo regionInfo = region.getFirst();
if (regionInfo.isOffline() && (regionInfo.isSplit() || regionInfo.isSplitParent())) {
- if (!fs.exists(new Path(snapshotDir, regionInfo.getEncodedName()))) {
- LOG.info("Take disabled snapshot of offline region=" + regionInfo);
- snapshotDisabledRegion(regionInfo);
- }
+ LOG.info("Take disabled snapshot of offline region=" + regionInfo);
+ snapshotDisabledRegion(regionInfo);
}
}
} catch (InterruptedException e) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java
index 9578c50..d5d9993 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java
@@ -19,35 +19,28 @@ package org.apache.hadoop.hbase.master.snapshot;
import java.io.IOException;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.catalog.MetaReader;
-import org.apache.hadoop.hbase.io.HFileLink;
import org.apache.hadoop.hbase.master.MasterServices;
-import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
-import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils;
import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
-import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils;
-import org.apache.hadoop.hbase.util.FSTableDescriptors;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.FSVisitor;
-import org.apache.hadoop.hbase.util.HFileArchiveUtil;
/**
* General snapshot verification on the master.
@@ -110,14 +103,16 @@ public final class MasterSnapshotVerifier {
*/
public void verifySnapshot(Path snapshotDir, Set<String> snapshotServers)
throws CorruptedSnapshotException, IOException {
+ SnapshotManifest manifest = SnapshotManifest.open(services.getConfiguration(), fs,
+ snapshotDir, snapshot);
// verify snapshot info matches
verifySnapshotDescription(snapshotDir);
// check that tableinfo is a valid table description
- verifyTableInfo(snapshotDir);
+ verifyTableInfo(manifest);
// check that each region is valid
- verifyRegions(snapshotDir);
+ verifyRegions(manifest);
}
/**
@@ -134,116 +129,78 @@ public final class MasterSnapshotVerifier {
/**
* Check that the table descriptor for the snapshot is a valid table descriptor
- * @param snapshotDir snapshot directory to check
+ * @param manifest snapshot manifest to inspect
*/
- private void verifyTableInfo(Path snapshotDir) throws IOException {
- FSTableDescriptors.getTableDescriptorFromFs(fs, snapshotDir);
+ private void verifyTableInfo(final SnapshotManifest manifest) throws IOException {
+ HTableDescriptor htd = manifest.getTableDescriptor();
+ if (htd == null) {
+ throw new CorruptedSnapshotException("Missing Table Descriptor", snapshot);
+ }
+
+ if (!htd.getNameAsString().equals(snapshot.getTable())) {
+ throw new CorruptedSnapshotException("Invalid Table Descriptor. Expected "
+ + snapshot.getTable() + " name, got " + htd.getNameAsString(), snapshot);
+ }
}
/**
* Check that all the regions in the snapshot are valid, and accounted for.
- * @param snapshotDir snapshot directory to check
+ * @param manifest snapshot manifest to inspect
* @throws IOException if we can't reach hbase:meta or read the files from the FS
*/
- private void verifyRegions(Path snapshotDir) throws IOException {
+ private void verifyRegions(final SnapshotManifest manifest) throws IOException {
List<HRegionInfo> regions = MetaReader.getTableRegions(this.services.getCatalogTracker(),
tableName);
- Set<String> snapshotRegions = SnapshotReferenceUtil.getSnapshotRegionNames(fs, snapshotDir);
- if (snapshotRegions == null) {
+ Map<String, SnapshotRegionManifest> regionManifests = manifest.getRegionManifestsMap();
+ if (regionManifests == null) {
String msg = "Snapshot " + ClientSnapshotDescriptionUtils.toString(snapshot) + " looks empty";
LOG.error(msg);
throw new CorruptedSnapshotException(msg);
}
String errorMsg = "";
- if (snapshotRegions.size() != regions.size()) {
- errorMsg = "Regions moved during the snapshot '" +
+ if (regionManifests.size() != regions.size()) {
+ errorMsg = "Regions moved during the snapshot '" +
ClientSnapshotDescriptionUtils.toString(snapshot) + "'. expected=" +
- regions.size() + " snapshotted=" + snapshotRegions.size() + ".";
+ regions.size() + " snapshotted=" + regionManifests.size() + ".";
LOG.error(errorMsg);
}
+ // Verify HRegionInfo
for (HRegionInfo region : regions) {
- if (!snapshotRegions.contains(region.getEncodedName())) {
+ SnapshotRegionManifest regionManifest = regionManifests.get(region.getEncodedName());
+ if (regionManifest == null) {
// could happen due to a move or split race.
String mesg = " No snapshot region directory found for region:" + region;
if (errorMsg.isEmpty()) errorMsg = mesg;
LOG.error(mesg);
+ continue;
}
- verifyRegion(fs, snapshotDir, region);
+ verifyRegionInfo(region, regionManifest);
}
+
if (!errorMsg.isEmpty()) {
throw new CorruptedSnapshotException(errorMsg);
}
+
+ // Verify Snapshot HFiles
+ SnapshotReferenceUtil.verifySnapshot(services.getConfiguration(), fs, manifest);
}
/**
- * Verify that the region (regioninfo, hfiles) are valid
- * @param fs the FileSystem instance
- * @param snapshotDir snapshot directory to check
+ * Verify that the regionInfo is valid
* @param region the region to check
+ * @param manifest snapshot manifest to inspect
*/
- private void verifyRegion(final FileSystem fs, final Path snapshotDir, final HRegionInfo region)
- throws IOException {
- // make sure we have region in the snapshot
- Path regionDir = new Path(snapshotDir, region.getEncodedName());
-
- // make sure we have the region info in the snapshot
- Path regionInfo = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE);
- // make sure the file exists
- if (!fs.exists(regionInfo)) {
- throw new CorruptedSnapshotException("No region info found for region:" + region, snapshot);
- }
-
- HRegionInfo found = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
- if (!region.equals(found)) {
- throw new CorruptedSnapshotException("Found region info (" + found
- + ") doesn't match expected region:" + region, snapshot);
- }
-
- // make sure we have the expected recovered edits files
- TakeSnapshotUtils.verifyRecoveredEdits(fs, snapshotDir, found, snapshot);
-
- // make sure we have all the expected store files
- SnapshotReferenceUtil.visitRegionStoreFiles(fs, regionDir, new FSVisitor.StoreFileVisitor() {
- public void storeFile(final String regionNameSuffix, final String family,
- final String hfileName) throws IOException {
- verifyStoreFile(snapshotDir, region, family, hfileName);
- }
- });
- }
-
- private void verifyStoreFile(final Path snapshotDir, final HRegionInfo regionInfo,
- final String family, final String fileName) throws IOException {
- Path refPath = null;
- if (StoreFileInfo.isReference(fileName)) {
- // If is a reference file check if the parent file is present in the snapshot
- Path snapshotHFilePath = new Path(new Path(
- new Path(snapshotDir, regionInfo.getEncodedName()), family), fileName);
- refPath = StoreFileInfo.getReferredToFile(snapshotHFilePath);
- if (!fs.exists(refPath)) {
- throw new CorruptedSnapshotException("Missing parent hfile for: " + fileName, snapshot);
- }
- }
-
- Path linkPath;
- if (refPath != null && HFileLink.isHFileLink(refPath)) {
- linkPath = new Path(family, refPath.getName());
- } else if (HFileLink.isHFileLink(fileName)) {
- linkPath = new Path(family, fileName);
- } else {
- linkPath = new Path(family, HFileLink.createHFileLinkName(tableName,
- regionInfo.getEncodedName(), fileName));
- }
-
- // check if the linked file exists (in the archive, or in the table dir)
- HFileLink link = new HFileLink(services.getConfiguration(), linkPath);
- if (!link.exists(fs)) {
- throw new CorruptedSnapshotException("Can't find hfile: " + fileName
- + " in the real (" + link.getOriginPath() + ") or archive (" + link.getArchivePath()
- + ") directory for the primary table.", snapshot);
+ private void verifyRegionInfo(final HRegionInfo region,
+ final SnapshotRegionManifest manifest) throws IOException {
+ HRegionInfo manifestRegionInfo = HRegionInfo.convert(manifest.getRegionInfo());
+ if (!region.equals(manifestRegionInfo)) {
+ String msg = "Manifest region info " + manifestRegionInfo +
+ "doesn't match expected region:" + region;
+ throw new CorruptedSnapshotException(msg, snapshot);
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
index 2e970a1..2b97505 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
@@ -51,6 +51,7 @@ import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils;
import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
/**
* Handler to Restore a snapshot.
@@ -120,9 +121,11 @@ public class RestoreSnapshotHandler extends TableEventHandler implements Snapsho
// 2. Execute the on-disk Restore
LOG.debug("Starting restore snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot));
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
+ SnapshotManifest manifest = SnapshotManifest.open(masterServices.getConfiguration(), fs,
+ snapshotDir, snapshot);
RestoreSnapshotHelper restoreHelper = new RestoreSnapshotHelper(
- masterServices.getConfiguration(), fs,
- snapshot, snapshotDir, hTableDescriptor, rootDir, monitor, status);
+ masterServices.getConfiguration(), fs, manifest,
+ this.hTableDescriptor, rootDir, monitor, status);
RestoreSnapshotHelper.RestoreMetaChanges metaChanges = restoreHelper.restoreHdfsRegions();
// 3. Forces all the RegionStates to be offline
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
index d2fe4a9..bd9e59f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
@@ -19,10 +19,14 @@ package org.apache.hadoop.hbase.master.snapshot;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.util.*;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.Timer;
+import java.util.TimerTask;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Lists;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -35,14 +39,13 @@ import org.apache.hadoop.hbase.Stoppable;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
import org.apache.hadoop.hbase.util.FSUtils;
-
/**
* Intelligently keep track of all the files for all the snapshots.
* <p>
* A cache of files is kept to avoid querying the {@link FileSystem} frequently. If there is a cache
* miss the directory modification time is used to ensure that we don't rescan directories that we
* already have in cache. We only check the modification times of the snapshot directories
- * (/hbase/.hbase-snapshot/[snapshot_name]) to determine if the files need to be loaded into the cache.
+ * (/hbase/.snapshot/[snapshot_name]) to determine if the files need to be loaded into the cache.
* <p>
* New snapshots will be added to the cache and deleted snapshots will be removed when we refresh
* the cache. If the files underneath a snapshot directory are changed, but not the snapshot itself,
@@ -60,8 +63,8 @@ import org.apache.hadoop.hbase.util.FSUtils;
* This allows you to only cache files under, for instance, all the logs in the .logs directory or
* all the files under all the regions.
* <p>
- * <tt>this</tt> also considers all running snapshots (those under /hbase/.hbase-snapshot/.tmp) as valid
- * snapshots but will not attempt to cache files from that directory.
+ * <tt>this</tt> also considers all running snapshots (those under /hbase/.snapshot/.tmp) as valid
+ * snapshots and will attempt to cache files from those snapshots as well.
* <p>
* Queries about a given file are thread-safe with respect to multiple queries and cache refreshes.
*/
@@ -162,42 +165,33 @@ public class SnapshotFileCache implements Stoppable {
* at that point, cache will still think the file system contains that file and return
* <tt>true</tt>, even if it is no longer present (false positive). However, if the file never was
* on the filesystem, we will never find it and always return <tt>false</tt>.
- * @param files file to check, NOTE: Relies that files are loaded from hdfs before method is called (NOT LAZY)
- * @return <tt>unReferencedFiles</tt> the collection of files that do not have snapshot references
+ * @param fileName file to check
+ * @return <tt>false</tt> if the file is not referenced in any current or running snapshot,
+ * <tt>true</tt> if the file is in the cache.
* @throws IOException if there is an unexpected error reaching the filesystem.
*/
// XXX this is inefficient to synchronize on the method, when what we really need to guard against
// is an illegal access to the cache. Really we could do a mutex-guarded pointer swap on the
// cache, but that seems overkill at the moment and isn't necessarily a bottleneck.
- public synchronized Iterable<FileStatus> getUnreferencedFiles(Iterable<FileStatus> files) throws IOException {
- List<FileStatus> unReferencedFiles = Lists.newArrayList();
- List<String> snapshotsInProgress = null;
- boolean refreshed = false;
- for (FileStatus file : files) {
- String fileName = file.getPath().getName();
- if (!refreshed && !cache.contains(fileName)) {
- refreshCache();
- refreshed = true;
- }
- if (cache.contains(fileName)) {
- continue;
- }
- if (snapshotsInProgress == null) {
- snapshotsInProgress = getSnapshotsInProgress();
- }
- if (snapshotsInProgress.contains(fileName)) {
- continue;
- }
- unReferencedFiles.add(file);
+ public synchronized boolean contains(String fileName) throws IOException {
+ boolean hasFile = this.cache.contains(fileName);
+ if (!hasFile) {
+ refreshCache();
+ // then check again
+ hasFile = this.cache.contains(fileName);
}
- return unReferencedFiles;
+ return hasFile;
}
private synchronized void refreshCache() throws IOException {
- // get the status of the snapshots directory
- FileStatus dirStatus;
+ long lastTimestamp = Long.MAX_VALUE;
+ boolean hasChanges = false;
+
+ // get the status of the snapshots directory and check if it is has changes
try {
- dirStatus = fs.getFileStatus(snapshotDir);
+ FileStatus dirStatus = fs.getFileStatus(snapshotDir);
+ lastTimestamp = dirStatus.getModificationTime();
+ hasChanges |= (lastTimestamp >= lastModifiedTime);
} catch (FileNotFoundException e) {
if (this.cache.size() > 0) {
LOG.error("Snapshot directory: " + snapshotDir + " doesn't exist");
@@ -205,7 +199,28 @@ public class SnapshotFileCache implements Stoppable {
return;
}
- if (dirStatus.getModificationTime() <= lastModifiedTime) {
+ // get the status of the snapshots temporary directory and check if it has changes
+ // The top-level directory timestamp is not updated, so we have to check the inner-level.
+ try {
+ Path snapshotTmpDir = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME);
+ FileStatus tempDirStatus = fs.getFileStatus(snapshotTmpDir);
+ lastTimestamp = Math.min(lastTimestamp, tempDirStatus.getModificationTime());
+ hasChanges |= (lastTimestamp >= lastModifiedTime);
+ if (!hasChanges) {
+ FileStatus[] tmpSnapshots = FSUtils.listStatus(fs, snapshotDir);
+ if (tmpSnapshots != null) {
+ for (FileStatus dirStatus: tmpSnapshots) {
+ lastTimestamp = Math.min(lastTimestamp, dirStatus.getModificationTime());
+ }
+ hasChanges |= (lastTimestamp >= lastModifiedTime);
+ }
+ }
+ } catch (FileNotFoundException e) {
+ // Nothing todo, if the tmp dir is empty
+ }
+
+ // if the snapshot directory wasn't modified since we last check, we are done
+ if (!hasChanges) {
return;
}
@@ -215,7 +230,7 @@ public class SnapshotFileCache implements Stoppable {
// However, snapshot directories are only created once, so this isn't an issue.
// 1. update the modified time
- this.lastModifiedTime = dirStatus.getModificationTime();
+ this.lastModifiedTime = lastTimestamp;
// 2.clear the cache
this.cache.clear();
@@ -235,7 +250,15 @@ public class SnapshotFileCache implements Stoppable {
// 3.1 iterate through the on-disk snapshots
for (FileStatus snapshot : snapshots) {
String name = snapshot.getPath().getName();
- if (!name.equals(SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME)) {
+ // its the tmp dir
+ if (name.equals(SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME)) {
+ // only add those files to the cache, but not to the known snapshots
+ FileStatus[] running = FSUtils.listStatus(fs, snapshot.getPath());
+ if (running == null) continue;
+ for (FileStatus run : running) {
+ this.cache.addAll(fileInspector.filesUnderSnapshot(run.getPath()));
+ }
+ } else {
SnapshotDirectoryInfo files = this.snapshots.remove(name);
// 3.1.1 if we don't know about the snapshot or its been modified, we need to update the files
// the latter could occur where I create a snapshot, then delete it, and then make a new
@@ -257,20 +280,6 @@ public class SnapshotFileCache implements Stoppable {
this.snapshots.putAll(known);
}
- @VisibleForTesting List<String> getSnapshotsInProgress() throws IOException {
- List<String> snapshotInProgress = Lists.newArrayList();
- // only add those files to the cache, but not to the known snapshots
- Path snapshotTmpDir = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME);
- // only add those files to the cache, but not to the known snapshots
- FileStatus[] running = FSUtils.listStatus(fs, snapshotTmpDir);
- if (running != null) {
- for (FileStatus run : running) {
- snapshotInProgress.addAll(fileInspector.filesUnderSnapshot(run.getPath()));
- }
- }
- return snapshotInProgress;
- }
-
/**
* Simple helper task that just periodically attempts to refresh the cache
*/
@@ -328,5 +337,4 @@ public class SnapshotFileCache implements Stoppable {
return this.lastModified < mtime;
}
}
-
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java
index 163bc31..562f682 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.hbase.master.snapshot;
import java.io.IOException;
import java.util.Collection;
-import java.util.Collections;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -56,22 +55,18 @@ public class SnapshotHFileCleaner extends BaseHFileCleanerDelegate {
private SnapshotFileCache cache;
@Override
- public synchronized Iterable<FileStatus> getDeletableFiles(Iterable<FileStatus> files) {
+ public synchronized boolean isFileDeletable(FileStatus fStat) {
try {
- return cache.getUnreferencedFiles(files);
+ return !cache.contains(fStat.getPath().getName());
} catch (IOException e) {
- LOG.error("Exception while checking if files were valid, keeping them just in case.", e);
- return Collections.emptyList();
+ LOG.error("Exception while checking if:" + fStat.getPath()
+ + " was valid, keeping it just in case.", e);
+ return false;
}
- }
-
- @Override
- protected boolean isFileDeletable(FileStatus fStat) {
- return false;
}
@Override
- public void setConf(Configuration conf) {
+ public void setConf(final Configuration conf) {
super.setConf(conf);
try {
long cacheRefreshPeriod = conf.getLong(HFILE_CACHE_REFRESH_PERIOD_CONF_KEY,
@@ -82,7 +77,7 @@ public class SnapshotHFileCleaner extends BaseHFileCleanerDelegate {
"snapshot-hfile-cleaner-cache-refresher", new SnapshotFileCache.SnapshotFileInspector() {
public Collection<String> filesUnderSnapshot(final Path snapshotDir)
throws IOException {
- return SnapshotReferenceUtil.getHFileNames(fs, snapshotDir);
+ return SnapshotReferenceUtil.getHFileNames(conf, fs, snapshotDir);
}
});
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotLogCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotLogCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotLogCleaner.java
index 7da64c5..cfe5e0d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotLogCleaner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotLogCleaner.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.hbase.master.snapshot;
import java.io.IOException;
import java.util.Collection;
-import java.util.Collections;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -55,13 +54,14 @@ public class SnapshotLogCleaner extends BaseLogCleanerDelegate {
private SnapshotFileCache cache;
@Override
- public synchronized Iterable<FileStatus> getDeletableFiles(Iterable<FileStatus> files) {
- if (null == cache) return Collections.emptyList();
+ public synchronized boolean isFileDeletable(FileStatus fStat) {
try {
- return cache.getUnreferencedFiles(files);
+ if (null == cache) return false;
+ return !cache.contains(fStat.getPath().getName());
} catch (IOException e) {
- LOG.error("Exception while checking if files were valid, keeping them just in case.", e);
- return Collections.emptyList();
+ LOG.error("Exception while checking if:" + fStat.getPath()
+ + " was valid, keeping it just in case.", e);
+ return false;
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
index f45f660..94bbf02 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
@@ -70,11 +70,11 @@ import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException;
import org.apache.hadoop.hbase.snapshot.SnapshotExistsException;
+import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
import org.apache.hadoop.hbase.snapshot.TablePartiallyOpenException;
import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.util.FSTableDescriptors;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.zookeeper.KeeperException;
@@ -160,6 +160,11 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
private Path rootDir;
private ExecutorService executorService;
+ /**
+ * Snapshot layout version to use when writing a new snapshot.
+ */
+ private int snapshotLayoutVersion = SnapshotDescriptionUtils.SNAPSHOT_LAYOUT_LATEST_FORMAT;
+
public SnapshotManager() {}
/**
@@ -177,6 +182,9 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
this.rootDir = master.getMasterFileSystem().getRootDir();
checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());
+ this.snapshotLayoutVersion = SnapshotDescriptionUtils.getDefaultSnapshotLayoutFormat(
+ master.getConfiguration());
+
this.coordinator = coordinator;
this.executorService = pool;
resetTempDir();
@@ -542,9 +550,12 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
+ "' doesn't exist, can't take snapshot.", snapshot);
}
- // set the snapshot version, now that we are ready to take it
- snapshot = snapshot.toBuilder().setVersion(SnapshotDescriptionUtils.SNAPSHOT_LAYOUT_VERSION)
- .build();
+ // if not specified, set the snapshot format
+ if (!snapshot.hasVersion()) {
+ snapshot = snapshot.toBuilder()
+ .setVersion(snapshotLayoutVersion)
+ .build();
+ }
// call pre coproc hook
MasterCoprocessorHost cpHost = master.getCoprocessorHost();
@@ -678,15 +689,16 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
// read snapshot information
SnapshotDescription fsSnapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
- HTableDescriptor snapshotTableDesc =
- FSTableDescriptors.getTableDescriptorFromFs(fs, snapshotDir);
+ SnapshotManifest manifest = SnapshotManifest.open(master.getConfiguration(), fs,
+ snapshotDir, fsSnapshot);
+ HTableDescriptor snapshotTableDesc = manifest.getTableDescriptor();
TableName tableName = TableName.valueOf(reqSnapshot.getTable());
// stop tracking "abandoned" handlers
cleanupSentinels();
// Verify snapshot validity
- SnapshotReferenceUtil.verifySnapshot(master.getConfiguration(), fs, snapshotDir, fsSnapshot);
+ SnapshotReferenceUtil.verifySnapshot(master.getConfiguration(), fs, manifest);
// Execute the restore/clone operation
if (MetaReader.tableExists(master.getCatalogTracker(), tableName)) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
index 19c03f9..4d6182b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
@@ -48,14 +48,11 @@ import org.apache.hadoop.hbase.master.TableLockManager.TableLock;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
import org.apache.hadoop.hbase.monitoring.TaskMonitor;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils;
-import org.apache.hadoop.hbase.snapshot.CopyRecoveredEditsTask;
-import org.apache.hadoop.hbase.snapshot.ReferenceRegionHFilesTask;
import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
-import org.apache.hadoop.hbase.snapshot.TableInfoCopyTask;
+import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
+import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.zookeeper.KeeperException;
@@ -88,6 +85,9 @@ public abstract class TakeSnapshotHandler extends EventHandler implements Snapsh
protected final TableLock tableLock;
protected final MonitoredTask status;
protected final TableName snapshotTable;
+ protected final SnapshotManifest snapshotManifest;
+
+ protected HTableDescriptor htd;
/**
* @param snapshot descriptor of the snapshot to take
@@ -107,6 +107,7 @@ public abstract class TakeSnapshotHandler extends EventHandler implements Snapsh
this.snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
this.workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
this.monitor = new ForeignExceptionDispatcher(snapshot.getName());
+ this.snapshotManifest = SnapshotManifest.create(conf, fs, workingDir, snapshot, monitor);
this.tableLockManager = master.getTableLockManager();
this.tableLock = this.tableLockManager.writeLock(
@@ -136,7 +137,7 @@ public abstract class TakeSnapshotHandler extends EventHandler implements Snapsh
// case of exceptions
boolean success = false;
try {
- loadTableDescriptor(); // check that .tableinfo is present
+ this.htd = loadTableDescriptor(); // check that .tableinfo is present
success = true;
} finally {
if (!success) {
@@ -162,8 +163,8 @@ public abstract class TakeSnapshotHandler extends EventHandler implements Snapsh
// an external exception that gets captured here.
// write down the snapshot info in the working directory
- SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, workingDir, this.fs);
- new TableInfoCopyTask(monitor, snapshot, fs, rootDir).call();
+ SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, workingDir, fs);
+ snapshotManifest.addTableDescriptor(this.htd);
monitor.rethrowException();
List<Pair<HRegionInfo, ServerName>> regionsAndLocations =
@@ -184,16 +185,19 @@ public abstract class TakeSnapshotHandler extends EventHandler implements Snapsh
}
}
+ // flush the in-memory state, and write the single manifest
+ status.setStatus("Consolidate snapshot: " + snapshot.getName());
+ snapshotManifest.consolidate();
+
// verify the snapshot is valid
status.setStatus("Verifying snapshot: " + snapshot.getName());
verifier.verifySnapshot(this.workingDir, serverNames);
// complete the snapshot, atomically moving from tmp to .snapshot dir.
completeSnapshot(this.snapshotDir, this.workingDir, this.fs);
- status.markComplete("Snapshot " + snapshot.getName() + " of table " + snapshotTable
- + " completed");
- LOG.info("Snapshot " + snapshot.getName() + " of table " + snapshotTable
- + " completed");
+ msg = "Snapshot " + snapshot.getName() + " of table " + snapshotTable + " completed";
+ status.markComplete(msg);
+ LOG.info(msg);
metricsSnapshot.addSnapshot(status.getCompletionTimestamp() - status.getStartTime());
} catch (Exception e) {
status.abort("Failed to complete snapshot " + snapshot.getName() + " on table " +
@@ -204,8 +208,7 @@ public abstract class TakeSnapshotHandler extends EventHandler implements Snapsh
ForeignException ee = new ForeignException(reason, e);
monitor.receive(ee);
// need to mark this completed to close off and allow cleanup to happen.
- cancel("Failed to take snapshot '" + ClientSnapshotDescriptionUtils.toString(snapshot)
- + "' due to exception");
+ cancel(reason);
} finally {
LOG.debug("Launching cleanup of working dir:" + workingDir);
try {
@@ -262,26 +265,10 @@ public abstract class TakeSnapshotHandler extends EventHandler implements Snapsh
*/
protected void snapshotDisabledRegion(final HRegionInfo regionInfo)
throws IOException {
- // 2 copy the regionInfo files to the snapshot
- HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(conf, fs,
- workingDir, regionInfo);
-
- // check for error for each region
- monitor.rethrowException();
-
- // 2 for each region, copy over its recovered.edits directory
- Path regionDir = HRegion.getRegionDir(rootDir, regionInfo);
- Path snapshotRegionDir = regionFs.getRegionDir();
- new CopyRecoveredEditsTask(snapshot, monitor, fs, regionDir, snapshotRegionDir).call();
- monitor.rethrowException();
- status.setStatus("Completed copying recovered edits for offline snapshot of table: "
- + snapshotTable);
-
- // 2 reference all the files in the region
- new ReferenceRegionHFilesTask(snapshot, monitor, regionDir, fs, snapshotRegionDir).call();
+ snapshotManifest.addRegion(FSUtils.getTableDir(rootDir, snapshotTable), regionInfo);
monitor.rethrowException();
- status.setStatus("Completed referencing HFiles for offline snapshot of table: " +
- snapshotTable);
+ status.setStatus("Completed referencing HFiles for offline region " + regionInfo.toString() +
+ " of table: " + snapshotTable);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureManager.java
index aa3dc78..8adc6db 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureManager.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.hbase.procedure;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hbase.regionserver.snapshot.RegionServerSnapshotManager;
@InterfaceAudience.Private
@InterfaceStability.Evolving
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index f80b01f..7169904 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -129,6 +129,7 @@ import org.apache.hadoop.hbase.regionserver.wal.HLogSplitter.MutationReplay;
import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.CancelableProgressable;
import org.apache.hadoop.hbase.util.ClassSize;
@@ -195,7 +196,7 @@ public class HRegion implements HeapSize { // , Writable{
public static final String LOAD_CFS_ON_DEMAND_CONFIG_KEY =
"hbase.hregion.scan.loadColumnFamiliesOnDemand";
-
+
/**
* This is the global default value for durability. All tables/mutations not
* defining a durability or using USE_DEFAULT will default to this value.
@@ -474,7 +475,7 @@ public class HRegion implements HeapSize { // , Writable{
private RegionServerAccounting rsAccounting;
private List<Pair<Long, Long>> recentFlushes = new ArrayList<Pair<Long,Long>>();
private long flushCheckInterval;
- // flushPerChanges is to prevent too many changes in memstore
+ // flushPerChanges is to prevent too many changes in memstore
private long flushPerChanges;
private long blockingMemStoreSize;
final long threadWakeFrequency;
@@ -574,7 +575,7 @@ public class HRegion implements HeapSize { // , Writable{
throw new IllegalArgumentException(MEMSTORE_FLUSH_PER_CHANGES + " can not exceed "
+ MAX_FLUSH_PER_CHANGES);
}
-
+
this.rowLockWaitDuration = conf.getInt("hbase.rowlock.wait.duration",
DEFAULT_ROWLOCK_WAIT_DURATION);
@@ -808,7 +809,7 @@ public class HRegion implements HeapSize { // , Writable{
for (Store store : this.stores.values()) {
try {
store.close();
- } catch (IOException e) {
+ } catch (IOException e) {
LOG.warn(e.getMessage());
}
}
@@ -1145,7 +1146,7 @@ public class HRegion implements HeapSize { // , Writable{
// so we do not lose data
throw new DroppedSnapshotException("Failed clearing memory after " +
actualFlushes + " attempts on region: " + Bytes.toStringBinary(getRegionName()));
- }
+ }
LOG.info("Running extra flush, " + actualFlushes +
" (carrying snapshot?) " + this);
}
@@ -2807,59 +2808,12 @@ public class HRegion implements HeapSize { // , Writable{
*/
public void addRegionToSnapshot(SnapshotDescription desc,
ForeignExceptionSnare exnSnare) throws IOException {
- // This should be "fast" since we don't rewrite store files but instead
- // back up the store files by creating a reference
- Path rootDir = FSUtils.getRootDir(this.rsServices.getConfiguration());
+ Path rootDir = FSUtils.getRootDir(conf);
Path snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
- // 1. dump region meta info into the snapshot directory
- LOG.debug("Storing region-info for snapshot.");
- HRegionFileSystem snapshotRegionFs = HRegionFileSystem.createRegionOnFileSystem(conf,
- this.fs.getFileSystem(), snapshotDir, getRegionInfo());
-
- // 2. iterate through all the stores in the region
- LOG.debug("Creating references for hfiles");
-
- // This ensures that we have an atomic view of the directory as long as we have < ls limit
- // (batch size of the files in a directory) on the namenode. Otherwise, we get back the files in
- // batches and may miss files being added/deleted. This could be more robust (iteratively
- // checking to see if we have all the files until we are sure), but the limit is currently 1000
- // files/batch, far more than the number of store files under a single column family.
- for (Store store : stores.values()) {
- // 2.1. build the snapshot reference directory for the store
- Path dstStoreDir = snapshotRegionFs.getStoreDir(store.getFamily().getNameAsString());
- List<StoreFile> storeFiles = new ArrayList<StoreFile>(store.getStorefiles());
- if (LOG.isDebugEnabled()) {
- LOG.debug("Adding snapshot references for " + storeFiles + " hfiles");
- }
-
- // 2.2. iterate through all the store's files and create "references".
- int sz = storeFiles.size();
- for (int i = 0; i < sz; i++) {
- if (exnSnare != null) {
- exnSnare.rethrowException();
- }
- StoreFile storeFile = storeFiles.get(i);
- Path file = storeFile.getPath();
-
- LOG.debug("Creating reference for file (" + (i+1) + "/" + sz + ") : " + file);
- Path referenceFile = new Path(dstStoreDir, file.getName());
- boolean success = true;
- if (storeFile.isReference()) {
- // write the Reference object to the snapshot
- storeFile.getFileInfo().getReference().write(fs.getFileSystem(), referenceFile);
- } else {
- // create "reference" to this store file. It is intentionally an empty file -- all
- // necessary information is captured by its fs location and filename. This allows us to
- // only figure out what needs to be done via a single nn operation (instead of having to
- // open and read the files as well).
- success = fs.getFileSystem().createNewFile(referenceFile);
- }
- if (!success) {
- throw new IOException("Failed to create reference file:" + referenceFile);
- }
- }
- }
+ SnapshotManifest manifest = SnapshotManifest.create(conf, getFilesystem(),
+ snapshotDir, desc, exnSnare);
+ manifest.addRegion(this);
}
/**
@@ -4028,14 +3982,14 @@ public class HRegion implements HeapSize { // , Writable{
isStopRow(nextKv.getBuffer(), nextKv.getRowOffset(), nextKv.getRowLength());
// save that the row was empty before filters applied to it.
final boolean isEmptyRow = results.isEmpty();
-
+
// We have the part of the row necessary for filtering (all of it, usually).
// First filter with the filterRow(List).
FilterWrapper.FilterRowRetCode ret = FilterWrapper.FilterRowRetCode.NOT_CALLED;
if (filter != null && filter.hasFilterRow()) {
ret = filter.filterRowCellsWithRet(results);
}
-
+
if ((isEmptyRow || ret == FilterWrapper.FilterRowRetCode.EXCLUDE) || filterRow()) {
results.clear();
boolean moreRows = nextRow(currentRow, offset, length);
@@ -4103,7 +4057,7 @@ public class HRegion implements HeapSize { // , Writable{
return filter != null && (!filter.hasFilterRow())
&& filter.filterRow();
}
-
+
private boolean filterRowKey(byte[] row, int offset, short length) throws IOException {
return filter != null
&& filter.filterRowKey(row, offset, length);
@@ -5807,7 +5761,7 @@ public class HRegion implements HeapSize { // , Writable{
* modifies data. It has to be called just before a try.
* #closeRegionOperation needs to be called in the try's finally block
* Acquires a read lock and checks if the region is closing or closed.
- * @throws IOException
+ * @throws IOException
*/
public void startRegionOperation() throws IOException {
startRegionOperation(Operation.ANY);
@@ -5815,7 +5769,7 @@ public class HRegion implements HeapSize { // , Writable{
/**
* @param op The operation is about to be taken on the region
- * @throws IOException
+ * @throws IOException
*/
protected void startRegionOperation(Operation op) throws IOException {
switch (op) {
@@ -5865,7 +5819,7 @@ public class HRegion implements HeapSize { // , Writable{
/**
* Closes the lock. This needs to be called in the finally block corresponding
* to the try block of #startRegionOperation
- * @throws IOException
+ * @throws IOException
*/
public void closeRegionOperation() throws IOException {
closeRegionOperation(Operation.ANY);
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
index af71914..c5091c2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
@@ -76,7 +76,7 @@ public class HRegionFileSystem {
private final Configuration conf;
private final Path tableDir;
private final FileSystem fs;
-
+
/**
* In order to handle NN connectivity hiccups, one need to retry non-idempotent operation at the
* client level.
@@ -148,7 +148,7 @@ public class HRegionFileSystem {
* @param familyName Column Family Name
* @return {@link Path} to the directory of the specified family
*/
- Path getStoreDir(final String familyName) {
+ public Path getStoreDir(final String familyName) {
return new Path(this.getRegionDir(), familyName);
}
@@ -175,20 +175,31 @@ public class HRegionFileSystem {
return getStoreFiles(Bytes.toString(familyName));
}
+ public Collection<StoreFileInfo> getStoreFiles(final String familyName) throws IOException {
+ return getStoreFiles(familyName, true);
+ }
+
/**
* Returns the store files available for the family.
* This methods performs the filtering based on the valid store files.
* @param familyName Column Family Name
* @return a set of {@link StoreFileInfo} for the specified family.
*/
- public Collection<StoreFileInfo> getStoreFiles(final String familyName) throws IOException {
+ public Collection<StoreFileInfo> getStoreFiles(final String familyName, final boolean validate)
+ throws IOException {
Path familyDir = getStoreDir(familyName);
FileStatus[] files = FSUtils.listStatus(this.fs, familyDir);
- if (files == null) return null;
+ if (files == null) {
+ LOG.debug("No StoreFiles for: " + familyDir);
+ return null;
+ }
ArrayList<StoreFileInfo> storeFiles = new ArrayList<StoreFileInfo>(files.length);
for (FileStatus status: files) {
- if (!StoreFileInfo.isValid(status)) continue;
+ if (validate && !StoreFileInfo.isValid(status)) {
+ LOG.warn("Invalid StoreFile: " + status.getPath());
+ continue;
+ }
storeFiles.add(new StoreFileInfo(this.conf, this.fs, status));
}
@@ -353,7 +364,7 @@ public class HRegionFileSystem {
Path storeDir = getStoreDir(familyName);
if(!fs.exists(storeDir) && !createDir(storeDir))
throw new IOException("Failed creating " + storeDir);
-
+
String name = buildPath.getName();
if (generateNewName) {
name = generateUniqueName((seqNum < 0) ? null : "_SeqId_" + seqNum + "_");
@@ -555,7 +566,7 @@ public class HRegionFileSystem {
*/
Path splitStoreFile(final HRegionInfo hri, final String familyName,
final StoreFile f, final byte[] splitRow, final boolean top) throws IOException {
-
+
// Check whether the split row lies in the range of the store file
// If it is outside the range, return directly.
if (top) {
@@ -564,7 +575,7 @@ public class HRegionFileSystem {
byte[] lastKey = f.createReader().getLastKey();
// If lastKey is null means storefile is empty.
if (lastKey == null) return null;
- if (f.getReader().getComparator().compareFlatKey(splitKey.getBuffer(),
+ if (f.getReader().getComparator().compareFlatKey(splitKey.getBuffer(),
splitKey.getKeyOffset(), splitKey.getKeyLength(), lastKey, 0, lastKey.length) > 0) {
return null;
}
@@ -574,14 +585,14 @@ public class HRegionFileSystem {
byte[] firstKey = f.createReader().getFirstKey();
// If firstKey is null means storefile is empty.
if (firstKey == null) return null;
- if (f.getReader().getComparator().compareFlatKey(splitKey.getBuffer(),
+ if (f.getReader().getComparator().compareFlatKey(splitKey.getBuffer(),
splitKey.getKeyOffset(), splitKey.getKeyLength(), firstKey, 0, firstKey.length) < 0) {
return null;
- }
+ }
}
-
+
f.getReader().close(true);
-
+
Path splitDir = new Path(getSplitsDir(hri), familyName);
// A reference to the bottom half of the hsf store file.
Reference r =
@@ -680,7 +691,7 @@ public class HRegionFileSystem {
* Commit a merged region, moving it from the merges temporary directory to
* the proper location in the filesystem.
* @param mergedRegionInfo merged region {@link HRegionInfo}
- * @throws IOException
+ * @throws IOException
*/
void commitMergedRegion(final HRegionInfo mergedRegionInfo) throws IOException {
Path regionDir = new Path(this.tableDir, mergedRegionInfo.getEncodedName());
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
index a500f0d..9a76932 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
@@ -222,7 +222,7 @@ public class StoreFile {
* @return the StoreFile object associated to this StoreFile.
* null if the StoreFile is not a reference.
*/
- StoreFileInfo getFileInfo() {
+ public StoreFileInfo getFileInfo() {
return this.fileInfo;
}
@@ -614,7 +614,7 @@ public class StoreFile {
if (comparator == null) {
comparator = KeyValue.COMPARATOR;
}
- return new Writer(fs, filePath,
+ return new Writer(fs, filePath,
conf, cacheConf, comparator, bloomType, maxKeyCount, favoredNodes, fileContext);
}
}
@@ -694,7 +694,7 @@ public class StoreFile {
/** Bytes per Checksum */
protected int bytesPerChecksum;
-
+
TimeRangeTracker timeRangeTracker = new TimeRangeTracker();
/* isTimeRangeTrackerSet keeps track if the timeRange has already been set
* When flushing a memstore, we set TimeRange and use this variable to
@@ -723,7 +723,7 @@ public class StoreFile {
final Configuration conf,
CacheConfig cacheConf,
final KVComparator comparator, BloomType bloomType, long maxKeys,
- InetSocketAddress[] favoredNodes, HFileContext fileContext)
+ InetSocketAddress[] favoredNodes, HFileContext fileContext)
throws IOException {
writer = HFile.getWriterFactory(conf, cacheConf)
.withPath(fs, path)
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
index 2d8a8ea..8bf1796 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
@@ -144,7 +144,7 @@ public class StoreFileInfo {
* @return the Reference object associated to this StoreFileInfo.
* null if the StoreFile is not a reference.
*/
- Reference getReference() {
+ public Reference getReference() {
return this.reference;
}
@@ -223,6 +223,20 @@ public class StoreFileInfo {
*/
public HDFSBlocksDistribution computeHDFSBlocksDistribution(final FileSystem fs)
throws IOException {
+ FileStatus status = getReferencedFileStatus(fs);
+ if (this.reference != null) {
+ return computeRefFileHDFSBlockDistribution(fs, reference, status);
+ } else {
+ return FSUtils.computeHDFSBlocksDistribution(fs, status, 0, status.getLen());
+ }
+ }
+
+ /**
+ * Get the {@link FileStatus} of the file referenced by this StoreFileInfo
+ * @param fs The current file system to use.
+ * @return The {@link FileStatus} of the file referenced by this StoreFileInfo
+ */
+ public FileStatus getReferencedFileStatus(final FileSystem fs) throws IOException {
FileStatus status;
if (this.reference != null) {
if (this.link != null) {
@@ -233,7 +247,6 @@ public class StoreFileInfo {
Path referencePath = getReferredToFile(this.getPath());
status = fs.getFileStatus(referencePath);
}
- return computeRefFileHDFSBlockDistribution(fs, reference, status);
} else {
if (this.link != null) {
// HFileLink
@@ -241,8 +254,8 @@ public class StoreFileInfo {
} else {
status = this.fileStatus;
}
- return FSUtils.computeHDFSBlocksDistribution(fs, status, 0, status.getLen());
}
+ return status;
}
/** @return The {@link Path} of the file */
[6/7] HBASE-11742 (Backport HBASE-7987 and HBASE-11185 to 0.98)
(Esteban Gutierrez and Matteo Bertozzi)
Posted by ap...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/SnapshotProtos.java
----------------------------------------------------------------------
diff --git a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/SnapshotProtos.java b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/SnapshotProtos.java
new file mode 100644
index 0000000..75c5502
--- /dev/null
+++ b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/SnapshotProtos.java
@@ -0,0 +1,4787 @@
+// Generated by the protocol buffer compiler. DO NOT EDIT!
+// source: Snapshot.proto
+
+package org.apache.hadoop.hbase.protobuf.generated;
+
+public final class SnapshotProtos {
+ private SnapshotProtos() {}
+ public static void registerAllExtensions(
+ com.google.protobuf.ExtensionRegistry registry) {
+ }
+ public interface SnapshotFileInfoOrBuilder
+ extends com.google.protobuf.MessageOrBuilder {
+
+ // required .SnapshotFileInfo.Type type = 1;
+ /**
+ * <code>required .SnapshotFileInfo.Type type = 1;</code>
+ */
+ boolean hasType();
+ /**
+ * <code>required .SnapshotFileInfo.Type type = 1;</code>
+ */
+ org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo.Type getType();
+
+ // optional string hfile = 3;
+ /**
+ * <code>optional string hfile = 3;</code>
+ */
+ boolean hasHfile();
+ /**
+ * <code>optional string hfile = 3;</code>
+ */
+ java.lang.String getHfile();
+ /**
+ * <code>optional string hfile = 3;</code>
+ */
+ com.google.protobuf.ByteString
+ getHfileBytes();
+
+ // optional string wal_server = 4;
+ /**
+ * <code>optional string wal_server = 4;</code>
+ */
+ boolean hasWalServer();
+ /**
+ * <code>optional string wal_server = 4;</code>
+ */
+ java.lang.String getWalServer();
+ /**
+ * <code>optional string wal_server = 4;</code>
+ */
+ com.google.protobuf.ByteString
+ getWalServerBytes();
+
+ // optional string wal_name = 5;
+ /**
+ * <code>optional string wal_name = 5;</code>
+ */
+ boolean hasWalName();
+ /**
+ * <code>optional string wal_name = 5;</code>
+ */
+ java.lang.String getWalName();
+ /**
+ * <code>optional string wal_name = 5;</code>
+ */
+ com.google.protobuf.ByteString
+ getWalNameBytes();
+ }
+ /**
+ * Protobuf type {@code SnapshotFileInfo}
+ */
+ public static final class SnapshotFileInfo extends
+ com.google.protobuf.GeneratedMessage
+ implements SnapshotFileInfoOrBuilder {
+ // Use SnapshotFileInfo.newBuilder() to construct.
+ private SnapshotFileInfo(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
+ super(builder);
+ this.unknownFields = builder.getUnknownFields();
+ }
+ private SnapshotFileInfo(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+ private static final SnapshotFileInfo defaultInstance;
+ public static SnapshotFileInfo getDefaultInstance() {
+ return defaultInstance;
+ }
+
+ public SnapshotFileInfo getDefaultInstanceForType() {
+ return defaultInstance;
+ }
+
+ private final com.google.protobuf.UnknownFieldSet unknownFields;
+ @java.lang.Override
+ public final com.google.protobuf.UnknownFieldSet
+ getUnknownFields() {
+ return this.unknownFields;
+ }
+ private SnapshotFileInfo(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ initFields();
+ int mutable_bitField0_ = 0;
+ com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+ com.google.protobuf.UnknownFieldSet.newBuilder();
+ try {
+ boolean done = false;
+ while (!done) {
+ int tag = input.readTag();
+ switch (tag) {
+ case 0:
+ done = true;
+ break;
+ default: {
+ if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+ done = true;
+ }
+ break;
+ }
+ case 8: {
+ int rawValue = input.readEnum();
+ org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo.Type value = org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo.Type.valueOf(rawValue);
+ if (value == null) {
+ unknownFields.mergeVarintField(1, rawValue);
+ } else {
+ bitField0_ |= 0x00000001;
+ type_ = value;
+ }
+ break;
+ }
+ case 26: {
+ bitField0_ |= 0x00000002;
+ hfile_ = input.readBytes();
+ break;
+ }
+ case 34: {
+ bitField0_ |= 0x00000004;
+ walServer_ = input.readBytes();
+ break;
+ }
+ case 42: {
+ bitField0_ |= 0x00000008;
+ walName_ = input.readBytes();
+ break;
+ }
+ }
+ }
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ throw e.setUnfinishedMessage(this);
+ } catch (java.io.IOException e) {
+ throw new com.google.protobuf.InvalidProtocolBufferException(
+ e.getMessage()).setUnfinishedMessage(this);
+ } finally {
+ this.unknownFields = unknownFields.build();
+ makeExtensionsImmutable();
+ }
+ }
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.internal_static_SnapshotFileInfo_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.internal_static_SnapshotFileInfo_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo.class, org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo.Builder.class);
+ }
+
+ public static com.google.protobuf.Parser<SnapshotFileInfo> PARSER =
+ new com.google.protobuf.AbstractParser<SnapshotFileInfo>() {
+ public SnapshotFileInfo parsePartialFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return new SnapshotFileInfo(input, extensionRegistry);
+ }
+ };
+
+ @java.lang.Override
+ public com.google.protobuf.Parser<SnapshotFileInfo> getParserForType() {
+ return PARSER;
+ }
+
+ /**
+ * Protobuf enum {@code SnapshotFileInfo.Type}
+ */
+ public enum Type
+ implements com.google.protobuf.ProtocolMessageEnum {
+ /**
+ * <code>HFILE = 1;</code>
+ */
+ HFILE(0, 1),
+ /**
+ * <code>WAL = 2;</code>
+ */
+ WAL(1, 2),
+ ;
+
+ /**
+ * <code>HFILE = 1;</code>
+ */
+ public static final int HFILE_VALUE = 1;
+ /**
+ * <code>WAL = 2;</code>
+ */
+ public static final int WAL_VALUE = 2;
+
+
+ public final int getNumber() { return value; }
+
+ public static Type valueOf(int value) {
+ switch (value) {
+ case 1: return HFILE;
+ case 2: return WAL;
+ default: return null;
+ }
+ }
+
+ public static com.google.protobuf.Internal.EnumLiteMap<Type>
+ internalGetValueMap() {
+ return internalValueMap;
+ }
+ private static com.google.protobuf.Internal.EnumLiteMap<Type>
+ internalValueMap =
+ new com.google.protobuf.Internal.EnumLiteMap<Type>() {
+ public Type findValueByNumber(int number) {
+ return Type.valueOf(number);
+ }
+ };
+
+ public final com.google.protobuf.Descriptors.EnumValueDescriptor
+ getValueDescriptor() {
+ return getDescriptor().getValues().get(index);
+ }
+ public final com.google.protobuf.Descriptors.EnumDescriptor
+ getDescriptorForType() {
+ return getDescriptor();
+ }
+ public static final com.google.protobuf.Descriptors.EnumDescriptor
+ getDescriptor() {
+ return org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo.getDescriptor().getEnumTypes().get(0);
+ }
+
+ private static final Type[] VALUES = values();
+
+ public static Type valueOf(
+ com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
+ if (desc.getType() != getDescriptor()) {
+ throw new java.lang.IllegalArgumentException(
+ "EnumValueDescriptor is not for this type.");
+ }
+ return VALUES[desc.getIndex()];
+ }
+
+ private final int index;
+ private final int value;
+
+ private Type(int index, int value) {
+ this.index = index;
+ this.value = value;
+ }
+
+ // @@protoc_insertion_point(enum_scope:SnapshotFileInfo.Type)
+ }
+
+ private int bitField0_;
+ // required .SnapshotFileInfo.Type type = 1;
+ public static final int TYPE_FIELD_NUMBER = 1;
+ private org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo.Type type_;
+ /**
+ * <code>required .SnapshotFileInfo.Type type = 1;</code>
+ */
+ public boolean hasType() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>required .SnapshotFileInfo.Type type = 1;</code>
+ */
+ public org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo.Type getType() {
+ return type_;
+ }
+
+ // optional string hfile = 3;
+ public static final int HFILE_FIELD_NUMBER = 3;
+ private java.lang.Object hfile_;
+ /**
+ * <code>optional string hfile = 3;</code>
+ */
+ public boolean hasHfile() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ /**
+ * <code>optional string hfile = 3;</code>
+ */
+ public java.lang.String getHfile() {
+ java.lang.Object ref = hfile_;
+ if (ref instanceof java.lang.String) {
+ return (java.lang.String) ref;
+ } else {
+ com.google.protobuf.ByteString bs =
+ (com.google.protobuf.ByteString) ref;
+ java.lang.String s = bs.toStringUtf8();
+ if (bs.isValidUtf8()) {
+ hfile_ = s;
+ }
+ return s;
+ }
+ }
+ /**
+ * <code>optional string hfile = 3;</code>
+ */
+ public com.google.protobuf.ByteString
+ getHfileBytes() {
+ java.lang.Object ref = hfile_;
+ if (ref instanceof java.lang.String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ hfile_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+
+ // optional string wal_server = 4;
+ public static final int WAL_SERVER_FIELD_NUMBER = 4;
+ private java.lang.Object walServer_;
+ /**
+ * <code>optional string wal_server = 4;</code>
+ */
+ public boolean hasWalServer() {
+ return ((bitField0_ & 0x00000004) == 0x00000004);
+ }
+ /**
+ * <code>optional string wal_server = 4;</code>
+ */
+ public java.lang.String getWalServer() {
+ java.lang.Object ref = walServer_;
+ if (ref instanceof java.lang.String) {
+ return (java.lang.String) ref;
+ } else {
+ com.google.protobuf.ByteString bs =
+ (com.google.protobuf.ByteString) ref;
+ java.lang.String s = bs.toStringUtf8();
+ if (bs.isValidUtf8()) {
+ walServer_ = s;
+ }
+ return s;
+ }
+ }
+ /**
+ * <code>optional string wal_server = 4;</code>
+ */
+ public com.google.protobuf.ByteString
+ getWalServerBytes() {
+ java.lang.Object ref = walServer_;
+ if (ref instanceof java.lang.String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ walServer_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+
+ // optional string wal_name = 5;
+ public static final int WAL_NAME_FIELD_NUMBER = 5;
+ private java.lang.Object walName_;
+ /**
+ * <code>optional string wal_name = 5;</code>
+ */
+ public boolean hasWalName() {
+ return ((bitField0_ & 0x00000008) == 0x00000008);
+ }
+ /**
+ * <code>optional string wal_name = 5;</code>
+ */
+ public java.lang.String getWalName() {
+ java.lang.Object ref = walName_;
+ if (ref instanceof java.lang.String) {
+ return (java.lang.String) ref;
+ } else {
+ com.google.protobuf.ByteString bs =
+ (com.google.protobuf.ByteString) ref;
+ java.lang.String s = bs.toStringUtf8();
+ if (bs.isValidUtf8()) {
+ walName_ = s;
+ }
+ return s;
+ }
+ }
+ /**
+ * <code>optional string wal_name = 5;</code>
+ */
+ public com.google.protobuf.ByteString
+ getWalNameBytes() {
+ java.lang.Object ref = walName_;
+ if (ref instanceof java.lang.String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ walName_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+
+ private void initFields() {
+ type_ = org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo.Type.HFILE;
+ hfile_ = "";
+ walServer_ = "";
+ walName_ = "";
+ }
+ private byte memoizedIsInitialized = -1;
+ public final boolean isInitialized() {
+ byte isInitialized = memoizedIsInitialized;
+ if (isInitialized != -1) return isInitialized == 1;
+
+ if (!hasType()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ memoizedIsInitialized = 1;
+ return true;
+ }
+
+ public void writeTo(com.google.protobuf.CodedOutputStream output)
+ throws java.io.IOException {
+ getSerializedSize();
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ output.writeEnum(1, type_.getNumber());
+ }
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ output.writeBytes(3, getHfileBytes());
+ }
+ if (((bitField0_ & 0x00000004) == 0x00000004)) {
+ output.writeBytes(4, getWalServerBytes());
+ }
+ if (((bitField0_ & 0x00000008) == 0x00000008)) {
+ output.writeBytes(5, getWalNameBytes());
+ }
+ getUnknownFields().writeTo(output);
+ }
+
+ private int memoizedSerializedSize = -1;
+ public int getSerializedSize() {
+ int size = memoizedSerializedSize;
+ if (size != -1) return size;
+
+ size = 0;
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeEnumSize(1, type_.getNumber());
+ }
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(3, getHfileBytes());
+ }
+ if (((bitField0_ & 0x00000004) == 0x00000004)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(4, getWalServerBytes());
+ }
+ if (((bitField0_ & 0x00000008) == 0x00000008)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(5, getWalNameBytes());
+ }
+ size += getUnknownFields().getSerializedSize();
+ memoizedSerializedSize = size;
+ return size;
+ }
+
+ private static final long serialVersionUID = 0L;
+ @java.lang.Override
+ protected java.lang.Object writeReplace()
+ throws java.io.ObjectStreamException {
+ return super.writeReplace();
+ }
+
+ @java.lang.Override
+ public boolean equals(final java.lang.Object obj) {
+ if (obj == this) {
+ return true;
+ }
+ if (!(obj instanceof org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo)) {
+ return super.equals(obj);
+ }
+ org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo other = (org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo) obj;
+
+ boolean result = true;
+ result = result && (hasType() == other.hasType());
+ if (hasType()) {
+ result = result &&
+ (getType() == other.getType());
+ }
+ result = result && (hasHfile() == other.hasHfile());
+ if (hasHfile()) {
+ result = result && getHfile()
+ .equals(other.getHfile());
+ }
+ result = result && (hasWalServer() == other.hasWalServer());
+ if (hasWalServer()) {
+ result = result && getWalServer()
+ .equals(other.getWalServer());
+ }
+ result = result && (hasWalName() == other.hasWalName());
+ if (hasWalName()) {
+ result = result && getWalName()
+ .equals(other.getWalName());
+ }
+ result = result &&
+ getUnknownFields().equals(other.getUnknownFields());
+ return result;
+ }
+
+ private int memoizedHashCode = 0;
+ @java.lang.Override
+ public int hashCode() {
+ if (memoizedHashCode != 0) {
+ return memoizedHashCode;
+ }
+ int hash = 41;
+ hash = (19 * hash) + getDescriptorForType().hashCode();
+ if (hasType()) {
+ hash = (37 * hash) + TYPE_FIELD_NUMBER;
+ hash = (53 * hash) + hashEnum(getType());
+ }
+ if (hasHfile()) {
+ hash = (37 * hash) + HFILE_FIELD_NUMBER;
+ hash = (53 * hash) + getHfile().hashCode();
+ }
+ if (hasWalServer()) {
+ hash = (37 * hash) + WAL_SERVER_FIELD_NUMBER;
+ hash = (53 * hash) + getWalServer().hashCode();
+ }
+ if (hasWalName()) {
+ hash = (37 * hash) + WAL_NAME_FIELD_NUMBER;
+ hash = (53 * hash) + getWalName().hashCode();
+ }
+ hash = (29 * hash) + getUnknownFields().hashCode();
+ memoizedHashCode = hash;
+ return hash;
+ }
+
+ public static org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo parseFrom(
+ com.google.protobuf.ByteString data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo parseFrom(
+ com.google.protobuf.ByteString data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo parseFrom(byte[] data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo parseFrom(
+ byte[] data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo parseFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo parseFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo parseDelimitedFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo parseDelimitedFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input, extensionRegistry);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo parseFrom(
+ com.google.protobuf.CodedInputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo parseFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+
+ public static Builder newBuilder() { return Builder.create(); }
+ public Builder newBuilderForType() { return newBuilder(); }
+ public static Builder newBuilder(org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo prototype) {
+ return newBuilder().mergeFrom(prototype);
+ }
+ public Builder toBuilder() { return newBuilder(this); }
+
+ @java.lang.Override
+ protected Builder newBuilderForType(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ Builder builder = new Builder(parent);
+ return builder;
+ }
+ /**
+ * Protobuf type {@code SnapshotFileInfo}
+ */
+ public static final class Builder extends
+ com.google.protobuf.GeneratedMessage.Builder<Builder>
+ implements org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfoOrBuilder {
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.internal_static_SnapshotFileInfo_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.internal_static_SnapshotFileInfo_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo.class, org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo.Builder.class);
+ }
+
+ // Construct using org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo.newBuilder()
+ private Builder() {
+ maybeForceBuilderInitialization();
+ }
+
+ private Builder(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ super(parent);
+ maybeForceBuilderInitialization();
+ }
+ private void maybeForceBuilderInitialization() {
+ if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+ }
+ }
+ private static Builder create() {
+ return new Builder();
+ }
+
+ public Builder clear() {
+ super.clear();
+ type_ = org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo.Type.HFILE;
+ bitField0_ = (bitField0_ & ~0x00000001);
+ hfile_ = "";
+ bitField0_ = (bitField0_ & ~0x00000002);
+ walServer_ = "";
+ bitField0_ = (bitField0_ & ~0x00000004);
+ walName_ = "";
+ bitField0_ = (bitField0_ & ~0x00000008);
+ return this;
+ }
+
+ public Builder clone() {
+ return create().mergeFrom(buildPartial());
+ }
+
+ public com.google.protobuf.Descriptors.Descriptor
+ getDescriptorForType() {
+ return org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.internal_static_SnapshotFileInfo_descriptor;
+ }
+
+ public org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo getDefaultInstanceForType() {
+ return org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo.getDefaultInstance();
+ }
+
+ public org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo build() {
+ org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo result = buildPartial();
+ if (!result.isInitialized()) {
+ throw newUninitializedMessageException(result);
+ }
+ return result;
+ }
+
+ public org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo buildPartial() {
+ org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo result = new org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo(this);
+ int from_bitField0_ = bitField0_;
+ int to_bitField0_ = 0;
+ if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+ to_bitField0_ |= 0x00000001;
+ }
+ result.type_ = type_;
+ if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+ to_bitField0_ |= 0x00000002;
+ }
+ result.hfile_ = hfile_;
+ if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+ to_bitField0_ |= 0x00000004;
+ }
+ result.walServer_ = walServer_;
+ if (((from_bitField0_ & 0x00000008) == 0x00000008)) {
+ to_bitField0_ |= 0x00000008;
+ }
+ result.walName_ = walName_;
+ result.bitField0_ = to_bitField0_;
+ onBuilt();
+ return result;
+ }
+
+ public Builder mergeFrom(com.google.protobuf.Message other) {
+ if (other instanceof org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo) {
+ return mergeFrom((org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo)other);
+ } else {
+ super.mergeFrom(other);
+ return this;
+ }
+ }
+
+ public Builder mergeFrom(org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo other) {
+ if (other == org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo.getDefaultInstance()) return this;
+ if (other.hasType()) {
+ setType(other.getType());
+ }
+ if (other.hasHfile()) {
+ bitField0_ |= 0x00000002;
+ hfile_ = other.hfile_;
+ onChanged();
+ }
+ if (other.hasWalServer()) {
+ bitField0_ |= 0x00000004;
+ walServer_ = other.walServer_;
+ onChanged();
+ }
+ if (other.hasWalName()) {
+ bitField0_ |= 0x00000008;
+ walName_ = other.walName_;
+ onChanged();
+ }
+ this.mergeUnknownFields(other.getUnknownFields());
+ return this;
+ }
+
+ public final boolean isInitialized() {
+ if (!hasType()) {
+
+ return false;
+ }
+ return true;
+ }
+
+ public Builder mergeFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo parsedMessage = null;
+ try {
+ parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ parsedMessage = (org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo) e.getUnfinishedMessage();
+ throw e;
+ } finally {
+ if (parsedMessage != null) {
+ mergeFrom(parsedMessage);
+ }
+ }
+ return this;
+ }
+ private int bitField0_;
+
+ // required .SnapshotFileInfo.Type type = 1;
+ private org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo.Type type_ = org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo.Type.HFILE;
+ /**
+ * <code>required .SnapshotFileInfo.Type type = 1;</code>
+ */
+ public boolean hasType() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>required .SnapshotFileInfo.Type type = 1;</code>
+ */
+ public org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo.Type getType() {
+ return type_;
+ }
+ /**
+ * <code>required .SnapshotFileInfo.Type type = 1;</code>
+ */
+ public Builder setType(org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo.Type value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000001;
+ type_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required .SnapshotFileInfo.Type type = 1;</code>
+ */
+ public Builder clearType() {
+ bitField0_ = (bitField0_ & ~0x00000001);
+ type_ = org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo.Type.HFILE;
+ onChanged();
+ return this;
+ }
+
+ // optional string hfile = 3;
+ private java.lang.Object hfile_ = "";
+ /**
+ * <code>optional string hfile = 3;</code>
+ */
+ public boolean hasHfile() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ /**
+ * <code>optional string hfile = 3;</code>
+ */
+ public java.lang.String getHfile() {
+ java.lang.Object ref = hfile_;
+ if (!(ref instanceof java.lang.String)) {
+ java.lang.String s = ((com.google.protobuf.ByteString) ref)
+ .toStringUtf8();
+ hfile_ = s;
+ return s;
+ } else {
+ return (java.lang.String) ref;
+ }
+ }
+ /**
+ * <code>optional string hfile = 3;</code>
+ */
+ public com.google.protobuf.ByteString
+ getHfileBytes() {
+ java.lang.Object ref = hfile_;
+ if (ref instanceof String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ hfile_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+ /**
+ * <code>optional string hfile = 3;</code>
+ */
+ public Builder setHfile(
+ java.lang.String value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000002;
+ hfile_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional string hfile = 3;</code>
+ */
+ public Builder clearHfile() {
+ bitField0_ = (bitField0_ & ~0x00000002);
+ hfile_ = getDefaultInstance().getHfile();
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional string hfile = 3;</code>
+ */
+ public Builder setHfileBytes(
+ com.google.protobuf.ByteString value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000002;
+ hfile_ = value;
+ onChanged();
+ return this;
+ }
+
+ // optional string wal_server = 4;
+ private java.lang.Object walServer_ = "";
+ /**
+ * <code>optional string wal_server = 4;</code>
+ */
+ public boolean hasWalServer() {
+ return ((bitField0_ & 0x00000004) == 0x00000004);
+ }
+ /**
+ * <code>optional string wal_server = 4;</code>
+ */
+ public java.lang.String getWalServer() {
+ java.lang.Object ref = walServer_;
+ if (!(ref instanceof java.lang.String)) {
+ java.lang.String s = ((com.google.protobuf.ByteString) ref)
+ .toStringUtf8();
+ walServer_ = s;
+ return s;
+ } else {
+ return (java.lang.String) ref;
+ }
+ }
+ /**
+ * <code>optional string wal_server = 4;</code>
+ */
+ public com.google.protobuf.ByteString
+ getWalServerBytes() {
+ java.lang.Object ref = walServer_;
+ if (ref instanceof String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ walServer_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+ /**
+ * <code>optional string wal_server = 4;</code>
+ */
+ public Builder setWalServer(
+ java.lang.String value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000004;
+ walServer_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional string wal_server = 4;</code>
+ */
+ public Builder clearWalServer() {
+ bitField0_ = (bitField0_ & ~0x00000004);
+ walServer_ = getDefaultInstance().getWalServer();
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional string wal_server = 4;</code>
+ */
+ public Builder setWalServerBytes(
+ com.google.protobuf.ByteString value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000004;
+ walServer_ = value;
+ onChanged();
+ return this;
+ }
+
+ // optional string wal_name = 5;
+ private java.lang.Object walName_ = "";
+ /**
+ * <code>optional string wal_name = 5;</code>
+ */
+ public boolean hasWalName() {
+ return ((bitField0_ & 0x00000008) == 0x00000008);
+ }
+ /**
+ * <code>optional string wal_name = 5;</code>
+ */
+ public java.lang.String getWalName() {
+ java.lang.Object ref = walName_;
+ if (!(ref instanceof java.lang.String)) {
+ java.lang.String s = ((com.google.protobuf.ByteString) ref)
+ .toStringUtf8();
+ walName_ = s;
+ return s;
+ } else {
+ return (java.lang.String) ref;
+ }
+ }
+ /**
+ * <code>optional string wal_name = 5;</code>
+ */
+ public com.google.protobuf.ByteString
+ getWalNameBytes() {
+ java.lang.Object ref = walName_;
+ if (ref instanceof String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ walName_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+ /**
+ * <code>optional string wal_name = 5;</code>
+ */
+ public Builder setWalName(
+ java.lang.String value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000008;
+ walName_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional string wal_name = 5;</code>
+ */
+ public Builder clearWalName() {
+ bitField0_ = (bitField0_ & ~0x00000008);
+ walName_ = getDefaultInstance().getWalName();
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional string wal_name = 5;</code>
+ */
+ public Builder setWalNameBytes(
+ com.google.protobuf.ByteString value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000008;
+ walName_ = value;
+ onChanged();
+ return this;
+ }
+
+ // @@protoc_insertion_point(builder_scope:SnapshotFileInfo)
+ }
+
+ static {
+ defaultInstance = new SnapshotFileInfo(true);
+ defaultInstance.initFields();
+ }
+
+ // @@protoc_insertion_point(class_scope:SnapshotFileInfo)
+ }
+
+ public interface SnapshotRegionManifestOrBuilder
+ extends com.google.protobuf.MessageOrBuilder {
+
+ // optional int32 version = 1;
+ /**
+ * <code>optional int32 version = 1;</code>
+ */
+ boolean hasVersion();
+ /**
+ * <code>optional int32 version = 1;</code>
+ */
+ int getVersion();
+
+ // required .RegionInfo region_info = 2;
+ /**
+ * <code>required .RegionInfo region_info = 2;</code>
+ */
+ boolean hasRegionInfo();
+ /**
+ * <code>required .RegionInfo region_info = 2;</code>
+ */
+ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo getRegionInfo();
+ /**
+ * <code>required .RegionInfo region_info = 2;</code>
+ */
+ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfoOrBuilder getRegionInfoOrBuilder();
+
+ // repeated .SnapshotRegionManifest.FamilyFiles family_files = 3;
+ /**
+ * <code>repeated .SnapshotRegionManifest.FamilyFiles family_files = 3;</code>
+ */
+ java.util.List<org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.FamilyFiles>
+ getFamilyFilesList();
+ /**
+ * <code>repeated .SnapshotRegionManifest.FamilyFiles family_files = 3;</code>
+ */
+ org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.FamilyFiles getFamilyFiles(int index);
+ /**
+ * <code>repeated .SnapshotRegionManifest.FamilyFiles family_files = 3;</code>
+ */
+ int getFamilyFilesCount();
+ /**
+ * <code>repeated .SnapshotRegionManifest.FamilyFiles family_files = 3;</code>
+ */
+ java.util.List<? extends org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.FamilyFilesOrBuilder>
+ getFamilyFilesOrBuilderList();
+ /**
+ * <code>repeated .SnapshotRegionManifest.FamilyFiles family_files = 3;</code>
+ */
+ org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.FamilyFilesOrBuilder getFamilyFilesOrBuilder(
+ int index);
+ }
+ /**
+ * Protobuf type {@code SnapshotRegionManifest}
+ */
+ public static final class SnapshotRegionManifest extends
+ com.google.protobuf.GeneratedMessage
+ implements SnapshotRegionManifestOrBuilder {
+ // Use SnapshotRegionManifest.newBuilder() to construct.
+ private SnapshotRegionManifest(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
+ super(builder);
+ this.unknownFields = builder.getUnknownFields();
+ }
+ private SnapshotRegionManifest(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+ private static final SnapshotRegionManifest defaultInstance;
+ public static SnapshotRegionManifest getDefaultInstance() {
+ return defaultInstance;
+ }
+
+ public SnapshotRegionManifest getDefaultInstanceForType() {
+ return defaultInstance;
+ }
+
+ private final com.google.protobuf.UnknownFieldSet unknownFields;
+ @java.lang.Override
+ public final com.google.protobuf.UnknownFieldSet
+ getUnknownFields() {
+ return this.unknownFields;
+ }
+ private SnapshotRegionManifest(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ initFields();
+ int mutable_bitField0_ = 0;
+ com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+ com.google.protobuf.UnknownFieldSet.newBuilder();
+ try {
+ boolean done = false;
+ while (!done) {
+ int tag = input.readTag();
+ switch (tag) {
+ case 0:
+ done = true;
+ break;
+ default: {
+ if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+ done = true;
+ }
+ break;
+ }
+ case 8: {
+ bitField0_ |= 0x00000001;
+ version_ = input.readInt32();
+ break;
+ }
+ case 18: {
+ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.Builder subBuilder = null;
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ subBuilder = regionInfo_.toBuilder();
+ }
+ regionInfo_ = input.readMessage(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.PARSER, extensionRegistry);
+ if (subBuilder != null) {
+ subBuilder.mergeFrom(regionInfo_);
+ regionInfo_ = subBuilder.buildPartial();
+ }
+ bitField0_ |= 0x00000002;
+ break;
+ }
+ case 26: {
+ if (!((mutable_bitField0_ & 0x00000004) == 0x00000004)) {
+ familyFiles_ = new java.util.ArrayList<org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.FamilyFiles>();
+ mutable_bitField0_ |= 0x00000004;
+ }
+ familyFiles_.add(input.readMessage(org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.FamilyFiles.PARSER, extensionRegistry));
+ break;
+ }
+ }
+ }
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ throw e.setUnfinishedMessage(this);
+ } catch (java.io.IOException e) {
+ throw new com.google.protobuf.InvalidProtocolBufferException(
+ e.getMessage()).setUnfinishedMessage(this);
+ } finally {
+ if (((mutable_bitField0_ & 0x00000004) == 0x00000004)) {
+ familyFiles_ = java.util.Collections.unmodifiableList(familyFiles_);
+ }
+ this.unknownFields = unknownFields.build();
+ makeExtensionsImmutable();
+ }
+ }
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.internal_static_SnapshotRegionManifest_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.internal_static_SnapshotRegionManifest_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.class, org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.Builder.class);
+ }
+
+ public static com.google.protobuf.Parser<SnapshotRegionManifest> PARSER =
+ new com.google.protobuf.AbstractParser<SnapshotRegionManifest>() {
+ public SnapshotRegionManifest parsePartialFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return new SnapshotRegionManifest(input, extensionRegistry);
+ }
+ };
+
+ @java.lang.Override
+ public com.google.protobuf.Parser<SnapshotRegionManifest> getParserForType() {
+ return PARSER;
+ }
+
+ public interface StoreFileOrBuilder
+ extends com.google.protobuf.MessageOrBuilder {
+
+ // required string name = 1;
+ /**
+ * <code>required string name = 1;</code>
+ */
+ boolean hasName();
+ /**
+ * <code>required string name = 1;</code>
+ */
+ java.lang.String getName();
+ /**
+ * <code>required string name = 1;</code>
+ */
+ com.google.protobuf.ByteString
+ getNameBytes();
+
+ // optional .Reference reference = 2;
+ /**
+ * <code>optional .Reference reference = 2;</code>
+ */
+ boolean hasReference();
+ /**
+ * <code>optional .Reference reference = 2;</code>
+ */
+ org.apache.hadoop.hbase.protobuf.generated.FSProtos.Reference getReference();
+ /**
+ * <code>optional .Reference reference = 2;</code>
+ */
+ org.apache.hadoop.hbase.protobuf.generated.FSProtos.ReferenceOrBuilder getReferenceOrBuilder();
+
+ // optional uint64 file_size = 3;
+ /**
+ * <code>optional uint64 file_size = 3;</code>
+ *
+ * <pre>
+ * TODO: Add checksums or other fields to verify the file
+ * </pre>
+ */
+ boolean hasFileSize();
+ /**
+ * <code>optional uint64 file_size = 3;</code>
+ *
+ * <pre>
+ * TODO: Add checksums or other fields to verify the file
+ * </pre>
+ */
+ long getFileSize();
+ }
+ /**
+ * Protobuf type {@code SnapshotRegionManifest.StoreFile}
+ */
+ public static final class StoreFile extends
+ com.google.protobuf.GeneratedMessage
+ implements StoreFileOrBuilder {
+ // Use StoreFile.newBuilder() to construct.
+ private StoreFile(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
+ super(builder);
+ this.unknownFields = builder.getUnknownFields();
+ }
+ private StoreFile(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+ private static final StoreFile defaultInstance;
+ public static StoreFile getDefaultInstance() {
+ return defaultInstance;
+ }
+
+ public StoreFile getDefaultInstanceForType() {
+ return defaultInstance;
+ }
+
+ private final com.google.protobuf.UnknownFieldSet unknownFields;
+ @java.lang.Override
+ public final com.google.protobuf.UnknownFieldSet
+ getUnknownFields() {
+ return this.unknownFields;
+ }
+ private StoreFile(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ initFields();
+ int mutable_bitField0_ = 0;
+ com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+ com.google.protobuf.UnknownFieldSet.newBuilder();
+ try {
+ boolean done = false;
+ while (!done) {
+ int tag = input.readTag();
+ switch (tag) {
+ case 0:
+ done = true;
+ break;
+ default: {
+ if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+ done = true;
+ }
+ break;
+ }
+ case 10: {
+ bitField0_ |= 0x00000001;
+ name_ = input.readBytes();
+ break;
+ }
+ case 18: {
+ org.apache.hadoop.hbase.protobuf.generated.FSProtos.Reference.Builder subBuilder = null;
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ subBuilder = reference_.toBuilder();
+ }
+ reference_ = input.readMessage(org.apache.hadoop.hbase.protobuf.generated.FSProtos.Reference.PARSER, extensionRegistry);
+ if (subBuilder != null) {
+ subBuilder.mergeFrom(reference_);
+ reference_ = subBuilder.buildPartial();
+ }
+ bitField0_ |= 0x00000002;
+ break;
+ }
+ case 24: {
+ bitField0_ |= 0x00000004;
+ fileSize_ = input.readUInt64();
+ break;
+ }
+ }
+ }
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ throw e.setUnfinishedMessage(this);
+ } catch (java.io.IOException e) {
+ throw new com.google.protobuf.InvalidProtocolBufferException(
+ e.getMessage()).setUnfinishedMessage(this);
+ } finally {
+ this.unknownFields = unknownFields.build();
+ makeExtensionsImmutable();
+ }
+ }
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.internal_static_SnapshotRegionManifest_StoreFile_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.internal_static_SnapshotRegionManifest_StoreFile_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile.class, org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile.Builder.class);
+ }
+
+ public static com.google.protobuf.Parser<StoreFile> PARSER =
+ new com.google.protobuf.AbstractParser<StoreFile>() {
+ public StoreFile parsePartialFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return new StoreFile(input, extensionRegistry);
+ }
+ };
+
+ @java.lang.Override
+ public com.google.protobuf.Parser<StoreFile> getParserForType() {
+ return PARSER;
+ }
+
+ private int bitField0_;
+ // required string name = 1;
+ public static final int NAME_FIELD_NUMBER = 1;
+ private java.lang.Object name_;
+ /**
+ * <code>required string name = 1;</code>
+ */
+ public boolean hasName() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>required string name = 1;</code>
+ */
+ public java.lang.String getName() {
+ java.lang.Object ref = name_;
+ if (ref instanceof java.lang.String) {
+ return (java.lang.String) ref;
+ } else {
+ com.google.protobuf.ByteString bs =
+ (com.google.protobuf.ByteString) ref;
+ java.lang.String s = bs.toStringUtf8();
+ if (bs.isValidUtf8()) {
+ name_ = s;
+ }
+ return s;
+ }
+ }
+ /**
+ * <code>required string name = 1;</code>
+ */
+ public com.google.protobuf.ByteString
+ getNameBytes() {
+ java.lang.Object ref = name_;
+ if (ref instanceof java.lang.String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ name_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+
+ // optional .Reference reference = 2;
+ public static final int REFERENCE_FIELD_NUMBER = 2;
+ private org.apache.hadoop.hbase.protobuf.generated.FSProtos.Reference reference_;
+ /**
+ * <code>optional .Reference reference = 2;</code>
+ */
+ public boolean hasReference() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ /**
+ * <code>optional .Reference reference = 2;</code>
+ */
+ public org.apache.hadoop.hbase.protobuf.generated.FSProtos.Reference getReference() {
+ return reference_;
+ }
+ /**
+ * <code>optional .Reference reference = 2;</code>
+ */
+ public org.apache.hadoop.hbase.protobuf.generated.FSProtos.ReferenceOrBuilder getReferenceOrBuilder() {
+ return reference_;
+ }
+
+ // optional uint64 file_size = 3;
+ public static final int FILE_SIZE_FIELD_NUMBER = 3;
+ private long fileSize_;
+ /**
+ * <code>optional uint64 file_size = 3;</code>
+ *
+ * <pre>
+ * TODO: Add checksums or other fields to verify the file
+ * </pre>
+ */
+ public boolean hasFileSize() {
+ return ((bitField0_ & 0x00000004) == 0x00000004);
+ }
+ /**
+ * <code>optional uint64 file_size = 3;</code>
+ *
+ * <pre>
+ * TODO: Add checksums or other fields to verify the file
+ * </pre>
+ */
+ public long getFileSize() {
+ return fileSize_;
+ }
+
+ private void initFields() {
+ name_ = "";
+ reference_ = org.apache.hadoop.hbase.protobuf.generated.FSProtos.Reference.getDefaultInstance();
+ fileSize_ = 0L;
+ }
+ private byte memoizedIsInitialized = -1;
+ public final boolean isInitialized() {
+ byte isInitialized = memoizedIsInitialized;
+ if (isInitialized != -1) return isInitialized == 1;
+
+ if (!hasName()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ if (hasReference()) {
+ if (!getReference().isInitialized()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ }
+ memoizedIsInitialized = 1;
+ return true;
+ }
+
+ public void writeTo(com.google.protobuf.CodedOutputStream output)
+ throws java.io.IOException {
+ getSerializedSize();
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ output.writeBytes(1, getNameBytes());
+ }
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ output.writeMessage(2, reference_);
+ }
+ if (((bitField0_ & 0x00000004) == 0x00000004)) {
+ output.writeUInt64(3, fileSize_);
+ }
+ getUnknownFields().writeTo(output);
+ }
+
+ private int memoizedSerializedSize = -1;
+ public int getSerializedSize() {
+ int size = memoizedSerializedSize;
+ if (size != -1) return size;
+
+ size = 0;
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(1, getNameBytes());
+ }
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeMessageSize(2, reference_);
+ }
+ if (((bitField0_ & 0x00000004) == 0x00000004)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeUInt64Size(3, fileSize_);
+ }
+ size += getUnknownFields().getSerializedSize();
+ memoizedSerializedSize = size;
+ return size;
+ }
+
+ private static final long serialVersionUID = 0L;
+ @java.lang.Override
+ protected java.lang.Object writeReplace()
+ throws java.io.ObjectStreamException {
+ return super.writeReplace();
+ }
+
+ @java.lang.Override
+ public boolean equals(final java.lang.Object obj) {
+ if (obj == this) {
+ return true;
+ }
+ if (!(obj instanceof org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile)) {
+ return super.equals(obj);
+ }
+ org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile other = (org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile) obj;
+
+ boolean result = true;
+ result = result && (hasName() == other.hasName());
+ if (hasName()) {
+ result = result && getName()
+ .equals(other.getName());
+ }
+ result = result && (hasReference() == other.hasReference());
+ if (hasReference()) {
+ result = result && getReference()
+ .equals(other.getReference());
+ }
+ result = result && (hasFileSize() == other.hasFileSize());
+ if (hasFileSize()) {
+ result = result && (getFileSize()
+ == other.getFileSize());
+ }
+ result = result &&
+ getUnknownFields().equals(other.getUnknownFields());
+ return result;
+ }
+
+ private int memoizedHashCode = 0;
+ @java.lang.Override
+ public int hashCode() {
+ if (memoizedHashCode != 0) {
+ return memoizedHashCode;
+ }
+ int hash = 41;
+ hash = (19 * hash) + getDescriptorForType().hashCode();
+ if (hasName()) {
+ hash = (37 * hash) + NAME_FIELD_NUMBER;
+ hash = (53 * hash) + getName().hashCode();
+ }
+ if (hasReference()) {
+ hash = (37 * hash) + REFERENCE_FIELD_NUMBER;
+ hash = (53 * hash) + getReference().hashCode();
+ }
+ if (hasFileSize()) {
+ hash = (37 * hash) + FILE_SIZE_FIELD_NUMBER;
+ hash = (53 * hash) + hashLong(getFileSize());
+ }
+ hash = (29 * hash) + getUnknownFields().hashCode();
+ memoizedHashCode = hash;
+ return hash;
+ }
+
+ public static org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile parseFrom(
+ com.google.protobuf.ByteString data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile parseFrom(
+ com.google.protobuf.ByteString data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile parseFrom(byte[] data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile parseFrom(
+ byte[] data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile parseFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile parseFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile parseDelimitedFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile parseDelimitedFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input, extensionRegistry);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile parseFrom(
+ com.google.protobuf.CodedInputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile parseFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+
+ public static Builder newBuilder() { return Builder.create(); }
+ public Builder newBuilderForType() { return newBuilder(); }
+ public static Builder newBuilder(org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile prototype) {
+ return newBuilder().mergeFrom(prototype);
+ }
+ public Builder toBuilder() { return newBuilder(this); }
+
+ @java.lang.Override
+ protected Builder newBuilderForType(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ Builder builder = new Builder(parent);
+ return builder;
+ }
+ /**
+ * Protobuf type {@code SnapshotRegionManifest.StoreFile}
+ */
+ public static final class Builder extends
+ com.google.protobuf.GeneratedMessage.Builder<Builder>
+ implements org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFileOrBuilder {
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.internal_static_SnapshotRegionManifest_StoreFile_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.internal_static_SnapshotRegionManifest_StoreFile_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile.class, org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile.Builder.class);
+ }
+
+ // Construct using org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile.newBuilder()
+ private Builder() {
+ maybeForceBuilderInitialization();
+ }
+
+ private Builder(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ super(parent);
+ maybeForceBuilderInitialization();
+ }
+ private void maybeForceBuilderInitialization() {
+ if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+ getReferenceFieldBuilder();
+ }
+ }
+ private static Builder create() {
+ return new Builder();
+ }
+
+ public Builder clear() {
+ super.clear();
+ name_ = "";
+ bitField0_ = (bitField0_ & ~0x00000001);
+ if (referenceBuilder_ == null) {
+ reference_ = org.apache.hadoop.hbase.protobuf.generated.FSProtos.Reference.getDefaultInstance();
+ } else {
+ referenceBuilder_.clear();
+ }
+ bitField0_ = (bitField0_ & ~0x00000002);
+ fileSize_ = 0L;
+ bitField0_ = (bitField0_ & ~0x00000004);
+ return this;
+ }
+
+ public Builder clone() {
+ return create().mergeFrom(buildPartial());
+ }
+
+ public com.google.protobuf.Descriptors.Descriptor
+ getDescriptorForType() {
+ return org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.internal_static_SnapshotRegionManifest_StoreFile_descriptor;
+ }
+
+ public org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile getDefaultInstanceForType() {
+ return org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile.getDefaultInstance();
+ }
+
+ public org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile build() {
+ org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile result = buildPartial();
+ if (!result.isInitialized()) {
+ throw newUninitializedMessageException(result);
+ }
+ return result;
+ }
+
+ public org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile buildPartial() {
+ org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile result = new org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile(this);
+ int from_bitField0_ = bitField0_;
+ int to_bitField0_ = 0;
+ if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+ to_bitField0_ |= 0x00000001;
+ }
+ result.name_ = name_;
+ if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+ to_bitField0_ |= 0x00000002;
+ }
+ if (referenceBuilder_ == null) {
+ result.reference_ = reference_;
+ } else {
+ result.reference_ = referenceBuilder_.build();
+ }
+ if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+ to_bitField0_ |= 0x00000004;
+ }
+ result.fileSize_ = fileSize_;
+ result.bitField0_ = to_bitField0_;
+ onBuilt();
+ return result;
+ }
+
+ public Builder mergeFrom(com.google.protobuf.Message other) {
+ if (other instanceof org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile) {
+ return mergeFrom((org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile)other);
+ } else {
+ super.mergeFrom(other);
+ return this;
+ }
+ }
+
+ public Builder mergeFrom(org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile other) {
+ if (other == org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile.getDefaultInstance()) return this;
+ if (other.hasName()) {
+ bitField0_ |= 0x00000001;
+ name_ = other.name_;
+ onChanged();
+ }
+ if (other.hasReference()) {
+ mergeReference(other.getReference());
+ }
+ if (other.hasFileSize()) {
+ setFileSize(other.getFileSize());
+ }
+ this.mergeUnknownFields(other.getUnknownFields());
+ return this;
+ }
+
+ public final boolean isInitialized() {
+ if (!hasName()) {
+
+ return false;
+ }
+ if (hasReference()) {
+ if (!getReference().isInitialized()) {
+
+ return false;
+ }
+ }
+ return true;
+ }
+
+ public Builder mergeFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile parsedMessage = null;
+ try {
+ parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ parsedMessage = (org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile) e.getUnfinishedMessage();
+ throw e;
+ } finally {
+ if (parsedMessage != null) {
+ mergeFrom(parsedMessage);
+ }
+ }
+ return this;
+ }
+ private int bitField0_;
+
+ // required string name = 1;
+ private java.lang.Object name_ = "";
+ /**
+ * <code>required string name = 1;</code>
+ */
+ public boolean hasName() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>required string name = 1;</code>
+ */
+ public java.lang.String getName() {
+ java.lang.Object ref = name_;
+ if (!(ref instanceof java.lang.String)) {
+ java.lang.String s = ((com.google.protobuf.ByteString) ref)
+ .toStringUtf8();
+ name_ = s;
+ return s;
+ } else {
+ return (java.lang.String) ref;
+ }
+ }
+ /**
+ * <code>required string name = 1;</code>
+ */
+ public com.google.protobuf.ByteString
+ getNameBytes() {
+ java.lang.Object ref = name_;
+ if (ref instanceof String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ name_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+ /**
+ * <code>required string name = 1;</code>
+ */
+ public Builder setName(
+ java.lang.String value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000001;
+ name_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required string name = 1;</code>
+ */
+ public Builder clearName() {
+ bitField0_ = (bitField0_ & ~0x00000001);
+ name_ = getDefaultInstance().getName();
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required string name = 1;</code>
+ */
+ public Builder setNameBytes(
+ com.google.protobuf.ByteString value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000001;
+ name_ = value;
+ onChanged();
+ return this;
+ }
+
+ // optional .Reference reference = 2;
+ private org.apache.hadoop.hbase.protobuf.generated.FSProtos.Reference reference_ = org.apache.hadoop.hbase.protobuf.generated.FSProtos.Reference.getDefaultInstance();
+ private com.google.protobuf.SingleFieldBuilder<
+ org.apache.hadoop.hbase.protobuf.generated.FSProtos.Reference, org.apache.hadoop.hbase.protobuf.generated.FSProtos.Reference.Builder, org.apache.hadoop.hbase.protobuf.generated.FSProtos.ReferenceOrBuilder> referenceBuilder_;
+ /**
+ * <code>optional .Reference reference = 2;</code>
+ */
+ public boolean hasReference() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ /**
+ * <code>optional .Reference reference = 2;</code>
+ */
+ public org.apache.hadoop.hbase.protobuf.generated.FSProtos.Reference getReference() {
+ if (referenceBuilder_ == null) {
+ return reference_;
+ } else {
+ return referenceBuilder_.getMessage();
+ }
+ }
+ /**
+ * <code>optional .Reference reference = 2;</code>
+ */
+ public Builder setReference(org.apache.hadoop.hbase.protobuf.generated.FSProtos.Reference value) {
+ if (referenceBuilder_ == null) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ reference_ = value;
+ onChanged();
+ } else {
+ referenceBuilder_.setMessage(value);
+ }
+ bitField0_ |= 0x00000002;
+ return this;
+ }
+ /**
+ * <code>optional .Reference reference = 2;</code>
+ */
+ public Builder setReference(
+ org.apache.hadoop.hbase.protobuf.generated.FSProtos.Reference.Builder builderForValue) {
+ if (referenceBuilder_ == null) {
+ reference_ = builderForValue.build();
+ onChanged();
+ } else {
+ referenceBuilder_.setMessage(builderForValue.build());
+ }
+ bitField0_ |= 0x00000002;
+ return this;
+ }
+ /**
+ * <code>optional .Reference reference = 2;</code>
+ */
+ public Builder mergeReference(org.apache.hadoop.hbase.protobuf.generated.FSProtos.Reference value) {
+ if (referenceBuilder_ == null) {
+ if (((bitField0_ & 0x00000002) == 0x00000002) &&
+ reference_ != org.apache.hadoop.hbase.protobuf.generated.FSProtos.Reference.getDefaultInstance()) {
+ reference_ =
+ org.apache.hadoop.hbase.protobuf.generated.FSProtos.Reference.newBuilder(reference_).mergeFrom(value).buildPartial();
+ } else {
+ reference_ = value;
+ }
+ onChanged();
+ } else {
+ referenceBuilder_.mergeFrom(value);
+ }
+ bitField0_ |= 0x00000002;
+ return this;
+ }
+ /**
+ * <code>optional .Reference reference = 2;</code>
+ */
+ public Builder clearReference() {
+ if (referenceBuilder_ == null) {
+ reference_ = org.apache.hadoop.hbase.protobuf.generated.FSProtos.Reference.getDefaultInstance();
+ onChanged();
+ } else {
+ referenceBuilder_.clear();
+ }
+ bitField0_ = (bitField0_ & ~0x00000002);
+ return this;
+ }
+ /**
+ * <code>optional .Reference reference = 2;</code>
+ */
+ public org.apache.hadoop.hbase.protobuf.generated.FSProtos.Reference.Builder getReferenceBuilder() {
+ bitField0_ |= 0x00000002;
+ onChanged();
+ return getReferenceFieldBuilder().getBuilder();
+ }
+ /**
+ * <code>optional .Reference reference = 2;</code>
+ */
+ public org.apache.hadoop.hbase.protobuf.generated.FSProtos.ReferenceOrBuilder getReferenceOrBuilder() {
+ if (referenceBuilder_ != null) {
+ return referenceBuilder_.getMessageOrBuilder();
+ } else {
+ return reference_;
+ }
+ }
+ /**
+ * <code>optional .Reference reference = 2;</code>
+ */
+ private com.google.protobuf.SingleFieldBuilder<
+ org.apache.hadoop.hbase.protobuf.generated.FSProtos.Reference, org.apache.hadoop.hbase.protobuf.generated.FSProtos.Reference.Builder, org.apache.hadoop.hbase.protobuf.generated.FSProtos.ReferenceOrBuilder>
+ getReferenceFieldBuilder() {
+ if (referenceBuilder_ == null) {
+ referenceBuilder_ = new com.google.protobuf.SingleFieldBuilder<
+ org.apache.hadoop.hbase.protobuf.generated.FSProtos.Reference, org.apache.hadoop.hbase.protobuf.generated.FSProtos.Reference.Builder, org.apache.hadoop.hbase.protobuf.generated.FSProtos.ReferenceOrBuilder>(
+ reference_,
+ getParentForChildren(),
+ isClean());
+ reference_ = null;
+ }
+ return referenceBuilder_;
+ }
+
+ // optional uint64 file_size = 3;
+ private long fileSize_ ;
+ /**
+ * <code>optional uint64 file_size = 3;</code>
+ *
+ * <pre>
+ * TODO: Add checksums or other fields to verify the file
+ * </pre>
+ */
+ public boolean hasFileSize() {
+ return ((bitField0_ & 0x00000004) == 0x00000004);
+ }
+ /**
+ * <code>optional uint64 file_size = 3;</code>
+ *
+ * <pre>
+ * TODO: Add checksums or other fields to verify the file
+ * </pre>
+ */
+ public long getFileSize() {
+ return fileSize_;
+ }
+ /**
+ * <code>optional uint64 file_size = 3;</code>
+ *
+ * <pre>
+ * TODO: Add checksums or other fields to verify the file
+ * </pre>
+ */
+ public Builder setFileSize(long value) {
+ bitField0_ |= 0x00000004;
+ fileSize_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional uint64 file_size = 3;</code>
+ *
+ * <pre>
+ * TODO: Add checksums or other fields to verify the file
+ * </pre>
+ */
+ public Builder clearFileSize() {
+ bitField0_ = (bitField0_ & ~0x00000004);
+ fileSize_ = 0L;
+ onChanged();
+ return this;
+ }
+
+ // @@protoc_insertion_point(builder_scope:SnapshotRegionManifest.StoreFile)
+ }
+
+ static {
+ defaultInstance = new StoreFile(true);
+ defaultInstance.initFields();
+ }
+
+ // @@protoc_insertion_point(class_scope:SnapshotRegionManifest.StoreFile)
+ }
+
+ public interface FamilyFilesOrBuilder
+ extends com.google.protobuf.MessageOrBuilder {
+
+ // required bytes family_name = 1;
+ /**
+ * <code>required bytes family_name = 1;</code>
+ */
+ boolean hasFamilyName();
+ /**
+ * <code>required bytes family_name = 1;</code>
+ */
+ com.google.protobuf.ByteString getFamilyName();
+
+ // repeated .SnapshotRegionManifest.StoreFile store_files = 2;
+ /**
+ * <code>repeated .SnapshotRegionManifest.StoreFile store_files = 2;</code>
+ */
+ java.util.List<org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile>
+ getStoreFilesList();
+ /**
+ * <code>repeated .SnapshotRegionManifest.StoreFile store_files = 2;</code>
+ */
+ org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile getStoreFiles(int index);
+ /**
+ * <code>repeated .SnapshotRegionManifest.StoreFile store_files = 2;</code>
+ */
+ int getStoreFilesCount();
+ /**
+ * <code>repeated .SnapshotRegionManifest.StoreFile store_files = 2;</code>
+ */
+ java.util.List<? extends org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFileOrBuilder>
+ getStoreFilesOrBuilderList();
+ /**
+ * <code>repeated .SnapshotRegionManifest.StoreFile store_files = 2;</code>
+ */
+ org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFileOrBuilder getStoreFilesOrBuilder(
+ int index);
+ }
+ /**
+ * Protobuf type {@code SnapshotRegionManifest.FamilyFiles}
+ */
+ public static final class FamilyFiles extends
+ com.google.protobuf.GeneratedMessage
+ implements FamilyFilesOrBuilder {
+ // Use FamilyFiles.newBuilder() to construct.
+ private FamilyFiles(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
+ super(builder);
+ this.unknownFields = builder.getUnknownFields();
+ }
+ private FamilyFiles(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+ private static final FamilyFiles defaultInstance;
+ public static FamilyFiles getDefaultInstance() {
+ return defaultInstance;
+ }
+
+ public FamilyFiles getDefaultInstanceForType() {
+ return defaultInstance;
+ }
+
+ private final com.google.protobuf.UnknownFieldSet unknownFields;
+ @java.lang.Override
+ public final com.google.protobuf.UnknownFieldSet
+ getUnknownFields() {
+ return this.unknownFields;
+ }
+ private FamilyFiles(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ initFields();
+ int mutable_bitField0_ = 0;
+ com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+ com.google.protobuf.UnknownFieldSet.newBuilder();
+ try {
+ boolean done = false;
+ while (!done) {
+ int tag = input.readTag();
+ switch (tag) {
+ case 0:
+ done = true;
+ break;
+ default: {
+ if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+ done = true;
+ }
+ break;
+ }
+ case 10: {
+ bitField0_ |= 0x00000001;
+ familyName_ = input.readBytes();
+ break;
+ }
+ case 18: {
+ if (!((mutable_bitField0_ & 0x00000002) == 0x00000002)) {
+ storeFiles_ = new java.util.ArrayList<org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile>();
+ mutable_bitField0_ |= 0x00000002;
+ }
+ storeFiles_.add(input.readMessage(org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile.PARSER, extensionRegistry));
+ break;
+ }
+ }
+ }
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ throw e.setUnfinishedMessage(this);
+ } catch (java.io.IOException e) {
+ throw new com.google.protobuf.InvalidProtocolBufferException(
+ e.getMessage()).setUnfinishedMessage(this);
+ } finally {
+ if (((mutable_bitField0_ & 0x00000002) == 0x00000002)) {
+ storeFiles_ = java.util.Collections.unmodifiableList(storeFiles_);
+ }
+ this.unknownFields = unknownFields.build();
+ makeExtensionsImmutable();
+ }
+ }
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.internal_static_SnapshotRegionManifest_FamilyFiles_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.internal_static_SnapshotRegionManifest_FamilyFiles_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.FamilyFiles.class, org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.FamilyFiles.Builder.class);
+ }
+
+ public static com.google.protobuf.Parser<FamilyFiles> PARSER =
+ new com.google.protobuf.AbstractParser<FamilyFiles>() {
+ public FamilyFiles parsePartialFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return new FamilyFiles(input, extensionRegistry);
+ }
+ };
+
+ @java.lang.Override
+ public com.google.protobuf.Parser<FamilyFiles> getParserForType() {
+ return PARSER;
+ }
+
+ private int bitField0_;
+ // required bytes family_name = 1;
+ public static final int FAMILY_NAME_FIELD_NUMBER = 1;
+ private com.google.protobuf.ByteString familyName_;
+ /**
+ * <code>required bytes family_name = 1;</code>
+ */
+ public boolean hasFamilyName() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>required bytes family_name = 1;</code>
+ */
+ public com.google.protobuf.ByteString getFamilyName() {
+ return familyName_;
+ }
+
+ // repeated .SnapshotRegionManifest.StoreFile store_files = 2;
+ public static final int STORE_FILES_FIELD_NUMBER = 2;
+ private java.util.List<org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile> storeFiles_;
+ /**
+ * <code>repeated .SnapshotRegionManifest.StoreFile store_files = 2;</code>
+ */
+ public java.util.List<org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile> getStoreFilesList() {
+ return storeFiles_;
+ }
+ /**
+ * <code>repeated .SnapshotRegionManifest.StoreFile store_files = 2;</code>
+ */
+ public java.util.List<? extends org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFileOrBuilder>
+ getStoreFilesOrBuilderList() {
+ return storeFiles_;
+ }
+ /**
+ * <code>repeated .SnapshotRegionManifest.StoreFile store_files = 2;</code>
+ */
+ public int getStoreFilesCount() {
+ return storeFiles_.size();
+ }
+ /**
+ * <code>repeated .SnapshotRegionManifest.StoreFile store_files = 2;</code>
+ */
+ public org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile getStoreFiles(int index) {
+ return storeFiles_.get(index);
+ }
+ /**
+ * <code>repeated .SnapshotRegionManifest.StoreFile store_files = 2;</code>
+ */
+ public org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFileOrBuilder getStoreFilesOrBuilder(
+ int index) {
+ return storeFiles_.get(index);
+ }
+
+ private void initFields() {
+ familyName_ = com.google.protobuf.ByteString.EMPTY;
+ storeFiles_ = java.util.Collections.emptyList();
+ }
+ private byte memoizedIsInitialized = -1;
+ public final boolean isInitialized() {
+ byte isInitialized = memoizedIsInitialized;
+ if (isInitialized != -1) return isInitialized == 1;
+
+ if (!hasFamilyName()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ for (int i = 0; i < getStoreFilesCount(); i++) {
+ if (!getStoreFiles(i).isInitialized()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ }
+ memoizedIsInitialized = 1;
+ return true;
+ }
+
+ public void writeTo(com.google.protobuf.CodedOutputStream output)
+ throws java.io.IOException {
+ getSerializedSize();
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ output.writeBytes(1, familyName_);
+ }
+ for (int i = 0; i < storeFiles_.size(); i++) {
+ output.writeMessage(2, storeFiles_.get(i));
+ }
+ getUnknownFields().writeTo(output);
+ }
+
+ private int memoizedSerializedSize = -1;
+ public int getSerializedSize() {
+ int size = memoizedSerializedSize;
+ if (size != -1) return size;
+
+ size = 0;
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(1, familyName_);
+ }
+ for (int i = 0; i < storeFiles_.size(); i++) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeMessageSize(2, storeFiles_.get(i));
+ }
+ size += getUnknownFields().getSerializedSize();
+ memoizedSerializedSize = size;
+ return size;
+ }
+
+ private static final long serialVersionUID = 0L;
+ @java.lang.Override
+ protected java.lang.Object writeReplace()
+ throws java.io.ObjectStreamException {
+ return super.writeReplace();
+ }
+
+ @java.lang.Override
+ public boolean equals(final java.lang.Object obj) {
+ if (obj == this) {
+ return true;
+ }
+ if (!(obj instanceof org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.FamilyFiles)) {
+ return super.equals(obj);
+ }
+ org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.FamilyFiles other = (org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.FamilyFiles) obj;
+
+ boolean result = true;
+ result = result && (hasFamilyName() == other.hasFamilyName());
+ if (hasFamilyName()) {
+ result = result && getFamilyName()
+ .equals(other.getFamilyName());
+ }
+ result = result && getStoreFilesList()
+ .equals(other.getStoreFilesList());
+ result = result &&
+ getUnknownFields().equals(other.getUnknownFields());
+ return result;
+ }
+
+ private int memoizedHashCode = 0;
+ @java.lang.Override
+ public int hashCode() {
+ if (memoizedHashCode != 0) {
+ return memoizedHashCode;
+ }
+ int hash = 41;
+ hash = (19 * hash) + getDescriptorForType().hashCode();
+ if (hasFamilyName()) {
+ hash = (37 * hash) + FAMILY_NAME_FIELD_NUMBER;
+ hash = (53 * hash) + getFamilyName().hashCode();
+ }
+ if (getStoreFilesCount() > 0) {
+ hash = (37 * hash) + STORE_FILES_FIELD_NUMBER;
+ hash = (53 * hash) + getStoreFilesList().hashCode();
+ }
+ hash = (29 * hash) + getUnknownFields().hashCode();
+ memoizedHashCode =
<TRUNCATED>
[4/7] HBASE-11742 (Backport HBASE-7987 and HBASE-11185 to 0.98)
(Esteban Gutierrez and Matteo Bertozzi)
Posted by ap...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/CopyRecoveredEditsTask.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/CopyRecoveredEditsTask.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/CopyRecoveredEditsTask.java
deleted file mode 100644
index 3dc2d99..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/CopyRecoveredEditsTask.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.snapshot;
-
-import java.io.IOException;
-import java.util.NavigableSet;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
-import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
-
-/**
- * Copy over each of the files in a region's recovered.edits directory to the region's snapshot
- * directory.
- * <p>
- * This is a serial operation over each of the files in the recovered.edits directory and also
- * streams all the bytes to the client and then back to the filesystem, so the files being copied
- * should be <b>small</b> or it will (a) suck up a lot of bandwidth, and (b) take a long time.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class CopyRecoveredEditsTask extends SnapshotTask {
-
- private static final Log LOG = LogFactory.getLog(CopyRecoveredEditsTask.class);
- private final FileSystem fs;
- private final Path regiondir;
- private final Path outputDir;
-
- /**
- * @param snapshot Snapshot being taken
- * @param monitor error monitor for the snapshot
- * @param fs {@link FileSystem} where the snapshot is being taken
- * @param regionDir directory for the region to examine for edits
- * @param snapshotRegionDir directory for the region in the snapshot
- */
- public CopyRecoveredEditsTask(SnapshotDescription snapshot, ForeignExceptionDispatcher monitor,
- FileSystem fs, Path regionDir, Path snapshotRegionDir) {
- super(snapshot, monitor);
- this.fs = fs;
- this.regiondir = regionDir;
- this.outputDir = HLogUtil.getRegionDirRecoveredEditsDir(snapshotRegionDir);
- }
-
- @Override
- public Void call() throws IOException {
- NavigableSet<Path> files = HLogUtil.getSplitEditFilesSorted(this.fs, regiondir);
- if (files == null || files.size() == 0) return null;
-
- // copy over each file.
- // this is really inefficient (could be trivially parallelized), but is
- // really simple to reason about.
- for (Path source : files) {
- // check to see if the file is zero length, in which case we can skip it
- FileStatus stat = fs.getFileStatus(source);
- if (stat.getLen() <= 0) continue;
-
- // its not zero length, so copy over the file
- Path out = new Path(outputDir, source.getName());
- LOG.debug("Copying " + source + " to " + out);
- FileUtil.copy(fs, source, fs, out, true, fs.getConf());
-
- // check for errors to the running operation after each file
- this.rethrowException();
- }
- return null;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
index 69e1683..3ffed7a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
@@ -47,19 +47,22 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.io.FileLink;
import org.apache.hadoop.hbase.io.HFileLink;
import org.apache.hadoop.hbase.io.HLogLink;
import org.apache.hadoop.hbase.io.hadoopbackport.ThrottledInputStream;
import org.apache.hadoop.hbase.mapreduce.JobUtil;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
-import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo;
+import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
@@ -100,7 +103,8 @@ public class ExportSnapshot extends Configured implements Tool {
// Export Map-Reduce Counters, to keep track of the progress
public enum Counter { MISSING_FILES, COPY_FAILED, BYTES_EXPECTED, BYTES_COPIED, FILES_COPIED };
- private static class ExportMapper extends Mapper<Text, NullWritable, NullWritable, NullWritable> {
+ private static class ExportMapper extends Mapper<BytesWritable, NullWritable,
+ NullWritable, NullWritable> {
final static int REPORT_SIZE = 1 * 1024 * 1024;
final static int BUFFER_SIZE = 64 * 1024;
@@ -156,35 +160,35 @@ public class ExportSnapshot extends Configured implements Tool {
}
@Override
- public void map(Text key, NullWritable value, Context context)
+ public void map(BytesWritable key, NullWritable value, Context context)
throws InterruptedException, IOException {
- Path inputPath = new Path(key.toString());
- Path outputPath = getOutputPath(inputPath);
+ SnapshotFileInfo inputInfo = SnapshotFileInfo.parseFrom(key.copyBytes());
+ Path outputPath = getOutputPath(inputInfo);
- LOG.info("copy file input=" + inputPath + " output=" + outputPath);
- copyFile(context, inputPath, outputPath);
+ copyFile(context, inputInfo, outputPath);
}
/**
* Returns the location where the inputPath will be copied.
- * - hfiles are encoded as hfile links hfile-region-table
- * - logs are encoded as serverName/logName
*/
- private Path getOutputPath(final Path inputPath) throws IOException {
- Path path;
- if (HFileLink.isHFileLink(inputPath) || StoreFileInfo.isReference(inputPath)) {
- String family = inputPath.getParent().getName();
- TableName table =
- HFileLink.getReferencedTableName(inputPath.getName());
- String region = HFileLink.getReferencedRegionName(inputPath.getName());
- String hfile = HFileLink.getReferencedHFileName(inputPath.getName());
- path = new Path(FSUtils.getTableDir(new Path("./"), table),
- new Path(region, new Path(family, hfile)));
- } else if (isHLogLinkPath(inputPath)) {
- String logName = inputPath.getName();
- path = new Path(new Path(outputRoot, HConstants.HREGION_OLDLOGDIR_NAME), logName);
- } else {
- path = inputPath;
+ private Path getOutputPath(final SnapshotFileInfo inputInfo) throws IOException {
+ Path path = null;
+ switch (inputInfo.getType()) {
+ case HFILE:
+ Path inputPath = new Path(inputInfo.getHfile());
+ String family = inputPath.getParent().getName();
+ TableName table =HFileLink.getReferencedTableName(inputPath.getName());
+ String region = HFileLink.getReferencedRegionName(inputPath.getName());
+ String hfile = HFileLink.getReferencedHFileName(inputPath.getName());
+ path = new Path(FSUtils.getTableDir(new Path("./"), table),
+ new Path(region, new Path(family, hfile)));
+ break;
+ case WAL:
+ Path oldLogsDir = new Path(outputRoot, HConstants.HREGION_OLDLOGDIR_NAME);
+ path = new Path(oldLogsDir, inputInfo.getWalName());
+ break;
+ default:
+ throw new IOException("Invalid File Type: " + inputInfo.getType().toString());
}
return new Path(outputArchive, path);
}
@@ -192,7 +196,7 @@ public class ExportSnapshot extends Configured implements Tool {
/*
* Used by TestExportSnapshot to simulate a failure
*/
- private void injectTestFailure(final Context context, final Path inputPath)
+ private void injectTestFailure(final Context context, final SnapshotFileInfo inputInfo)
throws IOException {
if (testFailures) {
if (context.getConfiguration().getBoolean(CONF_TEST_RETRY, false)) {
@@ -204,37 +208,38 @@ public class ExportSnapshot extends Configured implements Tool {
// retry, but at least we reduce the number of test failures due to
// this test exception from the same map task.
if (random.nextFloat() < 0.03) {
- throw new IOException("TEST RETRY FAILURE: Unable to copy input=" + inputPath
+ throw new IOException("TEST RETRY FAILURE: Unable to copy input=" + inputInfo
+ " time=" + System.currentTimeMillis());
}
} else {
context.getCounter(Counter.COPY_FAILED).increment(1);
- throw new IOException("TEST FAILURE: Unable to copy input=" + inputPath);
+ throw new IOException("TEST FAILURE: Unable to copy input=" + inputInfo);
}
}
}
- private void copyFile(final Context context, final Path inputPath, final Path outputPath)
- throws IOException {
- injectTestFailure(context, inputPath);
+ private void copyFile(final Context context, final SnapshotFileInfo inputInfo,
+ final Path outputPath) throws IOException {
+ injectTestFailure(context, inputInfo);
// Get the file information
- FileStatus inputStat = getSourceFileStatus(context, inputPath);
+ FileStatus inputStat = getSourceFileStatus(context, inputInfo);
// Verify if the output file exists and is the same that we want to copy
if (outputFs.exists(outputPath)) {
FileStatus outputStat = outputFs.getFileStatus(outputPath);
if (outputStat != null && sameFile(inputStat, outputStat)) {
- LOG.info("Skip copy " + inputPath + " to " + outputPath + ", same file.");
+ LOG.info("Skip copy " + inputStat.getPath() + " to " + outputPath + ", same file.");
return;
}
}
- InputStream in = openSourceFile(context, inputPath);
+ InputStream in = openSourceFile(context, inputInfo);
int bandwidthMB = context.getConfiguration().getInt(CONF_BANDWIDTH_MB, 100);
if (Integer.MAX_VALUE != bandwidthMB) {
in = new ThrottledInputStream(new BufferedInputStream(in), bandwidthMB * 1024 * 1024);
}
+
try {
context.getCounter(Counter.BYTES_EXPECTED).increment(inputStat.getLen());
@@ -242,7 +247,7 @@ public class ExportSnapshot extends Configured implements Tool {
outputFs.mkdirs(outputPath.getParent());
FSDataOutputStream out = outputFs.create(outputPath, true);
try {
- copyData(context, inputPath, in, outputPath, out, inputStat.getLen());
+ copyData(context, inputStat.getPath(), in, outputPath, out, inputStat.getLen());
} finally {
out.close();
}
@@ -276,7 +281,7 @@ public class ExportSnapshot extends Configured implements Tool {
try {
if (filesMode > 0 && stat.getPermission().toShort() != filesMode) {
outputFs.setPermission(path, new FsPermission(filesMode));
- } else if (!stat.getPermission().equals(refStat.getPermission())) {
+ } else if (refStat != null && !stat.getPermission().equals(refStat.getPermission())) {
outputFs.setPermission(path, refStat.getPermission());
}
} catch (IOException e) {
@@ -284,8 +289,9 @@ public class ExportSnapshot extends Configured implements Tool {
return false;
}
- String user = stringIsNotEmpty(filesUser) ? filesUser : refStat.getOwner();
- String group = stringIsNotEmpty(filesGroup) ? filesGroup : refStat.getGroup();
+ boolean hasRefStat = (refStat != null);
+ String user = stringIsNotEmpty(filesUser) || !hasRefStat ? filesUser : refStat.getOwner();
+ String group = stringIsNotEmpty(filesGroup) || !hasRefStat ? filesGroup : refStat.getGroup();
if (stringIsNotEmpty(user) || stringIsNotEmpty(group)) {
try {
if (!(user.equals(stat.getOwner()) && group.equals(stat.getGroup()))) {
@@ -368,40 +374,53 @@ public class ExportSnapshot extends Configured implements Tool {
* Throws an IOException if the communication with the inputFs fail or
* if the file is not found.
*/
- private FSDataInputStream openSourceFile(Context context, final Path path) throws IOException {
+ private FSDataInputStream openSourceFile(Context context, final SnapshotFileInfo fileInfo)
+ throws IOException {
try {
- if (HFileLink.isHFileLink(path) || StoreFileInfo.isReference(path)) {
- return new HFileLink(inputRoot, inputArchive, path).open(inputFs);
- } else if (isHLogLinkPath(path)) {
- String serverName = path.getParent().getName();
- String logName = path.getName();
- return new HLogLink(inputRoot, serverName, logName).open(inputFs);
+ FileLink link = null;
+ switch (fileInfo.getType()) {
+ case HFILE:
+ Path inputPath = new Path(fileInfo.getHfile());
+ link = new HFileLink(inputRoot, inputArchive, inputPath);
+ break;
+ case WAL:
+ String serverName = fileInfo.getWalServer();
+ String logName = fileInfo.getWalName();
+ link = new HLogLink(inputRoot, serverName, logName);
+ break;
+ default:
+ throw new IOException("Invalid File Type: " + fileInfo.getType().toString());
}
- return inputFs.open(path);
+ return link.open(inputFs);
} catch (IOException e) {
context.getCounter(Counter.MISSING_FILES).increment(1);
- LOG.error("Unable to open source file=" + path, e);
+ LOG.error("Unable to open source file=" + fileInfo.toString(), e);
throw e;
}
}
- private FileStatus getSourceFileStatus(Context context, final Path path) throws IOException {
+ private FileStatus getSourceFileStatus(Context context, final SnapshotFileInfo fileInfo)
+ throws IOException {
try {
- if (HFileLink.isHFileLink(path) || StoreFileInfo.isReference(path)) {
- HFileLink link = new HFileLink(inputRoot, inputArchive, path);
- return link.getFileStatus(inputFs);
- } else if (isHLogLinkPath(path)) {
- String serverName = path.getParent().getName();
- String logName = path.getName();
- return new HLogLink(inputRoot, serverName, logName).getFileStatus(inputFs);
+ FileLink link = null;
+ switch (fileInfo.getType()) {
+ case HFILE:
+ Path inputPath = new Path(fileInfo.getHfile());
+ link = new HFileLink(inputRoot, inputArchive, inputPath);
+ break;
+ case WAL:
+ link = new HLogLink(inputRoot, fileInfo.getWalServer(), fileInfo.getWalName());
+ break;
+ default:
+ throw new IOException("Invalid File Type: " + fileInfo.getType().toString());
}
- return inputFs.getFileStatus(path);
+ return link.getFileStatus(inputFs);
} catch (FileNotFoundException e) {
context.getCounter(Counter.MISSING_FILES).increment(1);
- LOG.error("Unable to get the status for source file=" + path, e);
+ LOG.error("Unable to get the status for source file=" + fileInfo.toString(), e);
throw e;
} catch (IOException e) {
- LOG.error("Unable to get the status for source file=" + path, e);
+ LOG.error("Unable to get the status for source file=" + fileInfo.toString(), e);
throw e;
}
}
@@ -435,49 +454,54 @@ public class ExportSnapshot extends Configured implements Tool {
return inChecksum.equals(outChecksum);
}
-
- /**
- * HLog files are encoded as serverName/logName
- * and since all the other files should be in /hbase/table/..path..
- * we can rely on the depth, for now.
- */
- private static boolean isHLogLinkPath(final Path path) {
- return path.depth() == 2;
- }
}
/**
* Extract the list of files (HFiles/HLogs) to copy using Map-Reduce.
* @return list of files referenced by the snapshot (pair of path and size)
*/
- private List<Pair<Path, Long>> getSnapshotFiles(final FileSystem fs, final Path snapshotDir)
- throws IOException {
+ private List<Pair<SnapshotFileInfo, Long>> getSnapshotFiles(final FileSystem fs,
+ final Path snapshotDir) throws IOException {
SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
- final List<Pair<Path, Long>> files = new ArrayList<Pair<Path, Long>>();
- final TableName table =
- TableName.valueOf(snapshotDesc.getTable());
+ final List<Pair<SnapshotFileInfo, Long>> files = new ArrayList<Pair<SnapshotFileInfo, Long>>();
+ final TableName table = TableName.valueOf(snapshotDesc.getTable());
final Configuration conf = getConf();
// Get snapshot files
- SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
- new SnapshotReferenceUtil.FileVisitor() {
- public void storeFile (final String region, final String family, final String hfile)
- throws IOException {
- Path path = HFileLink.createPath(table, region, family, hfile);
- long size = new HFileLink(conf, path).getFileStatus(fs).getLen();
- files.add(new Pair<Path, Long>(path, size));
- }
-
- public void recoveredEdits (final String region, final String logfile)
- throws IOException {
- // copied with the snapshot referenecs
+ SnapshotReferenceUtil.visitReferencedFiles(conf, fs, snapshotDir, snapshotDesc,
+ new SnapshotReferenceUtil.SnapshotVisitor() {
+ @Override
+ public void storeFile(final HRegionInfo regionInfo, final String family,
+ final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+ if (storeFile.hasReference()) {
+ // copied as part of the manifest
+ } else {
+ String region = regionInfo.getEncodedName();
+ String hfile = storeFile.getName();
+ Path path = HFileLink.createPath(table, region, family, hfile);
+
+ SnapshotFileInfo fileInfo = SnapshotFileInfo.newBuilder()
+ .setType(SnapshotFileInfo.Type.HFILE)
+ .setHfile(path.toString())
+ .build();
+
+ long size = new HFileLink(conf, path).getFileStatus(fs).getLen();
+ files.add(new Pair<SnapshotFileInfo, Long>(fileInfo, size));
+ }
}
+ @Override
public void logFile (final String server, final String logfile)
throws IOException {
+ SnapshotFileInfo fileInfo = SnapshotFileInfo.newBuilder()
+ .setType(SnapshotFileInfo.Type.WAL)
+ .setWalServer(server)
+ .setWalName(logfile)
+ .build();
+
long size = new HLogLink(conf, server, logfile).getFileStatus(fs).getLen();
- files.add(new Pair<Path, Long>(new Path(server, logfile), size));
+ files.add(new Pair<SnapshotFileInfo, Long>(fileInfo, size));
}
});
@@ -492,34 +516,35 @@ public class ExportSnapshot extends Configured implements Tool {
* and then each group fetch the bigger file available, iterating through groups
* alternating the direction.
*/
- static List<List<Path>> getBalancedSplits(final List<Pair<Path, Long>> files, int ngroups) {
+ static List<List<SnapshotFileInfo>> getBalancedSplits(
+ final List<Pair<SnapshotFileInfo, Long>> files, final int ngroups) {
// Sort files by size, from small to big
- Collections.sort(files, new Comparator<Pair<Path, Long>>() {
- public int compare(Pair<Path, Long> a, Pair<Path, Long> b) {
+ Collections.sort(files, new Comparator<Pair<SnapshotFileInfo, Long>>() {
+ public int compare(Pair<SnapshotFileInfo, Long> a, Pair<SnapshotFileInfo, Long> b) {
long r = a.getSecond() - b.getSecond();
return (r < 0) ? -1 : ((r > 0) ? 1 : 0);
}
});
// create balanced groups
- List<List<Path>> fileGroups = new LinkedList<List<Path>>();
+ List<List<SnapshotFileInfo>> fileGroups = new LinkedList<List<SnapshotFileInfo>>();
long[] sizeGroups = new long[ngroups];
int hi = files.size() - 1;
int lo = 0;
- List<Path> group;
+ List<SnapshotFileInfo> group;
int dir = 1;
int g = 0;
while (hi >= lo) {
if (g == fileGroups.size()) {
- group = new LinkedList<Path>();
+ group = new LinkedList<SnapshotFileInfo>();
fileGroups.add(group);
} else {
group = fileGroups.get(g);
}
- Pair<Path, Long> fileInfo = files.get(hi--);
+ Pair<SnapshotFileInfo, Long> fileInfo = files.get(hi--);
// add the hi one
sizeGroups[g] += fileInfo.getSecond();
@@ -559,25 +584,25 @@ public class ExportSnapshot extends Configured implements Tool {
* and the number of the files to copy.
*/
private static Path[] createInputFiles(final Configuration conf, final Path inputFolderPath,
- final List<Pair<Path, Long>> snapshotFiles, int mappers)
+ final List<Pair<SnapshotFileInfo, Long>> snapshotFiles, int mappers)
throws IOException, InterruptedException {
FileSystem fs = inputFolderPath.getFileSystem(conf);
LOG.debug("Input folder location: " + inputFolderPath);
- List<List<Path>> splits = getBalancedSplits(snapshotFiles, mappers);
+ List<List<SnapshotFileInfo>> splits = getBalancedSplits(snapshotFiles, mappers);
Path[] inputFiles = new Path[splits.size()];
- Text key = new Text();
+ BytesWritable key = new BytesWritable();
for (int i = 0; i < inputFiles.length; i++) {
- List<Path> files = splits.get(i);
+ List<SnapshotFileInfo> files = splits.get(i);
inputFiles[i] = new Path(inputFolderPath, String.format("export-%d.seq", i));
SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, inputFiles[i],
- Text.class, NullWritable.class);
+ BytesWritable.class, NullWritable.class);
LOG.debug("Input split: " + i);
try {
- for (Path file: files) {
- LOG.debug(file.toString());
- key.set(file.toString());
+ for (SnapshotFileInfo file: files) {
+ byte[] pbFileInfo = file.toByteArray();
+ key.set(pbFileInfo, 0, pbFileInfo.length);
writer.append(key, NullWritable.get());
}
} finally {
@@ -592,7 +617,7 @@ public class ExportSnapshot extends Configured implements Tool {
* Run Map-Reduce Job to perform the files copy.
*/
private void runCopyJob(final Path inputRoot, final Path outputRoot,
- final List<Pair<Path, Long>> snapshotFiles, final boolean verifyChecksum,
+ final List<Pair<SnapshotFileInfo, Long>> snapshotFiles, final boolean verifyChecksum,
final String filesUser, final String filesGroup, final int filesMode,
final int mappers, final int bandwidthMB)
throws IOException, InterruptedException, ClassNotFoundException {
@@ -711,7 +736,7 @@ public class ExportSnapshot extends Configured implements Tool {
System.err.println("UNEXPECTED: " + cmd);
printUsageAndExit();
}
- } catch (Exception e) {
+ } catch (IOException e) {
printUsageAndExit();
}
}
@@ -777,7 +802,7 @@ public class ExportSnapshot extends Configured implements Tool {
// Step 0 - Extract snapshot files to copy
LOG.info("Loading Snapshot hfile list");
- final List<Pair<Path, Long>> files = getSnapshotFiles(inputFs, snapshotDir);
+ final List<Pair<SnapshotFileInfo, Long>> files = getSnapshotFiles(inputFs, snapshotDir);
if (mappers == 0 && files.size() > 0) {
mappers = 1 + (files.size() / conf.getInt(CONF_MAP_GROUP, 10));
mappers = Math.min(mappers, files.size());
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ReferenceRegionHFilesTask.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ReferenceRegionHFilesTask.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ReferenceRegionHFilesTask.java
deleted file mode 100644
index 60d48d9..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ReferenceRegionHFilesTask.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.snapshot;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
-import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
-import org.apache.hadoop.hbase.util.FSUtils;
-
-/**
- * Reference all the hfiles in a region for a snapshot.
- * <p>
- * Doesn't take into acccount if the hfiles are valid or not, just keeps track of what's in the
- * region's directory.
- */
-public class ReferenceRegionHFilesTask extends SnapshotTask {
-
- public static final Log LOG = LogFactory.getLog(ReferenceRegionHFilesTask.class);
- private final Path regiondir;
- private final FileSystem fs;
- private final PathFilter fileFilter;
- private final Path snapshotDir;
-
- /**
- * Reference all the files in the given region directory
- * @param snapshot snapshot for which to add references
- * @param monitor to check/send error
- * @param regionDir region directory to look for errors
- * @param fs {@link FileSystem} where the snapshot/region live
- * @param regionSnapshotDir directory in the snapshot to store region files
- */
- public ReferenceRegionHFilesTask(final SnapshotDescription snapshot,
- ForeignExceptionDispatcher monitor, Path regionDir, final FileSystem fs, Path regionSnapshotDir) {
- super(snapshot, monitor);
- this.regiondir = regionDir;
- this.fs = fs;
-
- this.fileFilter = new PathFilter() {
- @Override
- public boolean accept(Path path) {
- try {
- return fs.isFile(path);
- } catch (IOException e) {
- LOG.error("Failed to reach fs to check file:" + path + ", marking as not file");
- ReferenceRegionHFilesTask.this.snapshotFailure("Failed to reach fs to check file status",
- e);
- return false;
- }
- }
- };
- this.snapshotDir = regionSnapshotDir;
- }
-
- @Override
- public Void call() throws IOException {
- FileStatus[] families = FSUtils.listStatus(fs, regiondir, new FSUtils.FamilyDirFilter(fs));
-
- // if no families, then we are done again
- if (families == null || families.length == 0) {
- LOG.info("No families under region directory:" + regiondir
- + ", not attempting to add references.");
- return null;
- }
-
- // snapshot directories to store the hfile reference
- List<Path> snapshotFamilyDirs = TakeSnapshotUtils.getFamilySnapshotDirectories(snapshot,
- snapshotDir, families);
-
- LOG.debug("Add hfile references to snapshot directories:" + snapshotFamilyDirs);
- for (int i = 0; i < families.length; i++) {
- FileStatus family = families[i];
- Path familyDir = family.getPath();
- // get all the hfiles in the family
- FileStatus[] hfiles = FSUtils.listStatus(fs, familyDir, fileFilter);
-
- // if no hfiles, then we are done with this family
- if (hfiles == null || hfiles.length == 0) {
- LOG.debug("Not hfiles found for family: " + familyDir + ", skipping.");
- continue;
- }
-
- // make the snapshot's family directory
- Path snapshotFamilyDir = snapshotFamilyDirs.get(i);
- fs.mkdirs(snapshotFamilyDir);
-
- // create a reference for each hfile
- for (FileStatus hfile : hfiles) {
- // references are 0-length files, relying on file name.
- Path referenceFile = new Path(snapshotFamilyDir, hfile.getPath().getName());
- LOG.debug("Creating reference for:" + hfile.getPath() + " at " + referenceFile);
- if (!fs.createNewFile(referenceFile)) {
- throw new IOException("Failed to create reference file:" + referenceFile);
- }
- }
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("Finished referencing hfiles, current region state:");
- FSUtils.logFileSystemState(fs, regiondir, LOG);
- LOG.debug("and the snapshot directory:");
- FSUtils.logFileSystemState(fs, snapshotDir, LOG);
- }
- return null;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ReferenceServerWALsTask.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ReferenceServerWALsTask.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ReferenceServerWALsTask.java
deleted file mode 100644
index 9c987ab..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ReferenceServerWALsTask.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.snapshot;
-
-import java.io.IOException;
-import java.util.Arrays;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.errorhandling.ForeignException;
-import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
-import org.apache.hadoop.hbase.util.FSUtils;
-
-/**
- * Reference all the WAL files under a server's WAL directory
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class ReferenceServerWALsTask extends SnapshotTask {
- private static final Log LOG = LogFactory.getLog(ReferenceServerWALsTask.class);
- private final FileSystem fs;
- private final Configuration conf;
- private final String serverName;
- private Path logDir;
-
- /**
- * @param snapshot snapshot being run
- * @param failureListener listener to check for errors while running the operation and to
- * propagate errors found while running the task
- * @param logDir log directory for the server. Name of the directory is taken as the name of the
- * server
- * @param conf {@link Configuration} to extract filesystem information
- * @param fs filesystem where the log files are stored and should be referenced
- */
- public ReferenceServerWALsTask(SnapshotDescription snapshot,
- ForeignExceptionDispatcher failureListener, final Path logDir, final Configuration conf,
- final FileSystem fs) {
- super(snapshot, failureListener);
- this.fs = fs;
- this.conf = conf;
- this.serverName = logDir.getName();
- this.logDir = logDir;
- }
-
- /**
- * Create reference files (empty files with the same path and file name as original).
- * @throws IOException exception from hdfs or network problems
- * @throws ForeignException exception from an external procedure
- */
- @Override
- public Void call() throws IOException, ForeignException {
- // TODO switch to using a single file to reference all required WAL files
-
- // Iterate through each of the log files and add a reference to it.
- // assumes that all the files under the server's logs directory is a log
- FileStatus[] serverLogs = FSUtils.listStatus(fs, logDir, null);
- if (serverLogs == null) {
- LOG.debug("No logs for server directory:" + logDir + ", done referencing files.");
- return null;
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Adding references for WAL files:" + Arrays.toString(serverLogs));
- }
-
- for (FileStatus file : serverLogs) {
- this.rethrowException();
-
- // add the reference to the file. ex: hbase/.snapshots/.logs/<serverName>/<hlog>
- Path rootDir = FSUtils.getRootDir(conf);
- Path snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(this.snapshot, rootDir);
- Path snapshotLogDir = TakeSnapshotUtils.getSnapshotHLogsDir(snapshotDir, serverName);
- // actually store the reference on disk (small file)
- Path ref = new Path(snapshotLogDir, file.getPath().getName());
- if (!fs.createNewFile(ref)) {
- if (!fs.exists(ref)) {
- throw new IOException("Couldn't create reference for:" + file.getPath());
- }
- }
- LOG.debug("Completed WAL referencing for: " + file.getPath() + " to " + ref);
- }
-
- LOG.debug("Successfully completed WAL referencing for ALL files");
- return null;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
index 83dbaa1..d8d8642 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
@@ -19,8 +19,10 @@
package org.apache.hadoop.hbase.snapshot;
import java.io.IOException;
+import java.io.InterruptedIOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
@@ -29,6 +31,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
+import java.util.concurrent.ThreadPoolExecutor;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -46,17 +49,18 @@ import org.apache.hadoop.hbase.catalog.CatalogTracker;
import org.apache.hadoop.hbase.catalog.MetaEditor;
import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
import org.apache.hadoop.hbase.io.HFileLink;
+import org.apache.hadoop.hbase.io.Reference;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
import org.apache.hadoop.hbase.monitoring.TaskMonitor;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSTableDescriptors;
import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.FSVisitor;
import org.apache.hadoop.hbase.util.ModifyRegionUtils;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.io.IOUtils;
@@ -115,9 +119,9 @@ public class RestoreSnapshotHelper {
private final ForeignExceptionDispatcher monitor;
private final MonitoredTask status;
+ private final SnapshotManifest snapshotManifest;
private final SnapshotDescription snapshotDesc;
private final TableName snapshotTable;
- private final Path snapshotDir;
private final HTableDescriptor tableDesc;
private final Path rootDir;
@@ -128,8 +132,7 @@ public class RestoreSnapshotHelper {
public RestoreSnapshotHelper(final Configuration conf,
final FileSystem fs,
- final SnapshotDescription snapshotDescription,
- final Path snapshotDir,
+ final SnapshotManifest manifest,
final HTableDescriptor tableDescriptor,
final Path rootDir,
final ForeignExceptionDispatcher monitor,
@@ -137,9 +140,9 @@ public class RestoreSnapshotHelper {
{
this.fs = fs;
this.conf = conf;
- this.snapshotDesc = snapshotDescription;
- this.snapshotTable = TableName.valueOf(snapshotDescription.getTable());
- this.snapshotDir = snapshotDir;
+ this.snapshotManifest = manifest;
+ this.snapshotDesc = manifest.getSnapshotDescription();
+ this.snapshotTable = TableName.valueOf(snapshotDesc.getTable());
this.tableDesc = tableDescriptor;
this.rootDir = rootDir;
this.tableDir = FSUtils.getTableDir(rootDir, tableDesc.getTableName());
@@ -152,15 +155,29 @@ public class RestoreSnapshotHelper {
* @return the set of regions touched by the restore operation
*/
public RestoreMetaChanges restoreHdfsRegions() throws IOException {
+ ThreadPoolExecutor exec = SnapshotManifest.createExecutor(conf, "RestoreSnapshot");
+ try {
+ return restoreHdfsRegions(exec);
+ } finally {
+ exec.shutdown();
+ }
+ }
+
+ private RestoreMetaChanges restoreHdfsRegions(final ThreadPoolExecutor exec) throws IOException {
LOG.debug("starting restore");
- Set<String> snapshotRegionNames = SnapshotReferenceUtil.getSnapshotRegionNames(fs, snapshotDir);
- if (snapshotRegionNames == null) {
+
+ Map<String, SnapshotRegionManifest> regionManifests = snapshotManifest.getRegionManifestsMap();
+ if (regionManifests == null) {
LOG.warn("Nothing to restore. Snapshot " + snapshotDesc + " looks empty");
return null;
}
RestoreMetaChanges metaChanges = new RestoreMetaChanges(parentsMap);
+ // Take a copy of the manifest.keySet() since we are going to modify
+ // this instance, by removing the regions already present in the restore dir.
+ Set<String> regionNames = new HashSet<String>(regionManifests.keySet());
+
// Identify which region are still available and which not.
// NOTE: we rely upon the region name as: "table name, start key, end key"
List<HRegionInfo> tableRegions = getTableRegions();
@@ -168,9 +185,9 @@ public class RestoreSnapshotHelper {
monitor.rethrowException();
for (HRegionInfo regionInfo: tableRegions) {
String regionName = regionInfo.getEncodedName();
- if (snapshotRegionNames.contains(regionName)) {
+ if (regionNames.contains(regionName)) {
LOG.info("region to restore: " + regionName);
- snapshotRegionNames.remove(regionName);
+ regionNames.remove(regionName);
metaChanges.addRegionToRestore(regionInfo);
} else {
LOG.info("region to remove: " + regionName);
@@ -181,41 +198,34 @@ public class RestoreSnapshotHelper {
// Restore regions using the snapshot data
monitor.rethrowException();
status.setStatus("Restoring table regions...");
- restoreHdfsRegions(metaChanges.getRegionsToRestore());
+ restoreHdfsRegions(exec, regionManifests, metaChanges.getRegionsToRestore());
status.setStatus("Finished restoring all table regions.");
// Remove regions from the current table
monitor.rethrowException();
status.setStatus("Starting to delete excess regions from table");
- removeHdfsRegions(metaChanges.getRegionsToRemove());
+ removeHdfsRegions(exec, metaChanges.getRegionsToRemove());
status.setStatus("Finished deleting excess regions from table.");
}
// Regions to Add: present in the snapshot but not in the current table
- if (snapshotRegionNames.size() > 0) {
- List<HRegionInfo> regionsToAdd = new LinkedList<HRegionInfo>();
+ if (regionNames.size() > 0) {
+ List<HRegionInfo> regionsToAdd = new ArrayList<HRegionInfo>(regionNames.size());
monitor.rethrowException();
- for (String regionName: snapshotRegionNames) {
+ for (String regionName: regionNames) {
LOG.info("region to add: " + regionName);
- Path regionDir = new Path(snapshotDir, regionName);
- regionsToAdd.add(HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir));
+ regionsToAdd.add(HRegionInfo.convert(regionManifests.get(regionName).getRegionInfo()));
}
// Create new regions cloning from the snapshot
monitor.rethrowException();
status.setStatus("Cloning regions...");
- HRegionInfo[] clonedRegions = cloneHdfsRegions(regionsToAdd);
+ HRegionInfo[] clonedRegions = cloneHdfsRegions(exec, regionManifests, regionsToAdd);
metaChanges.setNewRegions(clonedRegions);
status.setStatus("Finished cloning regions.");
}
- // Restore WALs
- monitor.rethrowException();
- status.setStatus("Restoring WALs to table...");
- restoreWALs();
- status.setStatus("Finished restoring WALs to table.");
-
return metaChanges;
}
@@ -346,30 +356,52 @@ public class RestoreSnapshotHelper {
/**
* Remove specified regions from the file-system, using the archiver.
*/
- private void removeHdfsRegions(final List<HRegionInfo> regions) throws IOException {
- if (regions != null && regions.size() > 0) {
- for (HRegionInfo hri: regions) {
+ private void removeHdfsRegions(final ThreadPoolExecutor exec, final List<HRegionInfo> regions)
+ throws IOException {
+ if (regions == null || regions.size() == 0) return;
+ ModifyRegionUtils.editRegions(exec, regions, new ModifyRegionUtils.RegionEditTask() {
+ @Override
+ public void editRegion(final HRegionInfo hri) throws IOException {
HFileArchiver.archiveRegion(conf, fs, hri);
}
- }
+ });
}
/**
* Restore specified regions by restoring content to the snapshot state.
*/
- private void restoreHdfsRegions(final List<HRegionInfo> regions) throws IOException {
+ private void restoreHdfsRegions(final ThreadPoolExecutor exec,
+ final Map<String, SnapshotRegionManifest> regionManifests,
+ final List<HRegionInfo> regions) throws IOException {
if (regions == null || regions.size() == 0) return;
- for (HRegionInfo hri: regions) restoreRegion(hri);
+ ModifyRegionUtils.editRegions(exec, regions, new ModifyRegionUtils.RegionEditTask() {
+ @Override
+ public void editRegion(final HRegionInfo hri) throws IOException {
+ restoreRegion(hri, regionManifests.get(hri.getEncodedName()));
+ }
+ });
+ }
+
+ private Map<String, List<SnapshotRegionManifest.StoreFile>> getRegionHFileReferences(
+ final SnapshotRegionManifest manifest) {
+ Map<String, List<SnapshotRegionManifest.StoreFile>> familyMap =
+ new HashMap<String, List<SnapshotRegionManifest.StoreFile>>(manifest.getFamilyFilesCount());
+ for (SnapshotRegionManifest.FamilyFiles familyFiles: manifest.getFamilyFilesList()) {
+ familyMap.put(familyFiles.getFamilyName().toStringUtf8(),
+ new ArrayList<SnapshotRegionManifest.StoreFile>(familyFiles.getStoreFilesList()));
+ }
+ return familyMap;
}
/**
* Restore region by removing files not in the snapshot
* and adding the missing ones from the snapshot.
*/
- private void restoreRegion(HRegionInfo regionInfo) throws IOException {
- Path snapshotRegionDir = new Path(snapshotDir, regionInfo.getEncodedName());
- Map<String, List<String>> snapshotFiles =
- SnapshotReferenceUtil.getRegionHFileReferences(fs, snapshotRegionDir);
+ private void restoreRegion(final HRegionInfo regionInfo,
+ final SnapshotRegionManifest regionManifest) throws IOException {
+ Map<String, List<SnapshotRegionManifest.StoreFile>> snapshotFiles =
+ getRegionHFileReferences(regionManifest);
+
Path regionDir = new Path(tableDir, regionInfo.getEncodedName());
String tableName = tableDesc.getTableName().getNameAsString();
@@ -377,32 +409,34 @@ public class RestoreSnapshotHelper {
for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) {
byte[] family = Bytes.toBytes(familyDir.getName());
Set<String> familyFiles = getTableRegionFamilyFiles(familyDir);
- List<String> snapshotFamilyFiles = snapshotFiles.remove(familyDir.getName());
+ List<SnapshotRegionManifest.StoreFile> snapshotFamilyFiles =
+ snapshotFiles.remove(familyDir.getName());
if (snapshotFamilyFiles != null) {
- List<String> hfilesToAdd = new LinkedList<String>();
- for (String hfileName: snapshotFamilyFiles) {
- if (familyFiles.contains(hfileName)) {
+ List<SnapshotRegionManifest.StoreFile> hfilesToAdd =
+ new ArrayList<SnapshotRegionManifest.StoreFile>();
+ for (SnapshotRegionManifest.StoreFile storeFile: snapshotFamilyFiles) {
+ if (familyFiles.contains(storeFile.getName())) {
// HFile already present
- familyFiles.remove(hfileName);
+ familyFiles.remove(storeFile.getName());
} else {
// HFile missing
- hfilesToAdd.add(hfileName);
+ hfilesToAdd.add(storeFile);
}
}
// Remove hfiles not present in the snapshot
for (String hfileName: familyFiles) {
Path hfile = new Path(familyDir, hfileName);
- LOG.trace("Removing hfile=" + hfile +
+ LOG.trace("Removing hfile=" + hfileName +
" from region=" + regionInfo.getEncodedName() + " table=" + tableName);
HFileArchiver.archiveStoreFile(conf, fs, regionInfo, tableDir, family, hfile);
}
// Restore Missing files
- for (String hfileName: hfilesToAdd) {
- LOG.trace("Adding HFileLink " + hfileName +
+ for (SnapshotRegionManifest.StoreFile storeFile: hfilesToAdd) {
+ LOG.debug("Adding HFileLink " + storeFile.getName() +
" to region=" + regionInfo.getEncodedName() + " table=" + tableName);
- restoreStoreFile(familyDir, regionInfo, hfileName);
+ restoreStoreFile(familyDir, regionInfo, storeFile);
}
} else {
// Family doesn't exists in the snapshot
@@ -414,15 +448,16 @@ public class RestoreSnapshotHelper {
}
// Add families not present in the table
- for (Map.Entry<String, List<String>> familyEntry: snapshotFiles.entrySet()) {
+ for (Map.Entry<String, List<SnapshotRegionManifest.StoreFile>> familyEntry:
+ snapshotFiles.entrySet()) {
Path familyDir = new Path(regionDir, familyEntry.getKey());
if (!fs.mkdirs(familyDir)) {
throw new IOException("Unable to create familyDir=" + familyDir);
}
- for (String hfileName: familyEntry.getValue()) {
- LOG.trace("Adding HFileLink " + hfileName + " to table=" + tableName);
- restoreStoreFile(familyDir, regionInfo, hfileName);
+ for (SnapshotRegionManifest.StoreFile storeFile: familyEntry.getValue()) {
+ LOG.trace("Adding HFileLink " + storeFile.getName() + " to table=" + tableName);
+ restoreStoreFile(familyDir, regionInfo, storeFile);
}
}
}
@@ -448,7 +483,9 @@ public class RestoreSnapshotHelper {
* Clone specified regions. For each region create a new region
* and create a HFileLink for each hfile.
*/
- private HRegionInfo[] cloneHdfsRegions(final List<HRegionInfo> regions) throws IOException {
+ private HRegionInfo[] cloneHdfsRegions(final ThreadPoolExecutor exec,
+ final Map<String, SnapshotRegionManifest> regionManifests,
+ final List<HRegionInfo> regions) throws IOException {
if (regions == null || regions.size() == 0) return null;
final Map<String, HRegionInfo> snapshotRegions =
@@ -472,11 +509,12 @@ public class RestoreSnapshotHelper {
}
// create the regions on disk
- ModifyRegionUtils.createRegions(conf, rootDir, tableDir,
+ ModifyRegionUtils.createRegions(exec, conf, rootDir, tableDir,
tableDesc, clonedRegionsInfo, new ModifyRegionUtils.RegionFillTask() {
@Override
public void fillRegion(final HRegion region) throws IOException {
- cloneRegion(region, snapshotRegions.get(region.getRegionInfo().getEncodedName()));
+ HRegionInfo snapshotHri = snapshotRegions.get(region.getRegionInfo().getEncodedName());
+ cloneRegion(region, snapshotHri, regionManifests.get(snapshotHri.getEncodedName()));
}
});
@@ -494,21 +532,17 @@ public class RestoreSnapshotHelper {
* @param region {@link HRegion} cloned
* @param snapshotRegionInfo
*/
- private void cloneRegion(final HRegion region, final HRegionInfo snapshotRegionInfo)
- throws IOException {
- final Path snapshotRegionDir = new Path(snapshotDir, snapshotRegionInfo.getEncodedName());
+ private void cloneRegion(final HRegion region, final HRegionInfo snapshotRegionInfo,
+ final SnapshotRegionManifest manifest) throws IOException {
final Path regionDir = new Path(tableDir, region.getRegionInfo().getEncodedName());
final String tableName = tableDesc.getTableName().getNameAsString();
- SnapshotReferenceUtil.visitRegionStoreFiles(fs, snapshotRegionDir,
- new FSVisitor.StoreFileVisitor() {
- @Override
- public void storeFile (final String region, final String family, final String hfile)
- throws IOException {
- LOG.info("Adding HFileLink " + hfile + " to table=" + tableName);
- Path familyDir = new Path(regionDir, family);
- restoreStoreFile(familyDir, snapshotRegionInfo, hfile);
- }
- });
+ for (SnapshotRegionManifest.FamilyFiles familyFiles: manifest.getFamilyFilesList()) {
+ Path familyDir = new Path(regionDir, familyFiles.getFamilyName().toStringUtf8());
+ for (SnapshotRegionManifest.StoreFile storeFile: familyFiles.getStoreFilesList()) {
+ LOG.info("Adding HFileLink " + storeFile.getName() + " to table=" + tableName);
+ restoreStoreFile(familyDir, snapshotRegionInfo, storeFile);
+ }
+ }
}
/**
@@ -524,11 +558,12 @@ public class RestoreSnapshotHelper {
* @param hfileName store file name (can be a Reference, HFileLink or simple HFile)
*/
private void restoreStoreFile(final Path familyDir, final HRegionInfo regionInfo,
- final String hfileName) throws IOException {
+ final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+ String hfileName = storeFile.getName();
if (HFileLink.isHFileLink(hfileName)) {
HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName);
} else if (StoreFileInfo.isReference(hfileName)) {
- restoreReferenceFile(familyDir, regionInfo, hfileName);
+ restoreReferenceFile(familyDir, regionInfo, storeFile);
} else {
HFileLink.create(conf, fs, familyDir, regionInfo, hfileName);
}
@@ -553,7 +588,9 @@ public class RestoreSnapshotHelper {
* @param hfileName reference file name
*/
private void restoreReferenceFile(final Path familyDir, final HRegionInfo regionInfo,
- final String hfileName) throws IOException {
+ final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+ String hfileName = storeFile.getName();
+
// Extract the referred information (hfile name and parent region)
Path refPath = StoreFileInfo.getReferredToFile(new Path(new Path(new Path(
snapshotTable.getNameAsString(), regionInfo.getEncodedName()), familyDir.getName()),
@@ -577,16 +614,21 @@ public class RestoreSnapshotHelper {
Path outPath = new Path(familyDir, refLink + '.' + clonedRegionName);
// Create the new reference
- InputStream in;
- if (linkPath != null) {
- in = new HFileLink(conf, linkPath).open(fs);
+ if (storeFile.hasReference()) {
+ Reference reference = Reference.convert(storeFile.getReference());
+ reference.write(fs, outPath);
} else {
- linkPath = new Path(new Path(HRegion.getRegionDir(snapshotDir, regionInfo.getEncodedName()),
- familyDir.getName()), hfileName);
- in = fs.open(linkPath);
+ InputStream in;
+ if (linkPath != null) {
+ in = new HFileLink(conf, linkPath).open(fs);
+ } else {
+ linkPath = new Path(new Path(HRegion.getRegionDir(snapshotManifest.getSnapshotDir(),
+ regionInfo.getEncodedName()), familyDir.getName()), hfileName);
+ in = fs.open(linkPath);
+ }
+ OutputStream out = fs.create(outPath);
+ IOUtils.copyBytes(in, out, conf);
}
- OutputStream out = fs.create(outPath);
- IOUtils.copyBytes(in, out, conf);
// Add the daughter region to the map
String regionName = Bytes.toString(regionsMap.get(regionInfo.getEncodedNameAsBytes()));
@@ -619,43 +661,6 @@ public class RestoreSnapshotHelper {
}
/**
- * Restore snapshot WALs.
- *
- * Global Snapshot keep a reference to region servers logs present during the snapshot.
- * (/hbase/.snapshot/snapshotName/.logs/hostName/logName)
- *
- * Since each log contains different tables data, logs must be split to
- * extract the table that we are interested in.
- */
- private void restoreWALs() throws IOException {
- final SnapshotLogSplitter logSplitter = new SnapshotLogSplitter(conf, fs, tableDir,
- snapshotTable, regionsMap);
- // TODO: use executors to parallelize splitting
- // TODO: once split, we do not need to split again for other restores
- try {
- // Recover.Edits
- SnapshotReferenceUtil.visitRecoveredEdits(fs, snapshotDir,
- new FSVisitor.RecoveredEditsVisitor() {
- @Override
- public void recoveredEdits (final String region, final String logfile) throws IOException {
- Path path = SnapshotReferenceUtil.getRecoveredEdits(snapshotDir, region, logfile);
- logSplitter.splitRecoveredEdit(path);
- }
- });
-
- // Region Server Logs
- SnapshotReferenceUtil.visitLogFiles(fs, snapshotDir, new FSVisitor.LogFileVisitor() {
- @Override
- public void logFile (final String server, final String logfile) throws IOException {
- logSplitter.splitLog(server, logfile);
- }
- });
- } finally {
- logSplitter.close();
- }
- }
-
- /**
* @return the set of the regions contained in the table
*/
private List<HRegionInfo> getTableRegions() throws IOException {
@@ -720,16 +725,14 @@ public class RestoreSnapshotHelper {
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
-
- //load table descriptor
- HTableDescriptor htd = FSTableDescriptors.getTableDescriptorFromFs(fs, snapshotDir);
+ SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
MonitoredTask status = TaskMonitor.get().createStatus(
"Restoring snapshot '" + snapshotName + "' to directory " + restoreDir);
ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher();
- RestoreSnapshotHelper helper = new RestoreSnapshotHelper(conf, fs, snapshotDesc,
- snapshotDir, htd, restoreDir, monitor, status);
+ RestoreSnapshotHelper helper = new RestoreSnapshotHelper(conf, fs,
+ manifest, manifest.getTableDescriptor(), restoreDir, monitor, status);
helper.restoreHdfsRegions(); // TODO: parallelize.
if (LOG.isDebugEnabled()) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
index 34d3224..670cf67 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.snapshot;
import java.io.IOException;
import java.util.Collections;
-import com.google.protobuf.ByteString;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -32,6 +31,8 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.snapshot.SnapshotManifestV1;
+import org.apache.hadoop.hbase.snapshot.SnapshotManifestV2;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -87,11 +88,13 @@ public class SnapshotDescriptionUtils {
}
private static final Log LOG = LogFactory.getLog(SnapshotDescriptionUtils.class);
+
/**
- * Version of the fs layout for a snapshot. Future snapshots may have different file layouts,
- * which we may need to read in differently.
+ * Version of the fs layout for new snapshot.
+ * Each version of hbase should be able to read older formats.
*/
- public static final int SNAPSHOT_LAYOUT_VERSION = 0;
+ public static final String SNAPSHOT_LAYOUT_CONF_KEY = "hbase.snapshot.format.version";
+ public static final int SNAPSHOT_LAYOUT_LATEST_FORMAT = SnapshotManifestV2.DESCRIPTOR_VERSION;
// snapshot directory constants
/**
@@ -114,6 +117,15 @@ public class SnapshotDescriptionUtils {
// private constructor for utility class
}
+ public static int getDefaultSnapshotLayoutFormat(final Configuration conf) {
+ int layoutFormat = conf.getInt(SNAPSHOT_LAYOUT_CONF_KEY, SNAPSHOT_LAYOUT_LATEST_FORMAT);
+ if (layoutFormat >= SnapshotManifestV2.DESCRIPTOR_VERSION) {
+ return SnapshotManifestV2.DESCRIPTOR_VERSION;
+ }
+ return SnapshotManifestV1.DESCRIPTOR_VERSION;
+ }
+
+
/**
* @param conf {@link Configuration} from which to check for the timeout
* @param type type of snapshot being taken
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
index b11939f..fd41edc 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
@@ -25,6 +25,8 @@ import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -35,8 +37,8 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
@@ -47,8 +49,8 @@ import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.io.HFileLink;
import org.apache.hadoop.hbase.io.HLogLink;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.FSTableDescriptors;
/**
* Tool for dumping snapshot information.
@@ -105,14 +107,14 @@ public final class SnapshotInfo extends Configured implements Tool {
}
}
- private int hfileArchiveCount = 0;
- private int hfilesMissing = 0;
- private int hfilesCount = 0;
- private int logsMissing = 0;
- private int logsCount = 0;
- private long hfileArchiveSize = 0;
- private long hfileSize = 0;
- private long logSize = 0;
+ private AtomicInteger hfileArchiveCount = new AtomicInteger();
+ private AtomicInteger hfilesMissing = new AtomicInteger();
+ private AtomicInteger hfilesCount = new AtomicInteger();
+ private AtomicInteger logsMissing = new AtomicInteger();
+ private AtomicInteger logsCount = new AtomicInteger();
+ private AtomicLong hfileArchiveSize = new AtomicLong();
+ private AtomicLong hfileSize = new AtomicLong();
+ private AtomicLong logSize = new AtomicLong();
private final SnapshotDescription snapshot;
private final TableName snapshotTable;
@@ -134,57 +136,57 @@ public final class SnapshotInfo extends Configured implements Tool {
/** @return true if the snapshot is corrupted */
public boolean isSnapshotCorrupted() {
- return hfilesMissing > 0 || logsMissing > 0;
+ return hfilesMissing.get() > 0 || logsMissing.get() > 0;
}
/** @return the number of available store files */
public int getStoreFilesCount() {
- return hfilesCount + hfileArchiveCount;
+ return hfilesCount.get() + hfileArchiveCount.get();
}
/** @return the number of available store files in the archive */
public int getArchivedStoreFilesCount() {
- return hfileArchiveCount;
+ return hfileArchiveCount.get();
}
/** @return the number of available log files */
public int getLogsCount() {
- return logsCount;
+ return logsCount.get();
}
/** @return the number of missing store files */
public int getMissingStoreFilesCount() {
- return hfilesMissing;
+ return hfilesMissing.get();
}
/** @return the number of missing log files */
public int getMissingLogsCount() {
- return logsMissing;
+ return logsMissing.get();
}
/** @return the total size of the store files referenced by the snapshot */
public long getStoreFilesSize() {
- return hfileSize + hfileArchiveSize;
+ return hfileSize.get() + hfileArchiveSize.get();
}
/** @return the total size of the store files shared */
public long getSharedStoreFilesSize() {
- return hfileSize;
+ return hfileSize.get();
}
/** @return the total size of the store files in the archive */
public long getArchivedStoreFileSize() {
- return hfileArchiveSize;
+ return hfileArchiveSize.get();
}
/** @return the percentage of the shared store files */
public float getSharedStoreFilePercentage() {
- return ((float)hfileSize / (hfileSize + hfileArchiveSize)) * 100;
+ return ((float)hfileSize.get() / (hfileSize.get() + hfileArchiveSize.get())) * 100;
}
/** @return the total log size */
public long getLogsSize() {
- return logSize;
+ return logSize.get();
}
/**
@@ -194,45 +196,29 @@ public final class SnapshotInfo extends Configured implements Tool {
* @param hfile store file name
* @return the store file information
*/
- FileInfo addStoreFile(final String region, final String family, final String hfile)
- throws IOException {
- TableName table = snapshotTable;
- HFileLink link = HFileLink.create(conf, table, region, family, hfile);
+ FileInfo addStoreFile(final HRegionInfo region, final String family,
+ final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+ HFileLink link = HFileLink.create(conf, snapshotTable, region.getEncodedName(),
+ family, storeFile.getName());
boolean inArchive = false;
long size = -1;
try {
if ((inArchive = fs.exists(link.getArchivePath()))) {
size = fs.getFileStatus(link.getArchivePath()).getLen();
- hfileArchiveSize += size;
- hfileArchiveCount++;
+ hfileArchiveSize.addAndGet(size);
+ hfileArchiveCount.incrementAndGet();
} else {
size = link.getFileStatus(fs).getLen();
- hfileSize += size;
- hfilesCount++;
+ hfileSize.addAndGet(size);
+ hfilesCount.incrementAndGet();
}
} catch (FileNotFoundException e) {
- hfilesMissing++;
+ hfilesMissing.incrementAndGet();
}
return new FileInfo(inArchive, size);
}
/**
- * Add the specified recovered.edits file to the stats
- * @param region region encoded name
- * @param logfile log file name
- * @return the recovered.edits information
- */
- FileInfo addRecoveredEdits(final String region, final String logfile) throws IOException {
- Path rootDir = FSUtils.getRootDir(conf);
- Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
- Path path = SnapshotReferenceUtil.getRecoveredEdits(snapshotDir, region, logfile);
- long size = fs.getFileStatus(path).getLen();
- logSize += size;
- logsCount++;
- return new FileInfo(true, size);
- }
-
- /**
* Add the specified log file to the stats
* @param server server name
* @param logfile log file name
@@ -243,10 +229,10 @@ public final class SnapshotInfo extends Configured implements Tool {
long size = -1;
try {
size = logLink.getFileStatus(fs).getLen();
- logSize += size;
- logsCount++;
+ logSize.addAndGet(size);
+ logsCount.incrementAndGet();
} catch (FileNotFoundException e) {
- logsMissing++;
+ logsMissing.incrementAndGet();
}
return new FileInfo(false, size);
}
@@ -256,9 +242,7 @@ public final class SnapshotInfo extends Configured implements Tool {
private FileSystem fs;
private Path rootDir;
- private HTableDescriptor snapshotTableDesc;
- private SnapshotDescription snapshotDesc;
- private Path snapshotDir;
+ private SnapshotManifest snapshotManifest;
@Override
public int run(String[] args) throws IOException, InterruptedException {
@@ -344,14 +328,14 @@ public final class SnapshotInfo extends Configured implements Tool {
* @return false if snapshot is not found
*/
private boolean loadSnapshotInfo(final String snapshotName) throws IOException {
- snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
+ Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
if (!fs.exists(snapshotDir)) {
LOG.warn("Snapshot '" + snapshotName + "' not found in: " + snapshotDir);
return false;
}
- snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
- snapshotTableDesc = FSTableDescriptors.getTableDescriptorFromFs(fs, snapshotDir);
+ SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
+ snapshotManifest = SnapshotManifest.open(getConf(), fs, snapshotDir, snapshotDesc);
return true;
}
@@ -359,12 +343,13 @@ public final class SnapshotInfo extends Configured implements Tool {
* Dump the {@link SnapshotDescription}
*/
private void printInfo() {
+ SnapshotDescription snapshotDesc = snapshotManifest.getSnapshotDescription();
SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
System.out.println("Snapshot Info");
System.out.println("----------------------------------------");
System.out.println(" Name: " + snapshotDesc.getName());
System.out.println(" Type: " + snapshotDesc.getType());
- System.out.println(" Table: " + snapshotTableDesc.getTableName().getNameAsString());
+ System.out.println(" Table: " + snapshotDesc.getTable());
System.out.println(" Format: " + snapshotDesc.getVersion());
System.out.println("Created: " + df.format(new Date(snapshotDesc.getCreationTime())));
System.out.println();
@@ -376,7 +361,7 @@ public final class SnapshotInfo extends Configured implements Tool {
private void printSchema() {
System.out.println("Table Descriptor");
System.out.println("----------------------------------------");
- System.out.println(snapshotTableDesc.toString());
+ System.out.println(snapshotManifest.getTableDescriptor().toString());
System.out.println();
}
@@ -391,33 +376,27 @@ public final class SnapshotInfo extends Configured implements Tool {
}
// Collect information about hfiles and logs in the snapshot
- final String table = snapshotTableDesc.getTableName().getNameAsString();
- final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, this.snapshotDesc);
- SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
- new SnapshotReferenceUtil.FileVisitor() {
- public void storeFile (final String region, final String family, final String hfile)
- throws IOException {
- SnapshotStats.FileInfo info = stats.addStoreFile(region, family, hfile);
-
+ final SnapshotDescription snapshotDesc = snapshotManifest.getSnapshotDescription();
+ final String table = snapshotDesc.getTable();
+ final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, snapshotDesc);
+ SnapshotReferenceUtil.concurrentVisitReferencedFiles(getConf(), fs, snapshotManifest,
+ new SnapshotReferenceUtil.SnapshotVisitor() {
+ @Override
+ public void storeFile(final HRegionInfo regionInfo, final String family,
+ final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+ if (storeFile.hasReference()) return;
+
+ SnapshotStats.FileInfo info = stats.addStoreFile(regionInfo, family, storeFile);
if (showFiles) {
String state = info.getStateToString();
System.out.printf("%8s %s/%s/%s/%s %s%n",
- (info.isMissing() ? "-" : fileSizeToString(info.getSize())),
- table, region, family, hfile,
- state == null ? "" : "(" + state + ")");
- }
- }
-
- public void recoveredEdits (final String region, final String logfile)
- throws IOException {
- SnapshotStats.FileInfo info = stats.addRecoveredEdits(region, logfile);
-
- if (showFiles) {
- System.out.printf("%8s recovered.edits %s on region %s%n",
- fileSizeToString(info.getSize()), logfile, region);
+ (info.isMissing() ? "-" : StringUtils.humanReadableInt(info.getSize())),
+ table, regionInfo.getEncodedName(), family, storeFile.getName(),
+ (info.inArchive() ? "(archive)" : info.isMissing() ? "(NOT FOUND)" : ""));
}
}
+ @Override
public void logFile (final String server, final String logfile)
throws IOException {
SnapshotStats.FileInfo info = stats.addLogFile(server, logfile);
@@ -486,18 +465,19 @@ public final class SnapshotInfo extends Configured implements Tool {
Path rootDir = FSUtils.getRootDir(conf);
FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
+ SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshot);
final SnapshotStats stats = new SnapshotStats(conf, fs, snapshot);
- SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
- new SnapshotReferenceUtil.FileVisitor() {
- public void storeFile (final String region, final String family, final String hfile)
- throws IOException {
- stats.addStoreFile(region, family, hfile);
- }
-
- public void recoveredEdits (final String region, final String logfile) throws IOException {
- stats.addRecoveredEdits(region, logfile);
+ SnapshotReferenceUtil.concurrentVisitReferencedFiles(conf, fs, manifest,
+ new SnapshotReferenceUtil.SnapshotVisitor() {
+ @Override
+ public void storeFile(final HRegionInfo regionInfo, final String family,
+ final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+ if (!storeFile.hasReference()) {
+ stats.addStoreFile(regionInfo, family, storeFile);
+ }
}
+ @Override
public void logFile (final String server, final String logfile) throws IOException {
stats.addLogFile(server, logfile);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e978a16/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotLogSplitter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotLogSplitter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotLogSplitter.java
deleted file mode 100644
index 3443c5f..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotLogSplitter.java
+++ /dev/null
@@ -1,200 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.snapshot;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.Map;
-import java.util.TreeMap;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.io.HLogLink;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.wal.HLog;
-import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
-import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
-import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
-
-/**
- * If the snapshot has references to one or more log files,
- * those must be split (each log contains multiple tables and regions)
- * and must be placed in the region/recovered.edits folder.
- * (recovered.edits files will be played on region startup)
- *
- * In case of Restore: the log can just be split in the recovered.edits folder.
- * In case of Clone: each entry in the log must be modified to use the new region name.
- * (region names are encoded with: tableName, startKey, regionIdTimeStamp)
- *
- * We can't use the normal split code, because the HLogKey contains the
- * table name and the region name, and in case of "clone from snapshot"
- * region name and table name will be different and must be replaced in
- * the recovered.edits.
- */
-@InterfaceAudience.Private
-class SnapshotLogSplitter implements Closeable {
- static final Log LOG = LogFactory.getLog(SnapshotLogSplitter.class);
-
- private final class LogWriter implements Closeable {
- private HLog.Writer writer;
- private Path logFile;
- private long seqId;
-
- public LogWriter(final Configuration conf, final FileSystem fs,
- final Path logDir, long seqId) throws IOException {
- logFile = new Path(logDir, logFileName(seqId, true));
- this.writer = HLogFactory.createRecoveredEditsWriter(fs, logFile, conf);
- this.seqId = seqId;
- }
-
- public void close() throws IOException {
- writer.close();
-
- Path finalFile = new Path(logFile.getParent(), logFileName(seqId, false));
- LOG.debug("LogWriter tmpLogFile=" + logFile + " -> logFile=" + finalFile);
- fs.rename(logFile, finalFile);
- }
-
- public void append(final HLog.Entry entry) throws IOException {
- writer.append(entry);
- if (seqId < entry.getKey().getLogSeqNum()) {
- seqId = entry.getKey().getLogSeqNum();
- }
- }
-
- private String logFileName(long seqId, boolean temp) {
- String fileName = String.format("%019d", seqId);
- if (temp) fileName += HLog.RECOVERED_LOG_TMPFILE_SUFFIX;
- return fileName;
- }
- }
-
- private final Map<byte[], LogWriter> regionLogWriters =
- new TreeMap<byte[], LogWriter>(Bytes.BYTES_COMPARATOR);
-
- private final Map<byte[], byte[]> regionsMap;
- private final Configuration conf;
- private final TableName snapshotTableName;
- private final TableName tableName;
- private final Path tableDir;
- private final FileSystem fs;
-
- /**
- * @params tableName snapshot table name
- * @params regionsMap maps original region names to the new ones.
- */
- public SnapshotLogSplitter(final Configuration conf, final FileSystem fs,
- final Path tableDir, final TableName snapshotTableName,
- final Map<byte[], byte[]> regionsMap) {
- this.regionsMap = regionsMap;
- this.snapshotTableName = snapshotTableName;
- this.tableName = FSUtils.getTableName(tableDir);
- this.tableDir = tableDir;
- this.conf = conf;
- this.fs = fs;
- }
-
- public void close() throws IOException {
- for (LogWriter writer: regionLogWriters.values()) {
- writer.close();
- }
- }
-
- public void splitLog(final String serverName, final String logfile) throws IOException {
- LOG.debug("Restore log=" + logfile + " server=" + serverName +
- " for snapshotTable=" + snapshotTableName +
- " to table=" + tableName);
- splitLog(new HLogLink(conf, serverName, logfile).getAvailablePath(fs));
- }
-
- public void splitRecoveredEdit(final Path editPath) throws IOException {
- LOG.debug("Restore recover.edits=" + editPath +
- " for snapshotTable=" + snapshotTableName +
- " to table=" + tableName);
- splitLog(editPath);
- }
-
- /**
- * Split the snapshot HLog reference into regions recovered.edits.
- *
- * The HLogKey contains the table name and the region name,
- * and they must be changed to the restored table names.
- *
- * @param logPath Snapshot HLog reference path
- */
- public void splitLog(final Path logPath) throws IOException {
- HLog.Reader log = HLogFactory.createReader(fs, logPath, conf);
- try {
- HLog.Entry entry;
- LogWriter writer = null;
- byte[] regionName = null;
- byte[] newRegionName = null;
- while ((entry = log.next()) != null) {
- HLogKey key = entry.getKey();
-
- // We're interested only in the snapshot table that we're restoring
- if (!key.getTablename().equals(snapshotTableName)) continue;
-
- // Writer for region.
- if (!Bytes.equals(regionName, key.getEncodedRegionName())) {
- regionName = key.getEncodedRegionName().clone();
-
- // Get the new region name in case of clone, or use the original one
- newRegionName = regionsMap.get(regionName);
- if (newRegionName == null) newRegionName = regionName;
-
- writer = getOrCreateWriter(newRegionName, key.getLogSeqNum());
- LOG.debug("+ regionName=" + Bytes.toString(regionName));
- }
-
- // Append Entry
- key = new HLogKey(newRegionName, tableName, key.getLogSeqNum(), key.getWriteTime(),
- key.getClusterIds(), key.getNonceGroup(), key.getNonce());
- writer.append(new HLog.Entry(key, entry.getEdit()));
- }
- } catch (IOException e) {
- LOG.warn("Something wrong during the log split", e);
- } finally {
- log.close();
- }
- }
-
- /**
- * Create a LogWriter for specified region if not already created.
- */
- private LogWriter getOrCreateWriter(final byte[] regionName, long seqId) throws IOException {
- LogWriter writer = regionLogWriters.get(regionName);
- if (writer == null) {
- Path regionDir = HRegion.getRegionDir(tableDir, Bytes.toString(regionName));
- Path dir = HLogUtil.getRegionDirRecoveredEditsDir(regionDir);
- fs.mkdirs(dir);
-
- writer = new LogWriter(conf, fs, dir, seqId);
- regionLogWriters.put(regionName, writer);
- }
- return(writer);
- }
-}