You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2014/10/31 22:27:00 UTC
[01/12] git commit: ACCUMULO-3182 coverity found an unused update to
a local variable
Repository: accumulo
Updated Branches:
refs/heads/1.6 1890bea20 -> d3900b21c
refs/heads/master b220cc30f -> 0c51290bc
ACCUMULO-3182 coverity found an unused update to a local variable
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7e0ba16e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7e0ba16e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7e0ba16e
Branch: refs/heads/master
Commit: 7e0ba16eddd0a21a649ffb9007d3156df879ba58
Parents: f74681b
Author: Eric C. Newton <er...@gmail.com>
Authored: Fri Oct 31 14:33:21 2014 -0400
Committer: Eric C. Newton <er...@gmail.com>
Committed: Fri Oct 31 14:33:21 2014 -0400
----------------------------------------------------------------------
.../src/main/java/org/apache/accumulo/tserver/log/LogSorter.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7e0ba16e/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java
index 6095f88..2cbc680 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java
@@ -173,7 +173,7 @@ public class LogSorter {
}
private void writeBuffer(String destPath, List<Pair<LogFileKey,LogFileValue>> buffer, int part) throws IOException {
- Path path = new Path(destPath, String.format("part-r-%05d", part++));
+ Path path = new Path(destPath, String.format("part-r-%05d", part));
FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
@SuppressWarnings("deprecation")
[05/12] git commit: ACCUMULO-3213 Make DeleteTable archive files if
configured.
Posted by el...@apache.org.
ACCUMULO-3213 Make DeleteTable archive files if configured.
Archiving a table directory is a little more complicated since we
need to account for the potential existence of files for that table
which were already archived. Ultimately, we're merging the table directory
into the correct directory inside the fileArchive directory.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/43cee41a
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/43cee41a
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/43cee41a
Branch: refs/heads/1.6
Commit: 43cee41a0d816c1bc72141b4c47aaca43aba1eed
Parents: ded67f1
Author: Josh Elser <el...@apache.org>
Authored: Tue Oct 28 23:59:13 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Fri Oct 31 14:59:47 2014 -0400
----------------------------------------------------------------------
.../accumulo/master/tableOps/DeleteTable.java | 156 ++++++++---
.../org/apache/accumulo/test/FileArchiveIT.java | 273 +++++++++++++++++++
2 files changed, 388 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/43cee41a/server/master/src/main/java/org/apache/accumulo/master/tableOps/DeleteTable.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/DeleteTable.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/DeleteTable.java
index d264f14..f6db45b 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/DeleteTable.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/DeleteTable.java
@@ -17,8 +17,8 @@
package org.apache.accumulo.master.tableOps;
import java.io.IOException;
-import java.util.Arrays;
import java.net.UnknownHostException;
+import java.util.Arrays;
import java.util.Map.Entry;
import org.apache.accumulo.core.client.BatchScanner;
@@ -29,6 +29,8 @@ import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.impl.Tables;
import org.apache.accumulo.core.client.impl.thrift.TableOperation;
import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.data.Range;
@@ -39,6 +41,7 @@ import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.schema.MetadataSchema;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.volume.Volume;
import org.apache.accumulo.fate.Repo;
import org.apache.accumulo.master.Master;
import org.apache.accumulo.server.ServerConstants;
@@ -52,52 +55,53 @@ import org.apache.accumulo.server.security.AuditedSecurityOperation;
import org.apache.accumulo.server.security.SystemCredentials;
import org.apache.accumulo.server.tables.TableManager;
import org.apache.accumulo.server.util.MetadataTableUtil;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Logger;
class CleanUp extends MasterRepo {
-
+
final private static Logger log = Logger.getLogger(CleanUp.class);
-
+
private static final long serialVersionUID = 1L;
-
+
private String tableId, namespaceId;
-
+
private long creationTime;
-
+
private void readObject(java.io.ObjectInputStream in) throws IOException, ClassNotFoundException {
in.defaultReadObject();
-
+
/*
* handle the case where we start executing on a new machine where the current time is in the past relative to the previous machine
- *
+ *
* if the new machine has time in the future, that will work ok w/ hasCycled
*/
if (System.currentTimeMillis() < creationTime) {
creationTime = System.currentTimeMillis();
}
-
+
}
-
+
public CleanUp(String tableId, String namespaceId) {
this.tableId = tableId;
this.namespaceId = namespaceId;
creationTime = System.currentTimeMillis();
}
-
+
@Override
public long isReady(long tid, Master master) throws Exception {
if (!master.hasCycled(creationTime)) {
return 50;
}
-
+
boolean done = true;
Range tableRange = new KeyExtent(new Text(tableId), null, null).toMetadataRange();
Scanner scanner = master.getConnector().createScanner(MetadataTable.NAME, Authorizations.EMPTY);
MetaDataTableScanner.configureScanner(scanner, master);
scanner.setRange(tableRange);
-
+
KeyExtent prevExtent = null;
for (Entry<Key,Value> entry : scanner) {
TabletLocationState locationState = MetaDataTableScanner.createTabletLocationState(entry.getKey(), entry.getValue());
@@ -107,7 +111,7 @@ class CleanUp extends MasterRepo {
break;
}
prevExtent = locationState.extent;
-
+
TabletState state = locationState.getState(master.onlineTabletServers());
if (state.equals(TabletState.ASSIGNED) || state.equals(TabletState.HOSTED)) {
log.debug("Still waiting for table to be deleted: " + tableId + " locationState: " + locationState);
@@ -115,20 +119,20 @@ class CleanUp extends MasterRepo {
break;
}
}
-
+
if (!done)
return 50;
-
+
return 0;
}
-
+
@Override
public Repo<Master> call(long tid, Master master) throws Exception {
-
+
master.clearMigrations(tableId);
-
+
int refCount = 0;
-
+
try {
// look for other tables that references this table's files
Connector conn = master.getConnector();
@@ -143,7 +147,7 @@ class CleanUp extends MasterRepo {
IteratorSetting cfg = new IteratorSetting(40, "grep", GrepIterator.class);
GrepIterator.setTerm(cfg, "/" + tableId + "/");
bs.addScanIterator(cfg);
-
+
for (Entry<Key,Value> entry : bs) {
if (entry.getKey().getColumnQualifier().toString().contains("/" + tableId + "/")) {
refCount++;
@@ -152,12 +156,12 @@ class CleanUp extends MasterRepo {
} finally {
bs.close();
}
-
+
} catch (Exception e) {
refCount = -1;
log.error("Failed to scan " + MetadataTable.NAME + " looking for references to deleted table " + tableId, e);
}
-
+
// remove metadata table entries
try {
// Intentionally do not pass master lock. If master loses lock, this operation may complete before master can kill itself.
@@ -167,20 +171,27 @@ class CleanUp extends MasterRepo {
} catch (Exception e) {
log.error("error deleting " + tableId + " from metadata table", e);
}
-
+
// remove any problem reports the table may have
try {
ProblemReports.getInstance().deleteProblemReports(tableId);
} catch (Exception e) {
log.error("Failed to delete problem reports for table " + tableId, e);
}
-
+
if (refCount == 0) {
+ final AccumuloConfiguration conf = master.getConfiguration().getConfiguration();
+ boolean archiveFiles = conf.getBoolean(Property.GC_FILE_ARCHIVE);
+
// delete the map files
try {
VolumeManager fs = master.getFileSystem();
for (String dir : ServerConstants.getTablesDirs()) {
- fs.deleteRecursively(new Path(dir, tableId));
+ if (archiveFiles) {
+ archiveFile(fs, dir, tableId);
+ } else {
+ fs.deleteRecursively(new Path(dir, tableId));
+ }
}
} catch (IOException e) {
log.error("Unable to remove deleted table directory", e);
@@ -193,7 +204,7 @@ class CleanUp extends MasterRepo {
}
}
}
-
+
// remove table from zookeeper
try {
TableManager.getInstance().removeTable(tableId);
@@ -201,59 +212,122 @@ class CleanUp extends MasterRepo {
} catch (Exception e) {
log.error("Failed to find table id in zookeeper", e);
}
-
+
// remove any permissions associated with this table
try {
AuditedSecurityOperation.getInstance().deleteTable(SystemCredentials.get().toThrift(master.getInstance()), tableId, namespaceId);
} catch (ThriftSecurityException e) {
log.error(e.getMessage(), e);
}
-
+
Utils.unreserveTable(tableId, tid, true);
Utils.unreserveNamespace(namespaceId, tid, false);
-
+
Logger.getLogger(CleanUp.class).debug("Deleted table " + tableId);
-
+
return null;
}
-
+
+ protected void archiveFile(VolumeManager fs, String dir, String tableId) throws IOException {
+ Path tableDirectory = new Path(dir, tableId);
+ Volume v = fs.getVolumeByPath(tableDirectory);
+ String basePath = v.getBasePath();
+
+ // Path component of URI
+ String tableDirPath = tableDirectory.toUri().getPath();
+
+ // Just the suffix of the path (after the Volume's base path)
+ String tableDirSuffix = tableDirPath.substring(basePath.length());
+
+ // Remove a leading path separator char because Path will treat the "child" as an absolute path with it
+ if (Path.SEPARATOR_CHAR == tableDirSuffix.charAt(0)) {
+ if (tableDirSuffix.length() > 1) {
+ tableDirSuffix = tableDirSuffix.substring(1);
+ } else {
+ tableDirSuffix = "";
+ }
+ }
+
+ // Get the file archive directory on this volume
+ final Path fileArchiveDir = new Path(basePath, ServerConstants.FILE_ARCHIVE_DIR);
+
+ // Make sure it exists just to be safe
+ fs.mkdirs(fileArchiveDir);
+
+ // The destination to archive this table to
+ final Path destTableDir = new Path(fileArchiveDir, tableDirSuffix);
+
+ log.debug("Archiving " + tableDirectory + " to " + tableDirectory);
+
+ if (fs.exists(destTableDir)) {
+ merge(fs, tableDirectory, destTableDir);
+ } else {
+ fs.rename(tableDirectory, destTableDir);
+ }
+ }
+
+ protected void merge(VolumeManager fs, Path src, Path dest) throws IOException {
+ for (FileStatus child : fs.listStatus(src)) {
+ final String childName = child.getPath().getName();
+ final Path childInSrc = new Path(src, childName), childInDest = new Path(dest, childName);
+
+ if (child.isFile()) {
+ if (fs.exists(childInDest)) {
+ log.warn("File already exists in archive, ignoring. " + childInDest);
+ } else {
+ fs.rename(childInSrc, childInDest);
+ }
+ } else if (child.isDirectory()) {
+ if (fs.exists(childInDest)) {
+ // Recurse
+ merge(fs, childInSrc, childInDest);
+ } else {
+ fs.rename(childInSrc, childInDest);
+ }
+ } else {
+ // Symlinks shouldn't exist in table directories..
+ log.warn("Ignoring archiving of non file/directory: " + child);
+ }
+ }
+ }
+
@Override
public void undo(long tid, Master environment) throws Exception {
// nothing to do
}
-
+
}
public class DeleteTable extends MasterRepo {
-
+
private static final long serialVersionUID = 1L;
-
+
private String tableId, namespaceId;
-
+
public DeleteTable(String tableId) {
this.tableId = tableId;
Instance inst = HdfsZooInstance.getInstance();
this.namespaceId = Tables.getNamespaceId(inst, tableId);
}
-
+
@Override
public long isReady(long tid, Master environment) throws Exception {
-
+
return Utils.reserveNamespace(namespaceId, tid, false, false, TableOperation.DELETE)
+ Utils.reserveTable(tableId, tid, true, true, TableOperation.DELETE);
}
-
+
@Override
public Repo<Master> call(long tid, Master environment) throws Exception {
TableManager.getInstance().transitionTableState(tableId, TableState.DELETING);
environment.getEventCoordinator().event("deleting table %s ", tableId);
return new CleanUp(tableId, namespaceId);
}
-
+
@Override
public void undo(long tid, Master environment) throws Exception {
Utils.unreserveNamespace(namespaceId, tid, false);
Utils.unreserveTable(tableId, tid, true);
}
-
+
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/43cee41a/test/src/test/java/org/apache/accumulo/test/FileArchiveIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/FileArchiveIT.java b/test/src/test/java/org/apache/accumulo/test/FileArchiveIT.java
new file mode 100644
index 0000000..2e45d80
--- /dev/null
+++ b/test/src/test/java/org/apache/accumulo/test/FileArchiveIT.java
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.test.functional.ConfigurableMacIT;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.google.common.collect.Iterables;
+
+/**
+ * Tests that files are archived instead of deleted when configured.
+ */
+public class FileArchiveIT extends ConfigurableMacIT {
+
+ @Override
+ public int defaultTimeoutSeconds() {
+ return 2 * 60;
+ }
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration coreSite) {
+ cfg.setProperty(Property.GC_FILE_ARCHIVE, "true");
+ cfg.setProperty(Property.GC_CYCLE_DELAY, "1s");
+ cfg.setProperty(Property.GC_CYCLE_START, "1s");
+ }
+
+ @Test
+ public void testUnusuedFilesAreArchived() throws Exception {
+ final Connector conn = getConnector();
+ final String tableName = getUniqueNames(1)[0];
+
+ conn.tableOperations().create(tableName);
+
+ final String tableId = conn.tableOperations().tableIdMap().get(tableName);
+ Assert.assertNotNull("Could not get table ID", tableId);
+
+ BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());
+ Mutation m = new Mutation("row");
+ m.put("", "", "value");
+ bw.addMutation(m);
+ bw.close();
+
+ // Compact memory to disk
+ conn.tableOperations().compact(tableName, null, null, true, true);
+
+ Scanner s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ s.setRange(MetadataSchema.TabletsSection.getRange(tableId));
+ s.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
+
+ Entry<Key,Value> entry = Iterables.getOnlyElement(s);
+ final String file = entry.getKey().getColumnQualifier().toString();
+ final Path p = new Path(file);
+
+ // Then force another to make an unreferenced file
+ conn.tableOperations().compact(tableName, null, null, true, true);
+
+ log.info("File for table: " + file);
+
+ FileSystem fs = LocalFileSystem.get(CachedConfiguration.getInstance());
+ int i = 0;
+ while (fs.exists(p)) {
+ i++;
+ Thread.sleep(1000);
+ if (0 == i % 10) {
+ log.info("Waited " + i + " iterations, file still exists");
+ }
+ }
+
+ log.info("File was removed");
+
+ String filePath = p.toUri().getPath().substring(getCluster().getConfig().getAccumuloDir().toString().length());
+
+ log.info("File relative to accumulo dir: " + filePath);
+
+ Path fileArchiveDir = new Path(getCluster().getConfig().getAccumuloDir().toString(), ServerConstants.FILE_ARCHIVE_DIR);
+
+ Assert.assertTrue("File archive directory didn't exist", fs.exists(fileArchiveDir));
+
+ // Remove the leading '/' to make sure Path treats the 2nd arg as a child.
+ Path archivedFile = new Path(fileArchiveDir, filePath.substring(1));
+
+ Assert.assertTrue("File doesn't exists in archive directory: " + archivedFile, fs.exists(archivedFile));
+ }
+
+ @Test
+ public void testDeletedTableIsArchived() throws Exception {
+ final Connector conn = getConnector();
+ final String tableName = getUniqueNames(1)[0];
+
+ conn.tableOperations().create(tableName);
+
+ final String tableId = conn.tableOperations().tableIdMap().get(tableName);
+ Assert.assertNotNull("Could not get table ID", tableId);
+
+ BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());
+ Mutation m = new Mutation("row");
+ m.put("", "", "value");
+ bw.addMutation(m);
+ bw.close();
+
+ // Compact memory to disk
+ conn.tableOperations().compact(tableName, null, null, true, true);
+
+ Scanner s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ s.setRange(MetadataSchema.TabletsSection.getRange(tableId));
+ s.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
+
+ Entry<Key,Value> entry = Iterables.getOnlyElement(s);
+ final String file = entry.getKey().getColumnQualifier().toString();
+ final Path p = new Path(file);
+
+ conn.tableOperations().delete(tableName);
+
+ log.info("File for table: " + file);
+
+ FileSystem fs = LocalFileSystem.get(CachedConfiguration.getInstance());
+ int i = 0;
+ while (fs.exists(p)) {
+ i++;
+ Thread.sleep(1000);
+ if (0 == i % 10) {
+ log.info("Waited " + i + " iterations, file still exists");
+ }
+ }
+
+ log.info("File was removed");
+
+ String filePath = p.toUri().getPath().substring(getCluster().getConfig().getAccumuloDir().toString().length());
+
+ log.info("File relative to accumulo dir: " + filePath);
+
+ Path fileArchiveDir = new Path(getCluster().getConfig().getAccumuloDir().toString(), ServerConstants.FILE_ARCHIVE_DIR);
+
+ Assert.assertTrue("File archive directory didn't exist", fs.exists(fileArchiveDir));
+
+ // Remove the leading '/' to make sure Path treats the 2nd arg as a child.
+ Path archivedFile = new Path(fileArchiveDir, filePath.substring(1));
+
+ Assert.assertTrue("File doesn't exists in archive directory: " + archivedFile, fs.exists(archivedFile));
+ }
+
+ @Test
+ public void testUnusuedFilesAndDeletedTable() throws Exception {
+ final Connector conn = getConnector();
+ final String tableName = getUniqueNames(1)[0];
+
+ conn.tableOperations().create(tableName);
+
+ final String tableId = conn.tableOperations().tableIdMap().get(tableName);
+ Assert.assertNotNull("Could not get table ID", tableId);
+
+ BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());
+ Mutation m = new Mutation("row");
+ m.put("", "", "value");
+ bw.addMutation(m);
+ bw.close();
+
+ // Compact memory to disk
+ conn.tableOperations().compact(tableName, null, null, true, true);
+
+ Scanner s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ s.setRange(MetadataSchema.TabletsSection.getRange(tableId));
+ s.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
+
+ Entry<Key,Value> entry = Iterables.getOnlyElement(s);
+ final String file = entry.getKey().getColumnQualifier().toString();
+ final Path p = new Path(file);
+
+ // Then force another to make an unreferenced file
+ conn.tableOperations().compact(tableName, null, null, true, true);
+
+ log.info("File for table: " + file);
+
+ FileSystem fs = LocalFileSystem.get(CachedConfiguration.getInstance());
+ int i = 0;
+ while (fs.exists(p)) {
+ i++;
+ Thread.sleep(1000);
+ if (0 == i % 10) {
+ log.info("Waited " + i + " iterations, file still exists");
+ }
+ }
+
+ log.info("File was removed");
+
+ String filePath = p.toUri().getPath().substring(getCluster().getConfig().getAccumuloDir().toString().length());
+
+ log.info("File relative to accumulo dir: " + filePath);
+
+ Path fileArchiveDir = new Path(getCluster().getConfig().getAccumuloDir().toString(), ServerConstants.FILE_ARCHIVE_DIR);
+
+ Assert.assertTrue("File archive directory didn't exist", fs.exists(fileArchiveDir));
+
+ // Remove the leading '/' to make sure Path treats the 2nd arg as a child.
+ Path archivedFile = new Path(fileArchiveDir, filePath.substring(1));
+
+ Assert.assertTrue("File doesn't exists in archive directory: " + archivedFile, fs.exists(archivedFile));
+
+ // Offline the table so we can be sure there is a single file
+ conn.tableOperations().offline(tableName, true);
+
+ // See that the file in metadata currently is
+ s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ s.setRange(MetadataSchema.TabletsSection.getRange(tableId));
+ s.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
+
+ entry = Iterables.getOnlyElement(s);
+ final String finalFile = entry.getKey().getColumnQualifier().toString();
+ final Path finalPath = new Path(finalFile);
+
+ conn.tableOperations().delete(tableName);
+
+ log.info("File for table: " + finalPath);
+
+ i = 0;
+ while (fs.exists(finalPath)) {
+ i++;
+ Thread.sleep(1000);
+ if (0 == i % 10) {
+ log.info("Waited " + i + " iterations, file still exists");
+ }
+ }
+
+ log.info("File was removed");
+
+ String finalFilePath = finalPath.toUri().getPath().substring(getCluster().getConfig().getAccumuloDir().toString().length());
+
+ log.info("File relative to accumulo dir: " + finalFilePath);
+
+ Assert.assertTrue("File archive directory didn't exist", fs.exists(fileArchiveDir));
+
+ // Remove the leading '/' to make sure Path treats the 2nd arg as a child.
+ Path finalArchivedFile = new Path(fileArchiveDir, finalFilePath.substring(1));
+
+ Assert.assertTrue("File doesn't exists in archive directory: " + finalArchivedFile, fs.exists(finalArchivedFile));
+ }
+
+}
[11/12] git commit: Merge branch '1.6'
Posted by el...@apache.org.
Merge branch '1.6'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/84d3b95e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/84d3b95e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/84d3b95e
Branch: refs/heads/master
Commit: 84d3b95eba4a8d57b789dd55b7a5515b3acf1d30
Parents: b220cc3 d3900b2
Author: Josh Elser <el...@apache.org>
Authored: Fri Oct 31 16:53:54 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Fri Oct 31 16:53:54 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/core/conf/Property.java | 1 +
.../apache/accumulo/server/ServerConstants.java | 8 +-
.../accumulo/server/util/RandomizeVolumes.java | 5 +-
.../accumulo/gc/SimpleGarbageCollector.java | 81 +++++-
.../accumulo/gc/SimpleGarbageCollectorTest.java | 34 +--
.../accumulo/master/tableOps/DeleteTable.java | 156 ++++++++---
.../apache/accumulo/tserver/log/LogSorter.java | 2 +-
.../org/apache/accumulo/test/FileArchiveIT.java | 273 +++++++++++++++++++
8 files changed, 490 insertions(+), 70 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/84d3b95e/core/src/main/java/org/apache/accumulo/core/conf/Property.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/84d3b95e/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/84d3b95e/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/84d3b95e/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/84d3b95e/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/84d3b95e/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java
----------------------------------------------------------------------
[10/12] git commit: ACCUMULO-3213 Fix up SimpleGarbageCollectorTest
Posted by el...@apache.org.
ACCUMULO-3213 Fix up SimpleGarbageCollectorTest
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d3900b21
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d3900b21
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d3900b21
Branch: refs/heads/1.6
Commit: d3900b21c79e1258c6a540fba866efef0ba8cd86
Parents: 43cee41
Author: Josh Elser <el...@apache.org>
Authored: Fri Oct 31 16:03:59 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Fri Oct 31 16:03:59 2014 -0400
----------------------------------------------------------------------
.../accumulo/gc/SimpleGarbageCollectorTest.java | 28 ++++++++++----------
1 file changed, 14 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d3900b21/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorTest.java
----------------------------------------------------------------------
diff --git a/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorTest.java b/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorTest.java
index 672aee2..192a858 100644
--- a/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorTest.java
+++ b/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorTest.java
@@ -16,6 +16,18 @@
*/
package org.apache.accumulo.gc;
+import static org.apache.accumulo.gc.SimpleGarbageCollector.CANDIDATE_MEMORY_PERCENTAGE;
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+
import java.io.FileNotFoundException;
import org.apache.accumulo.core.client.Instance;
@@ -24,9 +36,6 @@ import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.security.Credentials;
import org.apache.accumulo.core.security.thrift.TCredentials;
import org.apache.accumulo.gc.SimpleGarbageCollector.Opts;
-
-import static org.apache.accumulo.gc.SimpleGarbageCollector.CANDIDATE_MEMORY_PERCENTAGE;
-
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.trace.thrift.TInfo;
import org.apache.hadoop.fs.Path;
@@ -34,17 +43,6 @@ import org.easymock.EasyMock;
import org.junit.Before;
import org.junit.Test;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertSame;
-import static org.junit.Assert.assertTrue;
-import static org.easymock.EasyMock.createMock;
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.expectLastCall;
-import static org.easymock.EasyMock.replay;
-import static org.easymock.EasyMock.verify;
-
public class SimpleGarbageCollectorTest {
private VolumeManager volMgr;
private Instance instance;
@@ -76,6 +74,7 @@ public class SimpleGarbageCollectorTest {
expect(systemConfig.getTimeInMillis(Property.GC_CYCLE_DELAY)).andReturn(20000L);
expect(systemConfig.getCount(Property.GC_DELETE_THREADS)).andReturn(2).times(2);
expect(systemConfig.getBoolean(Property.GC_TRASH_IGNORE)).andReturn(false);
+ expect(systemConfig.getBoolean(Property.GC_FILE_ARCHIVE)).andReturn(false);
replay(systemConfig);
return systemConfig;
}
@@ -123,6 +122,7 @@ public class SimpleGarbageCollectorTest {
expect(systemConfig.getTimeInMillis(Property.GC_CYCLE_START)).andReturn(1000L);
expect(systemConfig.getTimeInMillis(Property.GC_CYCLE_DELAY)).andReturn(20000L);
expect(systemConfig.getCount(Property.GC_DELETE_THREADS)).andReturn(2);
+ expect(systemConfig.getBoolean(Property.GC_FILE_ARCHIVE)).andReturn(false);
expect(systemConfig.getBoolean(Property.GC_TRASH_IGNORE)).andReturn(true);
replay(systemConfig);
gc.init(volMgr, instance, credentials, systemConfig);
[07/12] git commit: ACCUMULO-3213 Fix the moveToTrash method name to
be more accurate (archive or move to trash)
Posted by el...@apache.org.
ACCUMULO-3213 Fix the moveToTrash method name to be more accurate (archive or move to trash)
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/ded67f18
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/ded67f18
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/ded67f18
Branch: refs/heads/master
Commit: ded67f1867874d46e69b4d28084eaed3a3844009
Parents: 7ce1a64
Author: Josh Elser <el...@apache.org>
Authored: Tue Oct 28 23:58:50 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Fri Oct 31 14:59:47 2014 -0400
----------------------------------------------------------------------
.../java/org/apache/accumulo/gc/SimpleGarbageCollector.java | 6 +++---
.../org/apache/accumulo/gc/SimpleGarbageCollectorTest.java | 6 +++---
2 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ded67f18/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
----------------------------------------------------------------------
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
index 308d7b9..55548e3 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
@@ -406,7 +406,7 @@ public class SimpleGarbageCollector implements Iface {
log.debug("Deleting " + fullPath);
- if (moveToTrash(fullPath) || fs.deleteRecursively(fullPath)) {
+ if (archiveOrMoveToTrash(fullPath) || fs.deleteRecursively(fullPath)) {
// delete succeeded, still want to delete
removeFlag = true;
synchronized (SimpleGarbageCollector.this) {
@@ -492,7 +492,7 @@ public class SimpleGarbageCollector implements Iface {
if (tabletDirs.length == 0) {
Path p = new Path(dir + "/" + tableID);
log.debug("Removing table dir " + p);
- if (!moveToTrash(p))
+ if (!archiveOrMoveToTrash(p))
fs.delete(p);
}
}
@@ -606,7 +606,7 @@ public class SimpleGarbageCollector implements Iface {
* @throws IOException
* if the volume manager encountered a problem
*/
- boolean moveToTrash(Path path) throws IOException {
+ boolean archiveOrMoveToTrash(Path path) throws IOException {
if (shouldArchiveFiles()) {
return archiveFile(path);
} else {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ded67f18/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorTest.java
----------------------------------------------------------------------
diff --git a/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorTest.java b/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorTest.java
index d4319b8..672aee2 100644
--- a/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorTest.java
+++ b/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorTest.java
@@ -103,7 +103,7 @@ public class SimpleGarbageCollectorTest {
Path path = createMock(Path.class);
expect(volMgr.moveToTrash(path)).andReturn(true);
replay(volMgr);
- assertTrue(gc.moveToTrash(path));
+ assertTrue(gc.archiveOrMoveToTrash(path));
verify(volMgr);
}
@@ -113,7 +113,7 @@ public class SimpleGarbageCollectorTest {
Path path = createMock(Path.class);
expect(volMgr.moveToTrash(path)).andThrow(new FileNotFoundException());
replay(volMgr);
- assertFalse(gc.moveToTrash(path));
+ assertFalse(gc.archiveOrMoveToTrash(path));
verify(volMgr);
}
@@ -127,7 +127,7 @@ public class SimpleGarbageCollectorTest {
replay(systemConfig);
gc.init(volMgr, instance, credentials, systemConfig);
Path path = createMock(Path.class);
- assertFalse(gc.moveToTrash(path));
+ assertFalse(gc.archiveOrMoveToTrash(path));
}
@Test
[12/12] git commit: ACCUMULO-3213 Fix up merge issues.
Posted by el...@apache.org.
ACCUMULO-3213 Fix up merge issues.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/0c51290b
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/0c51290b
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/0c51290b
Branch: refs/heads/master
Commit: 0c51290bce812e5d98462099f335d855ed2b91bc
Parents: 84d3b95
Author: Josh Elser <el...@apache.org>
Authored: Fri Oct 31 17:00:17 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Fri Oct 31 17:00:17 2014 -0400
----------------------------------------------------------------------
.../java/org/apache/accumulo/server/util/RandomizeVolumes.java | 3 ++-
.../java/org/apache/accumulo/master/tableOps/DeleteTable.java | 2 +-
2 files changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0c51290b/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java b/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
index 088d16b..4c5326e 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
@@ -17,6 +17,7 @@
package org.apache.accumulo.server.util;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.Map.Entry;
import org.apache.accumulo.core.Constants;
@@ -107,7 +108,7 @@ public class RandomizeVolumes {
Mutation m = new Mutation(key.getRow());
String newLocation = vm.choose(ServerConstants.getBaseUris()) + Path.SEPARATOR + ServerConstants.TABLE_DIR + Path.SEPARATOR + tableId + Path.SEPARATOR + directory;
- m.put(key.getColumnFamily(), key.getColumnQualifier(), new Value(newLocation.getBytes(Constants.UTF8)));
+ m.put(key.getColumnFamily(), key.getColumnQualifier(), new Value(newLocation.getBytes(StandardCharsets.UTF_8)));
if (log.isTraceEnabled()) {
log.trace("Replacing " + oldLocation + " with " + newLocation);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0c51290b/server/master/src/main/java/org/apache/accumulo/master/tableOps/DeleteTable.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/DeleteTable.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/DeleteTable.java
index f6db45b..2ad3f7b 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/DeleteTable.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/DeleteTable.java
@@ -180,7 +180,7 @@ class CleanUp extends MasterRepo {
}
if (refCount == 0) {
- final AccumuloConfiguration conf = master.getConfiguration().getConfiguration();
+ final AccumuloConfiguration conf = master.getConfiguration();
boolean archiveFiles = conf.getBoolean(Property.GC_FILE_ARCHIVE);
// delete the map files
[08/12] git commit: ACCUMULO-3213 Make DeleteTable archive files if
configured.
Posted by el...@apache.org.
ACCUMULO-3213 Make DeleteTable archive files if configured.
Archiving a table directory is a little more complicated since we
need to account for the potential existence of files for that table
which were already archived. Ultimately, we're merging the table directory
into the correct directory inside the fileArchive directory.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/43cee41a
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/43cee41a
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/43cee41a
Branch: refs/heads/master
Commit: 43cee41a0d816c1bc72141b4c47aaca43aba1eed
Parents: ded67f1
Author: Josh Elser <el...@apache.org>
Authored: Tue Oct 28 23:59:13 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Fri Oct 31 14:59:47 2014 -0400
----------------------------------------------------------------------
.../accumulo/master/tableOps/DeleteTable.java | 156 ++++++++---
.../org/apache/accumulo/test/FileArchiveIT.java | 273 +++++++++++++++++++
2 files changed, 388 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/43cee41a/server/master/src/main/java/org/apache/accumulo/master/tableOps/DeleteTable.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/DeleteTable.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/DeleteTable.java
index d264f14..f6db45b 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/DeleteTable.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/DeleteTable.java
@@ -17,8 +17,8 @@
package org.apache.accumulo.master.tableOps;
import java.io.IOException;
-import java.util.Arrays;
import java.net.UnknownHostException;
+import java.util.Arrays;
import java.util.Map.Entry;
import org.apache.accumulo.core.client.BatchScanner;
@@ -29,6 +29,8 @@ import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.impl.Tables;
import org.apache.accumulo.core.client.impl.thrift.TableOperation;
import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.data.Range;
@@ -39,6 +41,7 @@ import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.schema.MetadataSchema;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.volume.Volume;
import org.apache.accumulo.fate.Repo;
import org.apache.accumulo.master.Master;
import org.apache.accumulo.server.ServerConstants;
@@ -52,52 +55,53 @@ import org.apache.accumulo.server.security.AuditedSecurityOperation;
import org.apache.accumulo.server.security.SystemCredentials;
import org.apache.accumulo.server.tables.TableManager;
import org.apache.accumulo.server.util.MetadataTableUtil;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Logger;
class CleanUp extends MasterRepo {
-
+
final private static Logger log = Logger.getLogger(CleanUp.class);
-
+
private static final long serialVersionUID = 1L;
-
+
private String tableId, namespaceId;
-
+
private long creationTime;
-
+
private void readObject(java.io.ObjectInputStream in) throws IOException, ClassNotFoundException {
in.defaultReadObject();
-
+
/*
* handle the case where we start executing on a new machine where the current time is in the past relative to the previous machine
- *
+ *
* if the new machine has time in the future, that will work ok w/ hasCycled
*/
if (System.currentTimeMillis() < creationTime) {
creationTime = System.currentTimeMillis();
}
-
+
}
-
+
public CleanUp(String tableId, String namespaceId) {
this.tableId = tableId;
this.namespaceId = namespaceId;
creationTime = System.currentTimeMillis();
}
-
+
@Override
public long isReady(long tid, Master master) throws Exception {
if (!master.hasCycled(creationTime)) {
return 50;
}
-
+
boolean done = true;
Range tableRange = new KeyExtent(new Text(tableId), null, null).toMetadataRange();
Scanner scanner = master.getConnector().createScanner(MetadataTable.NAME, Authorizations.EMPTY);
MetaDataTableScanner.configureScanner(scanner, master);
scanner.setRange(tableRange);
-
+
KeyExtent prevExtent = null;
for (Entry<Key,Value> entry : scanner) {
TabletLocationState locationState = MetaDataTableScanner.createTabletLocationState(entry.getKey(), entry.getValue());
@@ -107,7 +111,7 @@ class CleanUp extends MasterRepo {
break;
}
prevExtent = locationState.extent;
-
+
TabletState state = locationState.getState(master.onlineTabletServers());
if (state.equals(TabletState.ASSIGNED) || state.equals(TabletState.HOSTED)) {
log.debug("Still waiting for table to be deleted: " + tableId + " locationState: " + locationState);
@@ -115,20 +119,20 @@ class CleanUp extends MasterRepo {
break;
}
}
-
+
if (!done)
return 50;
-
+
return 0;
}
-
+
@Override
public Repo<Master> call(long tid, Master master) throws Exception {
-
+
master.clearMigrations(tableId);
-
+
int refCount = 0;
-
+
try {
// look for other tables that references this table's files
Connector conn = master.getConnector();
@@ -143,7 +147,7 @@ class CleanUp extends MasterRepo {
IteratorSetting cfg = new IteratorSetting(40, "grep", GrepIterator.class);
GrepIterator.setTerm(cfg, "/" + tableId + "/");
bs.addScanIterator(cfg);
-
+
for (Entry<Key,Value> entry : bs) {
if (entry.getKey().getColumnQualifier().toString().contains("/" + tableId + "/")) {
refCount++;
@@ -152,12 +156,12 @@ class CleanUp extends MasterRepo {
} finally {
bs.close();
}
-
+
} catch (Exception e) {
refCount = -1;
log.error("Failed to scan " + MetadataTable.NAME + " looking for references to deleted table " + tableId, e);
}
-
+
// remove metadata table entries
try {
// Intentionally do not pass master lock. If master loses lock, this operation may complete before master can kill itself.
@@ -167,20 +171,27 @@ class CleanUp extends MasterRepo {
} catch (Exception e) {
log.error("error deleting " + tableId + " from metadata table", e);
}
-
+
// remove any problem reports the table may have
try {
ProblemReports.getInstance().deleteProblemReports(tableId);
} catch (Exception e) {
log.error("Failed to delete problem reports for table " + tableId, e);
}
-
+
if (refCount == 0) {
+ final AccumuloConfiguration conf = master.getConfiguration().getConfiguration();
+ boolean archiveFiles = conf.getBoolean(Property.GC_FILE_ARCHIVE);
+
// delete the map files
try {
VolumeManager fs = master.getFileSystem();
for (String dir : ServerConstants.getTablesDirs()) {
- fs.deleteRecursively(new Path(dir, tableId));
+ if (archiveFiles) {
+ archiveFile(fs, dir, tableId);
+ } else {
+ fs.deleteRecursively(new Path(dir, tableId));
+ }
}
} catch (IOException e) {
log.error("Unable to remove deleted table directory", e);
@@ -193,7 +204,7 @@ class CleanUp extends MasterRepo {
}
}
}
-
+
// remove table from zookeeper
try {
TableManager.getInstance().removeTable(tableId);
@@ -201,59 +212,122 @@ class CleanUp extends MasterRepo {
} catch (Exception e) {
log.error("Failed to find table id in zookeeper", e);
}
-
+
// remove any permissions associated with this table
try {
AuditedSecurityOperation.getInstance().deleteTable(SystemCredentials.get().toThrift(master.getInstance()), tableId, namespaceId);
} catch (ThriftSecurityException e) {
log.error(e.getMessage(), e);
}
-
+
Utils.unreserveTable(tableId, tid, true);
Utils.unreserveNamespace(namespaceId, tid, false);
-
+
Logger.getLogger(CleanUp.class).debug("Deleted table " + tableId);
-
+
return null;
}
-
+
+ protected void archiveFile(VolumeManager fs, String dir, String tableId) throws IOException {
+ Path tableDirectory = new Path(dir, tableId);
+ Volume v = fs.getVolumeByPath(tableDirectory);
+ String basePath = v.getBasePath();
+
+ // Path component of URI
+ String tableDirPath = tableDirectory.toUri().getPath();
+
+ // Just the suffix of the path (after the Volume's base path)
+ String tableDirSuffix = tableDirPath.substring(basePath.length());
+
+ // Remove a leading path separator char because Path will treat the "child" as an absolute path with it
+ if (Path.SEPARATOR_CHAR == tableDirSuffix.charAt(0)) {
+ if (tableDirSuffix.length() > 1) {
+ tableDirSuffix = tableDirSuffix.substring(1);
+ } else {
+ tableDirSuffix = "";
+ }
+ }
+
+ // Get the file archive directory on this volume
+ final Path fileArchiveDir = new Path(basePath, ServerConstants.FILE_ARCHIVE_DIR);
+
+ // Make sure it exists just to be safe
+ fs.mkdirs(fileArchiveDir);
+
+ // The destination to archive this table to
+ final Path destTableDir = new Path(fileArchiveDir, tableDirSuffix);
+
+ log.debug("Archiving " + tableDirectory + " to " + tableDirectory);
+
+ if (fs.exists(destTableDir)) {
+ merge(fs, tableDirectory, destTableDir);
+ } else {
+ fs.rename(tableDirectory, destTableDir);
+ }
+ }
+
+ protected void merge(VolumeManager fs, Path src, Path dest) throws IOException {
+ for (FileStatus child : fs.listStatus(src)) {
+ final String childName = child.getPath().getName();
+ final Path childInSrc = new Path(src, childName), childInDest = new Path(dest, childName);
+
+ if (child.isFile()) {
+ if (fs.exists(childInDest)) {
+ log.warn("File already exists in archive, ignoring. " + childInDest);
+ } else {
+ fs.rename(childInSrc, childInDest);
+ }
+ } else if (child.isDirectory()) {
+ if (fs.exists(childInDest)) {
+ // Recurse
+ merge(fs, childInSrc, childInDest);
+ } else {
+ fs.rename(childInSrc, childInDest);
+ }
+ } else {
+ // Symlinks shouldn't exist in table directories..
+ log.warn("Ignoring archiving of non file/directory: " + child);
+ }
+ }
+ }
+
@Override
public void undo(long tid, Master environment) throws Exception {
// nothing to do
}
-
+
}
public class DeleteTable extends MasterRepo {
-
+
private static final long serialVersionUID = 1L;
-
+
private String tableId, namespaceId;
-
+
public DeleteTable(String tableId) {
this.tableId = tableId;
Instance inst = HdfsZooInstance.getInstance();
this.namespaceId = Tables.getNamespaceId(inst, tableId);
}
-
+
@Override
public long isReady(long tid, Master environment) throws Exception {
-
+
return Utils.reserveNamespace(namespaceId, tid, false, false, TableOperation.DELETE)
+ Utils.reserveTable(tableId, tid, true, true, TableOperation.DELETE);
}
-
+
@Override
public Repo<Master> call(long tid, Master environment) throws Exception {
TableManager.getInstance().transitionTableState(tableId, TableState.DELETING);
environment.getEventCoordinator().event("deleting table %s ", tableId);
return new CleanUp(tableId, namespaceId);
}
-
+
@Override
public void undo(long tid, Master environment) throws Exception {
Utils.unreserveNamespace(namespaceId, tid, false);
Utils.unreserveTable(tableId, tid, true);
}
-
+
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/43cee41a/test/src/test/java/org/apache/accumulo/test/FileArchiveIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/FileArchiveIT.java b/test/src/test/java/org/apache/accumulo/test/FileArchiveIT.java
new file mode 100644
index 0000000..2e45d80
--- /dev/null
+++ b/test/src/test/java/org/apache/accumulo/test/FileArchiveIT.java
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.test.functional.ConfigurableMacIT;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.google.common.collect.Iterables;
+
+/**
+ * Tests that files are archived instead of deleted when configured.
+ */
+public class FileArchiveIT extends ConfigurableMacIT {
+
+ @Override
+ public int defaultTimeoutSeconds() {
+ return 2 * 60;
+ }
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration coreSite) {
+ cfg.setProperty(Property.GC_FILE_ARCHIVE, "true");
+ cfg.setProperty(Property.GC_CYCLE_DELAY, "1s");
+ cfg.setProperty(Property.GC_CYCLE_START, "1s");
+ }
+
+ @Test
+ public void testUnusuedFilesAreArchived() throws Exception {
+ final Connector conn = getConnector();
+ final String tableName = getUniqueNames(1)[0];
+
+ conn.tableOperations().create(tableName);
+
+ final String tableId = conn.tableOperations().tableIdMap().get(tableName);
+ Assert.assertNotNull("Could not get table ID", tableId);
+
+ BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());
+ Mutation m = new Mutation("row");
+ m.put("", "", "value");
+ bw.addMutation(m);
+ bw.close();
+
+ // Compact memory to disk
+ conn.tableOperations().compact(tableName, null, null, true, true);
+
+ Scanner s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ s.setRange(MetadataSchema.TabletsSection.getRange(tableId));
+ s.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
+
+ Entry<Key,Value> entry = Iterables.getOnlyElement(s);
+ final String file = entry.getKey().getColumnQualifier().toString();
+ final Path p = new Path(file);
+
+ // Then force another to make an unreferenced file
+ conn.tableOperations().compact(tableName, null, null, true, true);
+
+ log.info("File for table: " + file);
+
+ FileSystem fs = LocalFileSystem.get(CachedConfiguration.getInstance());
+ int i = 0;
+ while (fs.exists(p)) {
+ i++;
+ Thread.sleep(1000);
+ if (0 == i % 10) {
+ log.info("Waited " + i + " iterations, file still exists");
+ }
+ }
+
+ log.info("File was removed");
+
+ String filePath = p.toUri().getPath().substring(getCluster().getConfig().getAccumuloDir().toString().length());
+
+ log.info("File relative to accumulo dir: " + filePath);
+
+ Path fileArchiveDir = new Path(getCluster().getConfig().getAccumuloDir().toString(), ServerConstants.FILE_ARCHIVE_DIR);
+
+ Assert.assertTrue("File archive directory didn't exist", fs.exists(fileArchiveDir));
+
+ // Remove the leading '/' to make sure Path treats the 2nd arg as a child.
+ Path archivedFile = new Path(fileArchiveDir, filePath.substring(1));
+
+ Assert.assertTrue("File doesn't exists in archive directory: " + archivedFile, fs.exists(archivedFile));
+ }
+
+ @Test
+ public void testDeletedTableIsArchived() throws Exception {
+ final Connector conn = getConnector();
+ final String tableName = getUniqueNames(1)[0];
+
+ conn.tableOperations().create(tableName);
+
+ final String tableId = conn.tableOperations().tableIdMap().get(tableName);
+ Assert.assertNotNull("Could not get table ID", tableId);
+
+ BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());
+ Mutation m = new Mutation("row");
+ m.put("", "", "value");
+ bw.addMutation(m);
+ bw.close();
+
+ // Compact memory to disk
+ conn.tableOperations().compact(tableName, null, null, true, true);
+
+ Scanner s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ s.setRange(MetadataSchema.TabletsSection.getRange(tableId));
+ s.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
+
+ Entry<Key,Value> entry = Iterables.getOnlyElement(s);
+ final String file = entry.getKey().getColumnQualifier().toString();
+ final Path p = new Path(file);
+
+ conn.tableOperations().delete(tableName);
+
+ log.info("File for table: " + file);
+
+ FileSystem fs = LocalFileSystem.get(CachedConfiguration.getInstance());
+ int i = 0;
+ while (fs.exists(p)) {
+ i++;
+ Thread.sleep(1000);
+ if (0 == i % 10) {
+ log.info("Waited " + i + " iterations, file still exists");
+ }
+ }
+
+ log.info("File was removed");
+
+ String filePath = p.toUri().getPath().substring(getCluster().getConfig().getAccumuloDir().toString().length());
+
+ log.info("File relative to accumulo dir: " + filePath);
+
+ Path fileArchiveDir = new Path(getCluster().getConfig().getAccumuloDir().toString(), ServerConstants.FILE_ARCHIVE_DIR);
+
+ Assert.assertTrue("File archive directory didn't exist", fs.exists(fileArchiveDir));
+
+ // Remove the leading '/' to make sure Path treats the 2nd arg as a child.
+ Path archivedFile = new Path(fileArchiveDir, filePath.substring(1));
+
+ Assert.assertTrue("File doesn't exists in archive directory: " + archivedFile, fs.exists(archivedFile));
+ }
+
+ @Test
+ public void testUnusuedFilesAndDeletedTable() throws Exception {
+ final Connector conn = getConnector();
+ final String tableName = getUniqueNames(1)[0];
+
+ conn.tableOperations().create(tableName);
+
+ final String tableId = conn.tableOperations().tableIdMap().get(tableName);
+ Assert.assertNotNull("Could not get table ID", tableId);
+
+ BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());
+ Mutation m = new Mutation("row");
+ m.put("", "", "value");
+ bw.addMutation(m);
+ bw.close();
+
+ // Compact memory to disk
+ conn.tableOperations().compact(tableName, null, null, true, true);
+
+ Scanner s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ s.setRange(MetadataSchema.TabletsSection.getRange(tableId));
+ s.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
+
+ Entry<Key,Value> entry = Iterables.getOnlyElement(s);
+ final String file = entry.getKey().getColumnQualifier().toString();
+ final Path p = new Path(file);
+
+ // Then force another to make an unreferenced file
+ conn.tableOperations().compact(tableName, null, null, true, true);
+
+ log.info("File for table: " + file);
+
+ FileSystem fs = LocalFileSystem.get(CachedConfiguration.getInstance());
+ int i = 0;
+ while (fs.exists(p)) {
+ i++;
+ Thread.sleep(1000);
+ if (0 == i % 10) {
+ log.info("Waited " + i + " iterations, file still exists");
+ }
+ }
+
+ log.info("File was removed");
+
+ String filePath = p.toUri().getPath().substring(getCluster().getConfig().getAccumuloDir().toString().length());
+
+ log.info("File relative to accumulo dir: " + filePath);
+
+ Path fileArchiveDir = new Path(getCluster().getConfig().getAccumuloDir().toString(), ServerConstants.FILE_ARCHIVE_DIR);
+
+ Assert.assertTrue("File archive directory didn't exist", fs.exists(fileArchiveDir));
+
+ // Remove the leading '/' to make sure Path treats the 2nd arg as a child.
+ Path archivedFile = new Path(fileArchiveDir, filePath.substring(1));
+
+ Assert.assertTrue("File doesn't exists in archive directory: " + archivedFile, fs.exists(archivedFile));
+
+ // Offline the table so we can be sure there is a single file
+ conn.tableOperations().offline(tableName, true);
+
+ // See that the file in metadata currently is
+ s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ s.setRange(MetadataSchema.TabletsSection.getRange(tableId));
+ s.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
+
+ entry = Iterables.getOnlyElement(s);
+ final String finalFile = entry.getKey().getColumnQualifier().toString();
+ final Path finalPath = new Path(finalFile);
+
+ conn.tableOperations().delete(tableName);
+
+ log.info("File for table: " + finalPath);
+
+ i = 0;
+ while (fs.exists(finalPath)) {
+ i++;
+ Thread.sleep(1000);
+ if (0 == i % 10) {
+ log.info("Waited " + i + " iterations, file still exists");
+ }
+ }
+
+ log.info("File was removed");
+
+ String finalFilePath = finalPath.toUri().getPath().substring(getCluster().getConfig().getAccumuloDir().toString().length());
+
+ log.info("File relative to accumulo dir: " + finalFilePath);
+
+ Assert.assertTrue("File archive directory didn't exist", fs.exists(fileArchiveDir));
+
+ // Remove the leading '/' to make sure Path treats the 2nd arg as a child.
+ Path finalArchivedFile = new Path(fileArchiveDir, finalFilePath.substring(1));
+
+ Assert.assertTrue("File doesn't exists in archive directory: " + finalArchivedFile, fs.exists(finalArchivedFile));
+ }
+
+}
[03/12] git commit: ACCUMULO-3213 Fix the moveToTrash method name to
be more accurate (archive or move to trash)
Posted by el...@apache.org.
ACCUMULO-3213 Fix the moveToTrash method name to be more accurate (archive or move to trash)
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/ded67f18
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/ded67f18
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/ded67f18
Branch: refs/heads/1.6
Commit: ded67f1867874d46e69b4d28084eaed3a3844009
Parents: 7ce1a64
Author: Josh Elser <el...@apache.org>
Authored: Tue Oct 28 23:58:50 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Fri Oct 31 14:59:47 2014 -0400
----------------------------------------------------------------------
.../java/org/apache/accumulo/gc/SimpleGarbageCollector.java | 6 +++---
.../org/apache/accumulo/gc/SimpleGarbageCollectorTest.java | 6 +++---
2 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ded67f18/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
----------------------------------------------------------------------
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
index 308d7b9..55548e3 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
@@ -406,7 +406,7 @@ public class SimpleGarbageCollector implements Iface {
log.debug("Deleting " + fullPath);
- if (moveToTrash(fullPath) || fs.deleteRecursively(fullPath)) {
+ if (archiveOrMoveToTrash(fullPath) || fs.deleteRecursively(fullPath)) {
// delete succeeded, still want to delete
removeFlag = true;
synchronized (SimpleGarbageCollector.this) {
@@ -492,7 +492,7 @@ public class SimpleGarbageCollector implements Iface {
if (tabletDirs.length == 0) {
Path p = new Path(dir + "/" + tableID);
log.debug("Removing table dir " + p);
- if (!moveToTrash(p))
+ if (!archiveOrMoveToTrash(p))
fs.delete(p);
}
}
@@ -606,7 +606,7 @@ public class SimpleGarbageCollector implements Iface {
* @throws IOException
* if the volume manager encountered a problem
*/
- boolean moveToTrash(Path path) throws IOException {
+ boolean archiveOrMoveToTrash(Path path) throws IOException {
if (shouldArchiveFiles()) {
return archiveFile(path);
} else {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ded67f18/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorTest.java
----------------------------------------------------------------------
diff --git a/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorTest.java b/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorTest.java
index d4319b8..672aee2 100644
--- a/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorTest.java
+++ b/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorTest.java
@@ -103,7 +103,7 @@ public class SimpleGarbageCollectorTest {
Path path = createMock(Path.class);
expect(volMgr.moveToTrash(path)).andReturn(true);
replay(volMgr);
- assertTrue(gc.moveToTrash(path));
+ assertTrue(gc.archiveOrMoveToTrash(path));
verify(volMgr);
}
@@ -113,7 +113,7 @@ public class SimpleGarbageCollectorTest {
Path path = createMock(Path.class);
expect(volMgr.moveToTrash(path)).andThrow(new FileNotFoundException());
replay(volMgr);
- assertFalse(gc.moveToTrash(path));
+ assertFalse(gc.archiveOrMoveToTrash(path));
verify(volMgr);
}
@@ -127,7 +127,7 @@ public class SimpleGarbageCollectorTest {
replay(systemConfig);
gc.init(volMgr, instance, credentials, systemConfig);
Path path = createMock(Path.class);
- assertFalse(gc.moveToTrash(path));
+ assertFalse(gc.archiveOrMoveToTrash(path));
}
@Test
[02/12] git commit: ACCUMULO-3263 coverity found a strint-to-byte
conversion using the default character type
Posted by el...@apache.org.
ACCUMULO-3263 coverity found a strint-to-byte conversion using the default character type
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/1890bea2
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/1890bea2
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/1890bea2
Branch: refs/heads/master
Commit: 1890bea201084f9be08cf78c0e31a106f93ffd2d
Parents: 7e0ba16
Author: Eric C. Newton <er...@gmail.com>
Authored: Fri Oct 31 14:34:16 2014 -0400
Committer: Eric C. Newton <er...@gmail.com>
Committed: Fri Oct 31 14:34:16 2014 -0400
----------------------------------------------------------------------
.../java/org/apache/accumulo/server/util/RandomizeVolumes.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/1890bea2/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java b/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
index afc3902..dd540f2 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
@@ -19,8 +19,9 @@ package org.apache.accumulo.server.util;
import java.io.IOException;
import java.util.Map.Entry;
-import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.server.security.SystemCredentials;
import org.apache.accumulo.core.cli.ClientOnRequiredTable;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
@@ -106,7 +107,7 @@ public class RandomizeVolumes {
Mutation m = new Mutation(key.getRow());
String newLocation = vm.choose(ServerConstants.getBaseUris()) + Path.SEPARATOR + ServerConstants.TABLE_DIR + Path.SEPARATOR + tableId + Path.SEPARATOR + directory;
- m.put(key.getColumnFamily(), key.getColumnQualifier(), new Value(newLocation.getBytes()));
+ m.put(key.getColumnFamily(), key.getColumnQualifier(), new Value(newLocation.getBytes(Constants.UTF8)));
if (log.isTraceEnabled()) {
log.trace("Replacing " + oldLocation + " with " + newLocation);
}
[04/12] git commit: ACCUMULO-3213 Archive files that the garbage
collector would otherwise delete.
Posted by el...@apache.org.
ACCUMULO-3213 Archive files that the garbage collector would otherwise delete.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7ce1a649
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7ce1a649
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7ce1a649
Branch: refs/heads/master
Commit: 7ce1a649fe22120833c9c0505c4a1a4c41986eca
Parents: 1890bea
Author: Josh Elser <el...@apache.org>
Authored: Tue Oct 28 21:52:21 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Fri Oct 31 14:59:47 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/core/conf/Property.java | 1 +
.../apache/accumulo/server/ServerConstants.java | 8 ++-
.../accumulo/gc/SimpleGarbageCollector.java | 75 ++++++++++++++++++--
3 files changed, 78 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7ce1a649/core/src/main/java/org/apache/accumulo/core/conf/Property.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index ad83454..56f3d9c 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -289,6 +289,7 @@ public enum Property {
GC_PORT("gc.port.client", "50091", PropertyType.PORT, "The listening port for the garbage collector's monitor service"),
GC_DELETE_THREADS("gc.threads.delete", "16", PropertyType.COUNT, "The number of threads used to delete files"),
GC_TRASH_IGNORE("gc.trash.ignore", "false", PropertyType.BOOLEAN, "Do not use the Trash, even if it is configured"),
+ GC_FILE_ARCHIVE("gc.file.archive", "false", PropertyType.BOOLEAN, "Archive any files/directories instead of moving to the HDFS trash or deleting"),
// properties that are specific to the monitor server behavior
MONITOR_PREFIX("monitor.", null, PropertyType.PREFIX, "Properties in this category affect the behavior of the monitor web server."),
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7ce1a649/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
index 880e2db..d4c0b32 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
@@ -112,6 +112,8 @@ public class ServerConstants {
public static final String TABLE_DIR = "tables";
public static final String RECOVERY_DIR = "recovery";
public static final String WAL_DIR = "wal";
+ public static final String WALOG_ARCHIVE_DIR = "walogArchive";
+ public static final String FILE_ARCHIVE_DIR = "fileArchive";
public static String[] getTablesDirs() {
return VolumeConfiguration.prefix(getBaseUris(), TABLE_DIR);
@@ -126,7 +128,11 @@ public class ServerConstants {
}
public static String[] getWalogArchives() {
- return VolumeConfiguration.prefix(getBaseUris(), "walogArchive");
+ return VolumeConfiguration.prefix(getBaseUris(), WALOG_ARCHIVE_DIR);
+ }
+
+ public static String[] getFileArchives() {
+ return VolumeConfiguration.prefix(getBaseUris(), FILE_ARCHIVE_DIR);
}
public static Path getInstanceIdLocation(Volume v) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7ce1a649/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
----------------------------------------------------------------------
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
index 9b4af58..308d7b9 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
@@ -70,6 +70,7 @@ import org.apache.accumulo.core.util.ServerServices;
import org.apache.accumulo.core.util.ServerServices.Service;
import org.apache.accumulo.core.util.SslConnectionParams;
import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.volume.Volume;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.fate.zookeeper.ZooLock.LockLossReason;
import org.apache.accumulo.fate.zookeeper.ZooLock.LockWatcher;
@@ -225,6 +226,15 @@ public class SimpleGarbageCollector implements Iface {
}
/**
+ * Should files be archived (as opposed to preserved in trash)
+ *
+ * @return True if files should be archived, false otherwise
+ */
+ boolean shouldArchiveFiles() {
+ return config.getBoolean(Property.GC_FILE_ARCHIVE);
+ }
+
+ /**
* Initializes this garbage collector.
*
* @param fs
@@ -597,13 +607,68 @@ public class SimpleGarbageCollector implements Iface {
* if the volume manager encountered a problem
*/
boolean moveToTrash(Path path) throws IOException {
- if (!isUsingTrash())
- return false;
- try {
- return fs.moveToTrash(path);
- } catch (FileNotFoundException ex) {
+ if (shouldArchiveFiles()) {
+ return archiveFile(path);
+ } else {
+ if (!isUsingTrash())
+ return false;
+ try {
+ return fs.moveToTrash(path);
+ } catch (FileNotFoundException ex) {
+ return false;
+ }
+ }
+ }
+
+ /**
+ * Move a file, that would otherwise be deleted, to the archive directory for files
+ *
+ * @param fileToArchive
+ * Path to file that is to be archived
+ * @return True if the file was successfully moved to the file archive directory, false otherwise
+ */
+ boolean archiveFile(Path fileToArchive) throws IOException {
+ // Figure out what the base path this volume uses on this FileSystem
+ Volume sourceVolume = fs.getVolumeByPath(fileToArchive);
+ String sourceVolumeBasePath = sourceVolume.getBasePath();
+
+ log.debug("Base path for volume: " + sourceVolumeBasePath);
+
+ // Get the path for the file we want to archive
+ String sourcePathBasePath = fileToArchive.toUri().getPath();
+
+ // Strip off the common base path for the file to archive
+ String relativeVolumePath = sourcePathBasePath.substring(sourceVolumeBasePath.length());
+ if (Path.SEPARATOR_CHAR == relativeVolumePath.charAt(0)) {
+ if (relativeVolumePath.length() > 1) {
+ relativeVolumePath = relativeVolumePath.substring(1);
+ } else {
+ relativeVolumePath = "";
+ }
+ }
+
+ log.debug("Computed relative path for file to archive: " + relativeVolumePath);
+
+ // The file archive path on this volume (we can't archive this file to a different volume)
+ Path archivePath = new Path(sourceVolumeBasePath, ServerConstants.FILE_ARCHIVE_DIR);
+
+ log.debug("File archive path: " + archivePath);
+
+ fs.mkdirs(archivePath);
+
+ // Preserve the path beneath the Volume's base directory (e.g. tables/1/A_0000001.rf)
+ Path fileArchivePath = new Path(archivePath, relativeVolumePath);
+
+ log.debug("Create full path of " + fileArchivePath + " from " + archivePath + " and " + relativeVolumePath);
+
+ // Make sure that it doesn't already exist, something is wrong.
+ if (fs.exists(fileArchivePath)) {
+ log.warn("Tried to archive file, but it already exists: " + fileArchivePath);
return false;
}
+
+ log.debug("Moving " + fileToArchive + " to " + fileArchivePath);
+ return fs.rename(fileToArchive, fileArchivePath);
}
private void getZooLock(HostAndPort addr) throws KeeperException, InterruptedException {
[09/12] git commit: ACCUMULO-3213 Fix up SimpleGarbageCollectorTest
Posted by el...@apache.org.
ACCUMULO-3213 Fix up SimpleGarbageCollectorTest
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d3900b21
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d3900b21
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d3900b21
Branch: refs/heads/master
Commit: d3900b21c79e1258c6a540fba866efef0ba8cd86
Parents: 43cee41
Author: Josh Elser <el...@apache.org>
Authored: Fri Oct 31 16:03:59 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Fri Oct 31 16:03:59 2014 -0400
----------------------------------------------------------------------
.../accumulo/gc/SimpleGarbageCollectorTest.java | 28 ++++++++++----------
1 file changed, 14 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d3900b21/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorTest.java
----------------------------------------------------------------------
diff --git a/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorTest.java b/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorTest.java
index 672aee2..192a858 100644
--- a/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorTest.java
+++ b/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorTest.java
@@ -16,6 +16,18 @@
*/
package org.apache.accumulo.gc;
+import static org.apache.accumulo.gc.SimpleGarbageCollector.CANDIDATE_MEMORY_PERCENTAGE;
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+
import java.io.FileNotFoundException;
import org.apache.accumulo.core.client.Instance;
@@ -24,9 +36,6 @@ import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.security.Credentials;
import org.apache.accumulo.core.security.thrift.TCredentials;
import org.apache.accumulo.gc.SimpleGarbageCollector.Opts;
-
-import static org.apache.accumulo.gc.SimpleGarbageCollector.CANDIDATE_MEMORY_PERCENTAGE;
-
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.trace.thrift.TInfo;
import org.apache.hadoop.fs.Path;
@@ -34,17 +43,6 @@ import org.easymock.EasyMock;
import org.junit.Before;
import org.junit.Test;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertSame;
-import static org.junit.Assert.assertTrue;
-import static org.easymock.EasyMock.createMock;
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.expectLastCall;
-import static org.easymock.EasyMock.replay;
-import static org.easymock.EasyMock.verify;
-
public class SimpleGarbageCollectorTest {
private VolumeManager volMgr;
private Instance instance;
@@ -76,6 +74,7 @@ public class SimpleGarbageCollectorTest {
expect(systemConfig.getTimeInMillis(Property.GC_CYCLE_DELAY)).andReturn(20000L);
expect(systemConfig.getCount(Property.GC_DELETE_THREADS)).andReturn(2).times(2);
expect(systemConfig.getBoolean(Property.GC_TRASH_IGNORE)).andReturn(false);
+ expect(systemConfig.getBoolean(Property.GC_FILE_ARCHIVE)).andReturn(false);
replay(systemConfig);
return systemConfig;
}
@@ -123,6 +122,7 @@ public class SimpleGarbageCollectorTest {
expect(systemConfig.getTimeInMillis(Property.GC_CYCLE_START)).andReturn(1000L);
expect(systemConfig.getTimeInMillis(Property.GC_CYCLE_DELAY)).andReturn(20000L);
expect(systemConfig.getCount(Property.GC_DELETE_THREADS)).andReturn(2);
+ expect(systemConfig.getBoolean(Property.GC_FILE_ARCHIVE)).andReturn(false);
expect(systemConfig.getBoolean(Property.GC_TRASH_IGNORE)).andReturn(true);
replay(systemConfig);
gc.init(volMgr, instance, credentials, systemConfig);
[06/12] git commit: ACCUMULO-3213 Archive files that the garbage
collector would otherwise delete.
Posted by el...@apache.org.
ACCUMULO-3213 Archive files that the garbage collector would otherwise delete.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7ce1a649
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7ce1a649
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7ce1a649
Branch: refs/heads/1.6
Commit: 7ce1a649fe22120833c9c0505c4a1a4c41986eca
Parents: 1890bea
Author: Josh Elser <el...@apache.org>
Authored: Tue Oct 28 21:52:21 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Fri Oct 31 14:59:47 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/core/conf/Property.java | 1 +
.../apache/accumulo/server/ServerConstants.java | 8 ++-
.../accumulo/gc/SimpleGarbageCollector.java | 75 ++++++++++++++++++--
3 files changed, 78 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7ce1a649/core/src/main/java/org/apache/accumulo/core/conf/Property.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index ad83454..56f3d9c 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -289,6 +289,7 @@ public enum Property {
GC_PORT("gc.port.client", "50091", PropertyType.PORT, "The listening port for the garbage collector's monitor service"),
GC_DELETE_THREADS("gc.threads.delete", "16", PropertyType.COUNT, "The number of threads used to delete files"),
GC_TRASH_IGNORE("gc.trash.ignore", "false", PropertyType.BOOLEAN, "Do not use the Trash, even if it is configured"),
+ GC_FILE_ARCHIVE("gc.file.archive", "false", PropertyType.BOOLEAN, "Archive any files/directories instead of moving to the HDFS trash or deleting"),
// properties that are specific to the monitor server behavior
MONITOR_PREFIX("monitor.", null, PropertyType.PREFIX, "Properties in this category affect the behavior of the monitor web server."),
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7ce1a649/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
index 880e2db..d4c0b32 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
@@ -112,6 +112,8 @@ public class ServerConstants {
public static final String TABLE_DIR = "tables";
public static final String RECOVERY_DIR = "recovery";
public static final String WAL_DIR = "wal";
+ public static final String WALOG_ARCHIVE_DIR = "walogArchive";
+ public static final String FILE_ARCHIVE_DIR = "fileArchive";
public static String[] getTablesDirs() {
return VolumeConfiguration.prefix(getBaseUris(), TABLE_DIR);
@@ -126,7 +128,11 @@ public class ServerConstants {
}
public static String[] getWalogArchives() {
- return VolumeConfiguration.prefix(getBaseUris(), "walogArchive");
+ return VolumeConfiguration.prefix(getBaseUris(), WALOG_ARCHIVE_DIR);
+ }
+
+ public static String[] getFileArchives() {
+ return VolumeConfiguration.prefix(getBaseUris(), FILE_ARCHIVE_DIR);
}
public static Path getInstanceIdLocation(Volume v) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7ce1a649/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
----------------------------------------------------------------------
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
index 9b4af58..308d7b9 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
@@ -70,6 +70,7 @@ import org.apache.accumulo.core.util.ServerServices;
import org.apache.accumulo.core.util.ServerServices.Service;
import org.apache.accumulo.core.util.SslConnectionParams;
import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.volume.Volume;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.fate.zookeeper.ZooLock.LockLossReason;
import org.apache.accumulo.fate.zookeeper.ZooLock.LockWatcher;
@@ -225,6 +226,15 @@ public class SimpleGarbageCollector implements Iface {
}
/**
+ * Should files be archived (as opposed to preserved in trash)
+ *
+ * @return True if files should be archived, false otherwise
+ */
+ boolean shouldArchiveFiles() {
+ return config.getBoolean(Property.GC_FILE_ARCHIVE);
+ }
+
+ /**
* Initializes this garbage collector.
*
* @param fs
@@ -597,13 +607,68 @@ public class SimpleGarbageCollector implements Iface {
* if the volume manager encountered a problem
*/
boolean moveToTrash(Path path) throws IOException {
- if (!isUsingTrash())
- return false;
- try {
- return fs.moveToTrash(path);
- } catch (FileNotFoundException ex) {
+ if (shouldArchiveFiles()) {
+ return archiveFile(path);
+ } else {
+ if (!isUsingTrash())
+ return false;
+ try {
+ return fs.moveToTrash(path);
+ } catch (FileNotFoundException ex) {
+ return false;
+ }
+ }
+ }
+
+ /**
+ * Move a file, that would otherwise be deleted, to the archive directory for files
+ *
+ * @param fileToArchive
+ * Path to file that is to be archived
+ * @return True if the file was successfully moved to the file archive directory, false otherwise
+ */
+ boolean archiveFile(Path fileToArchive) throws IOException {
+ // Figure out what the base path this volume uses on this FileSystem
+ Volume sourceVolume = fs.getVolumeByPath(fileToArchive);
+ String sourceVolumeBasePath = sourceVolume.getBasePath();
+
+ log.debug("Base path for volume: " + sourceVolumeBasePath);
+
+ // Get the path for the file we want to archive
+ String sourcePathBasePath = fileToArchive.toUri().getPath();
+
+ // Strip off the common base path for the file to archive
+ String relativeVolumePath = sourcePathBasePath.substring(sourceVolumeBasePath.length());
+ if (Path.SEPARATOR_CHAR == relativeVolumePath.charAt(0)) {
+ if (relativeVolumePath.length() > 1) {
+ relativeVolumePath = relativeVolumePath.substring(1);
+ } else {
+ relativeVolumePath = "";
+ }
+ }
+
+ log.debug("Computed relative path for file to archive: " + relativeVolumePath);
+
+ // The file archive path on this volume (we can't archive this file to a different volume)
+ Path archivePath = new Path(sourceVolumeBasePath, ServerConstants.FILE_ARCHIVE_DIR);
+
+ log.debug("File archive path: " + archivePath);
+
+ fs.mkdirs(archivePath);
+
+ // Preserve the path beneath the Volume's base directory (e.g. tables/1/A_0000001.rf)
+ Path fileArchivePath = new Path(archivePath, relativeVolumePath);
+
+ log.debug("Create full path of " + fileArchivePath + " from " + archivePath + " and " + relativeVolumePath);
+
+ // Make sure that it doesn't already exist, something is wrong.
+ if (fs.exists(fileArchivePath)) {
+ log.warn("Tried to archive file, but it already exists: " + fileArchivePath);
return false;
}
+
+ log.debug("Moving " + fileToArchive + " to " + fileArchivePath);
+ return fs.rename(fileToArchive, fileArchivePath);
}
private void getZooLock(HostAndPort addr) throws KeeperException, InterruptedException {