You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2014/09/30 19:42:04 UTC
[1/6] git commit: ACCUMULO-3085 Fix log message
Repository: accumulo
Updated Branches:
refs/heads/1.5 5459950dd -> 5f2d3473e
refs/heads/1.6 6f029b0e9 -> 43850055c
refs/heads/master 377105a05 -> c2bdaf292
ACCUMULO-3085 Fix log message
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/5f2d3473
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/5f2d3473
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/5f2d3473
Branch: refs/heads/1.5
Commit: 5f2d3473eb3f2e7c77d13fd5803a1ac9e2350f06
Parents: 5459950
Author: Josh Elser <el...@apache.org>
Authored: Tue Sep 30 13:27:40 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Tue Sep 30 13:27:40 2014 -0400
----------------------------------------------------------------------
.../main/java/org/apache/accumulo/server/tabletserver/Tablet.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/5f2d3473/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java b/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
index 9bbae89..a7a9d4e 100644
--- a/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
+++ b/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
@@ -1572,7 +1572,7 @@ public class Tablet {
}
if (!filename.startsWith(Constants.MAPFILE_EXTENSION + "_") && !FileOperations.getValidExtensions().contains(filename.split("\\.")[1])) {
- log.error("unknown file in tablet" + path);
+ log.error("unknown file in tablet: " + path);
continue;
}
[4/6] git commit: Merge branch '1.5' into 1.6
Posted by el...@apache.org.
Merge branch '1.5' into 1.6
Conflicts:
server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/43850055
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/43850055
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/43850055
Branch: refs/heads/master
Commit: 43850055cf5f89e61214e301dc302c00837a77f9
Parents: 6f029b0 5f2d347
Author: Josh Elser <el...@apache.org>
Authored: Tue Sep 30 13:41:18 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Tue Sep 30 13:41:18 2014 -0400
----------------------------------------------------------------------
.../src/main/java/org/apache/accumulo/tserver/RootFiles.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/43850055/server/tserver/src/main/java/org/apache/accumulo/tserver/RootFiles.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/RootFiles.java
index f23c55d,0000000..0c043ed
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/RootFiles.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/RootFiles.java
@@@ -1,133 -1,0 +1,133 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Set;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.server.fs.FileRef;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+
+/**
+ *
+ */
+public class RootFiles {
+
+ private static Logger log = Logger.getLogger(RootFiles.class);
+
+ static void prepareReplacement(VolumeManager fs, Path location, Set<FileRef> oldDatafiles, String compactName) throws IOException {
+ for (FileRef ref : oldDatafiles) {
+ Path path = ref.path();
+ Tablet.rename(fs, path, new Path(location + "/delete+" + compactName + "+" + path.getName()));
+ }
+ }
+
+ static void renameReplacement(VolumeManager fs, FileRef tmpDatafile, FileRef newDatafile) throws IOException {
+ if (fs.exists(newDatafile.path())) {
+ log.error("Target map file already exist " + newDatafile, new Exception());
+ throw new IllegalStateException("Target map file already exist " + newDatafile);
+ }
+
+ Tablet.rename(fs, tmpDatafile.path(), newDatafile.path());
+ }
+
+ static void finishReplacement(AccumuloConfiguration acuTableConf, VolumeManager fs, Path location, Set<FileRef> oldDatafiles, String compactName)
+ throws IOException {
+ // start deleting files, if we do not finish they will be cleaned
+ // up later
+ for (FileRef ref : oldDatafiles) {
+ Path path = ref.path();
+ Path deleteFile = new Path(location + "/delete+" + compactName + "+" + path.getName());
+ if (acuTableConf.getBoolean(Property.GC_TRASH_IGNORE) || !fs.moveToTrash(deleteFile))
+ fs.deleteRecursively(deleteFile);
+ }
+ }
+
+ public static void replaceFiles(AccumuloConfiguration acuTableConf, VolumeManager fs, Path location, Set<FileRef> oldDatafiles, FileRef tmpDatafile,
+ FileRef newDatafile) throws IOException {
+ String compactName = newDatafile.path().getName();
+
+ prepareReplacement(fs, location, oldDatafiles, compactName);
+ renameReplacement(fs, tmpDatafile, newDatafile);
+ finishReplacement(acuTableConf, fs, location, oldDatafiles, compactName);
+ }
+
+ public static Collection<String> cleanupReplacement(VolumeManager fs, FileStatus[] files, boolean deleteTmp) throws IOException {
+ /*
+ * called in constructor and before major compactions
+ */
+ Collection<String> goodFiles = new ArrayList<String>(files.length);
+
+ for (FileStatus file : files) {
+
+ String path = file.getPath().toString();
+ if (file.getPath().toUri().getScheme() == null) {
+ // depending on the behavior of HDFS, if list status does not return fully qualified volumes then could switch to the default volume
+ throw new IllegalArgumentException("Require fully qualified paths " + file.getPath());
+ }
+
+ String filename = file.getPath().getName();
+
+ // check for incomplete major compaction, this should only occur
+ // for root tablet
+ if (filename.startsWith("delete+")) {
+ String expectedCompactedFile = path.substring(0, path.lastIndexOf("/delete+")) + "/" + filename.split("\\+")[1];
+ if (fs.exists(new Path(expectedCompactedFile))) {
+ // compaction finished, but did not finish deleting compacted files.. so delete it
+ if (!fs.deleteRecursively(file.getPath()))
+ log.warn("Delete of file: " + file.getPath().toString() + " return false");
+ continue;
+ }
+ // compaction did not finish, so put files back
+
+ // reset path and filename for rest of loop
+ filename = filename.split("\\+", 3)[2];
+ path = path.substring(0, path.lastIndexOf("/delete+")) + "/" + filename;
+
+ Tablet.rename(fs, file.getPath(), new Path(path));
+ }
+
+ if (filename.endsWith("_tmp")) {
+ if (deleteTmp) {
+ log.warn("cleaning up old tmp file: " + path);
+ if (!fs.deleteRecursively(file.getPath()))
+ log.warn("Delete of tmp file: " + file.getPath().toString() + " return false");
+
+ }
+ continue;
+ }
+
+ if (!filename.startsWith(Constants.MAPFILE_EXTENSION + "_") && !FileOperations.getValidExtensions().contains(filename.split("\\.")[1])) {
- log.error("unknown file in tablet" + path);
++ log.error("unknown file in tablet: " + path);
+ continue;
+ }
+
+ goodFiles.add(path);
+ }
+
+ return goodFiles;
+ }
+}
[2/6] git commit: ACCUMULO-3085 Fix log message
Posted by el...@apache.org.
ACCUMULO-3085 Fix log message
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/5f2d3473
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/5f2d3473
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/5f2d3473
Branch: refs/heads/1.6
Commit: 5f2d3473eb3f2e7c77d13fd5803a1ac9e2350f06
Parents: 5459950
Author: Josh Elser <el...@apache.org>
Authored: Tue Sep 30 13:27:40 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Tue Sep 30 13:27:40 2014 -0400
----------------------------------------------------------------------
.../main/java/org/apache/accumulo/server/tabletserver/Tablet.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/5f2d3473/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java b/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
index 9bbae89..a7a9d4e 100644
--- a/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
+++ b/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
@@ -1572,7 +1572,7 @@ public class Tablet {
}
if (!filename.startsWith(Constants.MAPFILE_EXTENSION + "_") && !FileOperations.getValidExtensions().contains(filename.split("\\.")[1])) {
- log.error("unknown file in tablet" + path);
+ log.error("unknown file in tablet: " + path);
continue;
}
[3/6] git commit: ACCUMULO-3085 Fix log message
Posted by el...@apache.org.
ACCUMULO-3085 Fix log message
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/5f2d3473
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/5f2d3473
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/5f2d3473
Branch: refs/heads/master
Commit: 5f2d3473eb3f2e7c77d13fd5803a1ac9e2350f06
Parents: 5459950
Author: Josh Elser <el...@apache.org>
Authored: Tue Sep 30 13:27:40 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Tue Sep 30 13:27:40 2014 -0400
----------------------------------------------------------------------
.../main/java/org/apache/accumulo/server/tabletserver/Tablet.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/5f2d3473/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java b/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
index 9bbae89..a7a9d4e 100644
--- a/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
+++ b/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
@@ -1572,7 +1572,7 @@ public class Tablet {
}
if (!filename.startsWith(Constants.MAPFILE_EXTENSION + "_") && !FileOperations.getValidExtensions().contains(filename.split("\\.")[1])) {
- log.error("unknown file in tablet" + path);
+ log.error("unknown file in tablet: " + path);
continue;
}
[6/6] git commit: Merge branch '1.6'
Posted by el...@apache.org.
Merge branch '1.6'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c2bdaf29
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c2bdaf29
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c2bdaf29
Branch: refs/heads/master
Commit: c2bdaf292461a79e13c649751cf4e06e396615bc
Parents: 377105a 4385005
Author: Josh Elser <el...@apache.org>
Authored: Tue Sep 30 13:41:24 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Tue Sep 30 13:41:24 2014 -0400
----------------------------------------------------------------------
.../main/java/org/apache/accumulo/tserver/tablet/RootFiles.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c2bdaf29/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/RootFiles.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/RootFiles.java
index efdfab4,0000000..875643e
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/RootFiles.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/RootFiles.java
@@@ -1,133 -1,0 +1,133 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver.tablet;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Set;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.server.fs.FileRef;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+
+/**
+ *
+ */
+public class RootFiles {
+
+ private static final Logger log = Logger.getLogger(RootFiles.class);
+
+ public static void prepareReplacement(VolumeManager fs, Path location, Set<FileRef> oldDatafiles, String compactName) throws IOException {
+ for (FileRef ref : oldDatafiles) {
+ Path path = ref.path();
+ DatafileManager.rename(fs, path, new Path(location + "/delete+" + compactName + "+" + path.getName()));
+ }
+ }
+
+ public static void renameReplacement(VolumeManager fs, FileRef tmpDatafile, FileRef newDatafile) throws IOException {
+ if (fs.exists(newDatafile.path())) {
+ log.error("Target map file already exist " + newDatafile, new Exception());
+ throw new IllegalStateException("Target map file already exist " + newDatafile);
+ }
+
+ DatafileManager.rename(fs, tmpDatafile.path(), newDatafile.path());
+ }
+
+ public static void finishReplacement(AccumuloConfiguration acuTableConf, VolumeManager fs, Path location, Set<FileRef> oldDatafiles, String compactName)
+ throws IOException {
+ // start deleting files, if we do not finish they will be cleaned
+ // up later
+ for (FileRef ref : oldDatafiles) {
+ Path path = ref.path();
+ Path deleteFile = new Path(location + "/delete+" + compactName + "+" + path.getName());
+ if (acuTableConf.getBoolean(Property.GC_TRASH_IGNORE) || !fs.moveToTrash(deleteFile))
+ fs.deleteRecursively(deleteFile);
+ }
+ }
+
+ public static void replaceFiles(AccumuloConfiguration acuTableConf, VolumeManager fs, Path location, Set<FileRef> oldDatafiles, FileRef tmpDatafile,
+ FileRef newDatafile) throws IOException {
+ String compactName = newDatafile.path().getName();
+
+ prepareReplacement(fs, location, oldDatafiles, compactName);
+ renameReplacement(fs, tmpDatafile, newDatafile);
+ finishReplacement(acuTableConf, fs, location, oldDatafiles, compactName);
+ }
+
+ public static Collection<String> cleanupReplacement(VolumeManager fs, FileStatus[] files, boolean deleteTmp) throws IOException {
+ /*
+ * called in constructor and before major compactions
+ */
+ Collection<String> goodFiles = new ArrayList<String>(files.length);
+
+ for (FileStatus file : files) {
+
+ String path = file.getPath().toString();
+ if (file.getPath().toUri().getScheme() == null) {
+ // depending on the behavior of HDFS, if list status does not return fully qualified volumes then could switch to the default volume
+ throw new IllegalArgumentException("Require fully qualified paths " + file.getPath());
+ }
+
+ String filename = file.getPath().getName();
+
+ // check for incomplete major compaction, this should only occur
+ // for root tablet
+ if (filename.startsWith("delete+")) {
+ String expectedCompactedFile = path.substring(0, path.lastIndexOf("/delete+")) + "/" + filename.split("\\+")[1];
+ if (fs.exists(new Path(expectedCompactedFile))) {
+ // compaction finished, but did not finish deleting compacted files.. so delete it
+ if (!fs.deleteRecursively(file.getPath()))
+ log.warn("Delete of file: " + file.getPath().toString() + " return false");
+ continue;
+ }
+ // compaction did not finish, so put files back
+
+ // reset path and filename for rest of loop
+ filename = filename.split("\\+", 3)[2];
+ path = path.substring(0, path.lastIndexOf("/delete+")) + "/" + filename;
+
+ DatafileManager.rename(fs, file.getPath(), new Path(path));
+ }
+
+ if (filename.endsWith("_tmp")) {
+ if (deleteTmp) {
+ log.warn("cleaning up old tmp file: " + path);
+ if (!fs.deleteRecursively(file.getPath()))
+ log.warn("Delete of tmp file: " + file.getPath().toString() + " return false");
+
+ }
+ continue;
+ }
+
+ if (!filename.startsWith(Constants.MAPFILE_EXTENSION + "_") && !FileOperations.getValidExtensions().contains(filename.split("\\.")[1])) {
- log.error("unknown file in tablet" + path);
++ log.error("unknown file in tablet: " + path);
+ continue;
+ }
+
+ goodFiles.add(path);
+ }
+
+ return goodFiles;
+ }
+}
[5/6] git commit: Merge branch '1.5' into 1.6
Posted by el...@apache.org.
Merge branch '1.5' into 1.6
Conflicts:
server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/43850055
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/43850055
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/43850055
Branch: refs/heads/1.6
Commit: 43850055cf5f89e61214e301dc302c00837a77f9
Parents: 6f029b0 5f2d347
Author: Josh Elser <el...@apache.org>
Authored: Tue Sep 30 13:41:18 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Tue Sep 30 13:41:18 2014 -0400
----------------------------------------------------------------------
.../src/main/java/org/apache/accumulo/tserver/RootFiles.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/43850055/server/tserver/src/main/java/org/apache/accumulo/tserver/RootFiles.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/RootFiles.java
index f23c55d,0000000..0c043ed
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/RootFiles.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/RootFiles.java
@@@ -1,133 -1,0 +1,133 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Set;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.server.fs.FileRef;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+
+/**
+ *
+ */
+public class RootFiles {
+
+ private static Logger log = Logger.getLogger(RootFiles.class);
+
+ static void prepareReplacement(VolumeManager fs, Path location, Set<FileRef> oldDatafiles, String compactName) throws IOException {
+ for (FileRef ref : oldDatafiles) {
+ Path path = ref.path();
+ Tablet.rename(fs, path, new Path(location + "/delete+" + compactName + "+" + path.getName()));
+ }
+ }
+
+ static void renameReplacement(VolumeManager fs, FileRef tmpDatafile, FileRef newDatafile) throws IOException {
+ if (fs.exists(newDatafile.path())) {
+ log.error("Target map file already exist " + newDatafile, new Exception());
+ throw new IllegalStateException("Target map file already exist " + newDatafile);
+ }
+
+ Tablet.rename(fs, tmpDatafile.path(), newDatafile.path());
+ }
+
+ static void finishReplacement(AccumuloConfiguration acuTableConf, VolumeManager fs, Path location, Set<FileRef> oldDatafiles, String compactName)
+ throws IOException {
+ // start deleting files, if we do not finish they will be cleaned
+ // up later
+ for (FileRef ref : oldDatafiles) {
+ Path path = ref.path();
+ Path deleteFile = new Path(location + "/delete+" + compactName + "+" + path.getName());
+ if (acuTableConf.getBoolean(Property.GC_TRASH_IGNORE) || !fs.moveToTrash(deleteFile))
+ fs.deleteRecursively(deleteFile);
+ }
+ }
+
+ public static void replaceFiles(AccumuloConfiguration acuTableConf, VolumeManager fs, Path location, Set<FileRef> oldDatafiles, FileRef tmpDatafile,
+ FileRef newDatafile) throws IOException {
+ String compactName = newDatafile.path().getName();
+
+ prepareReplacement(fs, location, oldDatafiles, compactName);
+ renameReplacement(fs, tmpDatafile, newDatafile);
+ finishReplacement(acuTableConf, fs, location, oldDatafiles, compactName);
+ }
+
+ public static Collection<String> cleanupReplacement(VolumeManager fs, FileStatus[] files, boolean deleteTmp) throws IOException {
+ /*
+ * called in constructor and before major compactions
+ */
+ Collection<String> goodFiles = new ArrayList<String>(files.length);
+
+ for (FileStatus file : files) {
+
+ String path = file.getPath().toString();
+ if (file.getPath().toUri().getScheme() == null) {
+ // depending on the behavior of HDFS, if list status does not return fully qualified volumes then could switch to the default volume
+ throw new IllegalArgumentException("Require fully qualified paths " + file.getPath());
+ }
+
+ String filename = file.getPath().getName();
+
+ // check for incomplete major compaction, this should only occur
+ // for root tablet
+ if (filename.startsWith("delete+")) {
+ String expectedCompactedFile = path.substring(0, path.lastIndexOf("/delete+")) + "/" + filename.split("\\+")[1];
+ if (fs.exists(new Path(expectedCompactedFile))) {
+ // compaction finished, but did not finish deleting compacted files.. so delete it
+ if (!fs.deleteRecursively(file.getPath()))
+ log.warn("Delete of file: " + file.getPath().toString() + " return false");
+ continue;
+ }
+ // compaction did not finish, so put files back
+
+ // reset path and filename for rest of loop
+ filename = filename.split("\\+", 3)[2];
+ path = path.substring(0, path.lastIndexOf("/delete+")) + "/" + filename;
+
+ Tablet.rename(fs, file.getPath(), new Path(path));
+ }
+
+ if (filename.endsWith("_tmp")) {
+ if (deleteTmp) {
+ log.warn("cleaning up old tmp file: " + path);
+ if (!fs.deleteRecursively(file.getPath()))
+ log.warn("Delete of tmp file: " + file.getPath().toString() + " return false");
+
+ }
+ continue;
+ }
+
+ if (!filename.startsWith(Constants.MAPFILE_EXTENSION + "_") && !FileOperations.getValidExtensions().contains(filename.split("\\.")[1])) {
- log.error("unknown file in tablet" + path);
++ log.error("unknown file in tablet: " + path);
+ continue;
+ }
+
+ goodFiles.add(path);
+ }
+
+ return goodFiles;
+ }
+}