You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2013/09/13 18:26:21 UTC
[5/5] git commit: ACCUMULO-1452 merged branches to master, fixed test
ACCUMULO-1452 merged branches to master, fixed test
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/eb5e5a48
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/eb5e5a48
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/eb5e5a48
Branch: refs/heads/master
Commit: eb5e5a4890c1346bf59be3327b442be76d734a1b
Parents: 886e56a
Author: Eric Newton <er...@gmail.com>
Authored: Fri Sep 13 12:26:24 2013 -0400
Committer: Eric Newton <er...@gmail.com>
Committed: Fri Sep 13 12:26:24 2013 -0400
----------------------------------------------------------------------
.../minicluster/MiniAccumuloCluster.java | 19 ++++++--
.../minicluster/MiniAccumuloConfig.java | 2 +-
.../accumulo/server/tabletserver/Tablet.java | 8 +---
.../accumulo/test/functional/CleanTmpIT.java | 50 +++++++++++++++-----
4 files changed, 57 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/eb5e5a48/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
----------------------------------------------------------------------
diff --git a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
index 473fe44..2e061dd 100644
--- a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
+++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
@@ -24,6 +24,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.InetSocketAddress;
+import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -58,6 +59,7 @@ import org.apache.accumulo.start.Main;
import org.apache.accumulo.start.classloader.vfs.MiniDFSUtil;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.zookeeper.server.ZooKeeperServerMain;
@@ -124,6 +126,7 @@ public class MiniAccumuloCluster {
private Set<Pair<ServerType,Integer>> debugPorts = new HashSet<Pair<ServerType,Integer>>();
private File zooCfgFile;
+ private String dfsUri;
public List<LogWriter> getLogWriters() {
return logWriters;
@@ -245,16 +248,18 @@ public class MiniAccumuloCluster {
System.setProperty("test.build.data", oldTestBuildData);
miniDFS.waitClusterUp();
InetSocketAddress dfsAddress = miniDFS.getNameNode().getNameNodeAddress();
- String uri = "hdfs://" + dfsAddress.getHostName() + ":" + dfsAddress.getPort();
+ dfsUri = "hdfs://" + dfsAddress.getHostName() + ":" + dfsAddress.getPort();
File coreFile = new File(config.getConfDir(), "core-site.xml");
- writeConfig(coreFile, Collections.singletonMap("fs.default.name", uri).entrySet());
+ writeConfig(coreFile, Collections.singletonMap("fs.default.name", dfsUri).entrySet());
File hdfsFile = new File(config.getConfDir(), "hdfs-site.xml");
writeConfig(hdfsFile, conf);
Map<String,String> siteConfig = config.getSiteConfig();
- siteConfig.put(Property.INSTANCE_DFS_URI.getKey(), uri);
+ siteConfig.put(Property.INSTANCE_DFS_URI.getKey(), dfsUri);
siteConfig.put(Property.INSTANCE_DFS_DIR.getKey(), "/accumulo");
config.setSiteConfig(siteConfig);
+ } else {
+ dfsUri = "file://";
}
File siteFile = new File(config.getConfDir(), "accumulo-site.xml");
@@ -501,4 +506,12 @@ public class MiniAccumuloCluster {
Instance instance = new ZooKeeperInstance(this.getInstanceName(), this.getZooKeepers());
return instance.getConnector(user, new PasswordToken(passwd));
}
+
+ public FileSystem getFileSystem() {
+ try {
+ return FileSystem.get(new URI(dfsUri), new Configuration());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/eb5e5a48/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
----------------------------------------------------------------------
diff --git a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
index f8ad3af..0b6c42c 100644
--- a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
+++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
@@ -265,7 +265,7 @@ public class MiniAccumuloConfig {
return zooKeeperDir;
}
- File getAccumuloDir() {
+ public File getAccumuloDir() {
return accumuloDir;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/eb5e5a48/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java b/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
index d33b26c..4e06c1d 100644
--- a/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
+++ b/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
@@ -1347,7 +1347,6 @@ public class Tablet {
configObserver.propertiesChanged();
tabletResources.setTablet(this, acuTableConf);
-
if (!logEntries.isEmpty()) {
log.info("Starting Write-Ahead Log recovery for " + this.extent);
final long[] count = new long[2];
@@ -1428,12 +1427,6 @@ public class Tablet {
// look for any temp files hanging around
removeOldTemporaryFiles();
}
-
- // look for hints of a failure on the previous tablet server
- if (!logEntries.isEmpty() || needsMajorCompaction(MajorCompactionReason.NORMAL)) {
- // look for any temp files hanging around
- removeOldTemporaryFiles();
- }
log.log(TLevel.TABLET_HIST, extent + " opened ");
}
@@ -1443,6 +1436,7 @@ public class Tablet {
try {
for (FileStatus tmp : fs.globStatus(new Path(location, "*_tmp"))){
try {
+ log.debug("Removing old temp file " + tmp.getPath());
fs.delete(tmp.getPath());
} catch (IOException ex) {
log.error("Unable to remove old temp file " + tmp.getPath() + ": " + ex);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/eb5e5a48/test/src/test/java/org/apache/accumulo/test/functional/CleanTmpIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/CleanTmpIT.java b/test/src/test/java/org/apache/accumulo/test/functional/CleanTmpIT.java
index 93b21f1..c4fb6df 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/CleanTmpIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/CleanTmpIT.java
@@ -16,20 +16,43 @@
*/
package org.apache.accumulo.test.functional;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import java.io.File;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.BatchWriterConfig;
import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.minicluster.MiniAccumuloConfig;
+import org.apache.accumulo.minicluster.ProcessReference;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.junit.Test;
-public class CleanTmpIT extends SimpleMacIT {
+public class CleanTmpIT extends ConfigurableMacIT {
- @Test(timeout = 2 * 60 * 1000)
+ @Override
+ public void configure(MiniAccumuloConfig cfg) {
+ Map<String, String> props = new HashMap<String, String>();
+ props.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "3s");
+ cfg.setSiteConfig(props);
+ cfg.setNumTservers(1);
+ cfg.useMiniDFS(true);
+ }
+
+ @Test(timeout = 4 * 60 * 1000)
public void test() throws Exception {
Connector c = getConnector();
// make a table
@@ -41,15 +64,20 @@ public class CleanTmpIT extends SimpleMacIT {
m.put("cf", "cq", "value");
bw.addMutation(m);
bw.close();
- // take it offline
- c.tableOperations().offline(tableName);
// create a fake _tmp file in its directory
String id = c.tableOperations().tableIdMap().get(tableName);
- File tmp = File.createTempFile("junk", "_tmp", new File(getCluster().getConfig().getDir(), "accumulo/tables/" + id + "/default_tablet"));
- assertTrue(tmp.exists());
- c.tableOperations().online(tableName);
- UtilWaitThread.sleep(1000);
- assertFalse(tmp.exists());
+ FileSystem fs = getCluster().getFileSystem();
+ Path tmp = new Path(getCluster().getConfig().getAccumuloDir().getPath() + "/tables/" + id + "/default_tablet/junk.rf_tmp");
+ fs.create(tmp).close();
+ for (ProcessReference tserver: getCluster().getProcesses().get(ServerType.TABLET_SERVER)) {
+ getCluster().killProcess(ServerType.TABLET_SERVER, tserver);
+ }
+ getCluster().start();
+
+ Scanner scanner = c.createScanner(tableName, Authorizations.EMPTY);
+ for (@SuppressWarnings("unused") Entry<Key,Value> entry : scanner)
+ ;
+ assertFalse(!fs.exists(tmp));
}
}