You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@crunch.apache.org by gr...@apache.org on 2015/05/23 21:34:33 UTC
crunch git commit: CRUNCH-524 Correctly delete temp files
Repository: crunch
Updated Branches:
refs/heads/master 9bb526fd1 -> 6c8c5ffd0
CRUNCH-524 Correctly delete temp files
Delete /tmp/crunch- on exit; delete ~/hbase after test finishes; move H2 db
file to /tmp
Signed-off-by: Gabriel Reid <gr...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/6c8c5ffd
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/6c8c5ffd
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/6c8c5ffd
Branch: refs/heads/master
Commit: 6c8c5ffd0447da82e63cf0b74ed2026c77a0c554
Parents: 9bb526f
Author: Sean Owen <so...@cloudera.com>
Authored: Thu May 21 22:34:16 2015 +0100
Committer: Gabriel Reid <gr...@apache.org>
Committed: Sat May 23 21:34:04 2015 +0200
----------------------------------------------------------------------
.../crunch/contrib/io/jdbc/DataBaseSourceIT.java | 5 +++--
.../crunch/impl/dist/DistributedPipeline.java | 4 +++-
.../apache/crunch/io/hbase/WordCountHBaseIT.java | 18 ++++++++++++++++++
3 files changed, 24 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/crunch/blob/6c8c5ffd/crunch-contrib/src/it/java/org/apache/crunch/contrib/io/jdbc/DataBaseSourceIT.java
----------------------------------------------------------------------
diff --git a/crunch-contrib/src/it/java/org/apache/crunch/contrib/io/jdbc/DataBaseSourceIT.java b/crunch-contrib/src/it/java/org/apache/crunch/contrib/io/jdbc/DataBaseSourceIT.java
index 1c48559..a19803e 100644
--- a/crunch-contrib/src/it/java/org/apache/crunch/contrib/io/jdbc/DataBaseSourceIT.java
+++ b/crunch-contrib/src/it/java/org/apache/crunch/contrib/io/jdbc/DataBaseSourceIT.java
@@ -46,12 +46,13 @@ public class DataBaseSourceIT extends CrunchTestSupport implements Serializable
public void start() throws Exception {
File file = tempDir.copyResourceFile("data.script");
server = Server.createTcpServer().start();
- RunScript.execute("jdbc:h2:tcp://localhost/~/test", "sa", "", file.getAbsolutePath(), "utf-8", false);
+ RunScript.execute("jdbc:h2:file:/tmp/test", "sa", "", file.getAbsolutePath(), "utf-8", false);
}
@After
public void stop() throws Exception {
server.stop();
+ new File("/tmp/test.h2.db").delete();
}
@Test
@@ -59,7 +60,7 @@ public class DataBaseSourceIT extends CrunchTestSupport implements Serializable
Pipeline pipeline = new MRPipeline(DataBaseSourceIT.class);
DataBaseSource<IdentifiableName> dbsrc = new DataBaseSource.Builder<IdentifiableName>(IdentifiableName.class)
.setDriverClass(org.h2.Driver.class)
- .setUrl("jdbc:h2:tcp://localhost/~/test").setUsername("sa").setPassword("")
+ .setUrl("jdbc:h2:file:/tmp/test").setUsername("sa").setPassword("")
.selectSQLQuery("SELECT ID, NAME FROM TEST").countSQLQuery("select count(*) from Test").build();
PCollection<IdentifiableName> cdidata = pipeline.read(dbsrc);
http://git-wip-us.apache.org/repos/asf/crunch/blob/6c8c5ffd/crunch-core/src/main/java/org/apache/crunch/impl/dist/DistributedPipeline.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/impl/dist/DistributedPipeline.java b/crunch-core/src/main/java/org/apache/crunch/impl/dist/DistributedPipeline.java
index 88da5a6..049046a 100644
--- a/crunch-core/src/main/java/org/apache/crunch/impl/dist/DistributedPipeline.java
+++ b/crunch-core/src/main/java/org/apache/crunch/impl/dist/DistributedPipeline.java
@@ -396,7 +396,9 @@ public abstract class DistributedPipeline implements Pipeline {
private static Path createTempDirectory(Configuration conf) {
Path dir = createTemporaryPath(conf);
try {
- dir.getFileSystem(conf).mkdirs(dir);
+ FileSystem fs = dir.getFileSystem(conf);
+ fs.mkdirs(dir);
+ fs.deleteOnExit(dir);
} catch (IOException e) {
throw new RuntimeException("Cannot create job output directory " + dir, e);
}
http://git-wip-us.apache.org/repos/asf/crunch/blob/6c8c5ffd/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/WordCountHBaseIT.java
----------------------------------------------------------------------
diff --git a/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/WordCountHBaseIT.java b/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/WordCountHBaseIT.java
index dd48352..ead2aea 100644
--- a/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/WordCountHBaseIT.java
+++ b/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/WordCountHBaseIT.java
@@ -19,6 +19,7 @@ package org.apache.crunch.io.hbase;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
@@ -39,6 +40,8 @@ import org.apache.crunch.test.TemporaryPath;
import org.apache.crunch.test.TemporaryPaths;
import org.apache.crunch.types.writable.Writables;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.client.Delete;
@@ -119,6 +122,18 @@ public class WordCountHBaseIT {
@Before
public void setUp() throws Exception {
Configuration conf = HBaseConfiguration.create(tmpDir.getDefaultConfiguration());
+
+ // HBase 0.98 HBaseTestingUtility hard-codes ~/hbase as its temp directory and leaves
+ // it lying around after this test. It's easy to delete, but to be sure it's safe to
+ // delete, we need to be sure that it doesn't exist already. The test probably won't
+ // work anyway if it does. If it doesn't exist, then deleting it later doesn't matter
+ // even if a future HBase is used (which doesn't have this problem) and this test
+ // isn't updated.
+ FileSystem fs = FileSystem.get(conf);
+ Path hbaseTestPath = new Path(fs.makeQualified(fs.getHomeDirectory()),"hbase");
+ assertFalse("The hbase directory in your home directory already exists. This test deletes this directory " +
+ "on exit, so it will not run unless you delete the directory yourself", fs.exists(hbaseTestPath));
+
hbaseTestUtil = new HBaseTestingUtility(conf);
hbaseTestUtil.startMiniZKCluster();
hbaseTestUtil.startMiniHBaseCluster(1, 1);
@@ -133,6 +148,9 @@ public class WordCountHBaseIT {
public void tearDown() throws Exception {
hbaseTestUtil.shutdownMiniHBaseCluster();
hbaseTestUtil.shutdownMiniZKCluster();
+ FileSystem fs = FileSystem.get(hbaseTestUtil.getConfiguration());
+ Path hbaseTestPath = new Path(fs.makeQualified(fs.getHomeDirectory()),"hbase");
+ fs.delete(hbaseTestPath, true);
}
public void run(Pipeline pipeline) throws Exception {