You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2018/02/26 15:31:52 UTC
carbondata git commit: [CARBONDATA-1114][Tests] Fix bugs in tests in
windows env
Repository: carbondata
Updated Branches:
refs/heads/carbonstore ab9b4cf89 -> 37080e9a3
[CARBONDATA-1114][Tests] Fix bugs in tests in windows env
Fix bugs in tests that will cause failure under windows env
This closes #1994
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/37080e9a
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/37080e9a
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/37080e9a
Branch: refs/heads/carbonstore
Commit: 37080e9a3eb9472265eea1b0387c3e4f3047619a
Parents: ab9b4cf
Author: xuchuanyin <xu...@hust.edu.cn>
Authored: Sat Feb 24 21:18:17 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Mon Feb 26 23:30:00 2018 +0800
----------------------------------------------------------------------
.../carbondata/core/locks/LocalFileLock.java | 38 +++++++++-----------
.../store/impl/DFSFileReaderImplUnitTest.java | 11 +++---
.../store/impl/FileFactoryImplUnitTest.java | 28 +++++++++++----
.../filesystem/HDFSCarbonFileTest.java | 3 +-
.../filesystem/LocalCarbonFileTest.java | 20 +++++++----
.../BooleanDataTypesInsertTest.scala | 5 +--
.../carbondata/lcm/locks/LocalFileLockTest.java | 2 +-
.../loading/csvinput/CSVInputFormatTest.java | 1 +
8 files changed, 63 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/37080e9a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
index c3dfb57..793d074 100644
--- a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
+++ b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
@@ -17,11 +17,12 @@
package org.apache.carbondata.core.locks;
-import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.channels.FileChannel;
import java.nio.channels.FileLock;
import java.nio.channels.OverlappingFileLockException;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
@@ -29,6 +30,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.util.CarbonUtil;
/**
* This class handles the file locking in the local file system.
@@ -41,11 +43,6 @@ public class LocalFileLock extends AbstractCarbonLock {
private String location;
/**
- * fileOutputStream of the local lock file
- */
- private FileOutputStream fileOutputStream;
-
- /**
* channel is the FileChannel of the lock file.
*/
private FileChannel channel;
@@ -105,8 +102,8 @@ public class LocalFileLock extends AbstractCarbonLock {
FileFactory.createNewLockFile(lockFilePath, FileFactory.getFileType(location));
}
- fileOutputStream = new FileOutputStream(lockFilePath);
- channel = fileOutputStream.getChannel();
+ channel = FileChannel.open(Paths.get(lockFilePath), StandardOpenOption.WRITE,
+ StandardOpenOption.APPEND);
try {
fileLock = channel.tryLock();
} catch (OverlappingFileLockException e) {
@@ -139,20 +136,17 @@ public class LocalFileLock extends AbstractCarbonLock {
} catch (IOException e) {
status = false;
} finally {
- if (null != fileOutputStream) {
- try {
- fileOutputStream.close();
- // deleting the lock file after releasing the lock.
- CarbonFile lockFile = FileFactory
- .getCarbonFile(lockFilePath, FileFactory.getFileType(lockFilePath));
- if (!lockFile.exists() || lockFile.delete()) {
- LOGGER.info("Successfully deleted the lock file " + lockFilePath);
- } else {
- LOGGER.error("Not able to delete the lock file " + lockFilePath);
- status = false;
- }
- } catch (IOException e) {
- LOGGER.error(e.getMessage());
+ CarbonUtil.closeStreams(channel);
+
+ // deleting the lock file after releasing the lock.
+ if (null != lockFilePath) {
+ CarbonFile lockFile = FileFactory.getCarbonFile(lockFilePath,
+ FileFactory.getFileType(lockFilePath));
+ if (!lockFile.exists() || lockFile.delete()) {
+ LOGGER.info("Successfully deleted the lock file " + lockFilePath);
+ } else {
+ LOGGER.error("Not able to delete the lock file " + lockFilePath);
+ status = false;
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/37080e9a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
index da61a94..30144c1 100644
--- a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
@@ -38,7 +38,6 @@ import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertThat;
public class DFSFileReaderImplUnitTest {
-
private static DFSFileReaderImpl dfsFileHolder;
private static String fileName;
private static String fileNameWithEmptyContent;
@@ -50,10 +49,8 @@ public class DFSFileReaderImplUnitTest {
file = new File("Test.carbondata");
fileWithEmptyContent = new File("TestEXception.carbondata");
- if (!file.exists()) try {
- file.createNewFile();
- } catch (IOException e) {
- e.printStackTrace();
+ if (file.exists()) {
+ file.delete();
}
if (!fileWithEmptyContent.exists()) try {
fileWithEmptyContent.createNewFile();
@@ -61,10 +58,12 @@ public class DFSFileReaderImplUnitTest {
e.printStackTrace();
}
try {
- FileOutputStream of = new FileOutputStream(file, true);
+ FileOutputStream of = new FileOutputStream(file, false);
BufferedWriter br = new BufferedWriter(new OutputStreamWriter(of, "UTF-8"));
br.write("Hello World");
br.close();
+ of.flush();
+ of.close();
} catch (Exception e) {
e.getMessage();
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/37080e9a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileFactoryImplUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileFactoryImplUnitTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileFactoryImplUnitTest.java
index 65590d6..0e7d1c9 100644
--- a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileFactoryImplUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileFactoryImplUnitTest.java
@@ -17,6 +17,7 @@
package org.apache.carbondata.core.carbon.datastorage.filesystem.store.impl;
+import java.io.DataOutputStream;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
@@ -43,6 +44,10 @@ public class FileFactoryImplUnitTest {
@AfterClass
public static void tearDown() {
+ cleanUp();
+ }
+
+ private static void cleanUp() {
File file = new File(filePath);
if (file.exists()) {
file.delete();
@@ -83,17 +88,17 @@ public class FileFactoryImplUnitTest {
}
@Test public void testCreateNewFileWithDefaultFileType() throws IOException {
- tearDown();
+ cleanUp();
assertTrue(FileFactory.createNewFile(filePath, FileFactory.FileType.LOCAL));
}
@Test public void testCreateNewLockFileWithDefaultFileType() throws IOException {
- tearDown();
+ cleanUp();
assertTrue(FileFactory.createNewLockFile(filePath, FileFactory.FileType.LOCAL));
}
@Test public void testCreateNewLockFileWithViewFsFileType() throws IOException {
- tearDown();
+ cleanUp();
assertTrue(FileFactory.createNewLockFile(filePath, FileFactory.FileType.VIEWFS));
}
@@ -129,20 +134,29 @@ public class FileFactoryImplUnitTest {
assertTrue(FileFactory.mkdirs(filePath, FileFactory.FileType.VIEWFS));
}
- @Test public void testGetDataOutputStreamUsingAppendeForException() {
+ @Test public void testGetDataOutputStreamUsingAppendeForException() throws IOException {
+ DataOutputStream outputStream = null;
try {
- FileFactory.getDataOutputStreamUsingAppend(filePath, FileFactory.FileType.VIEWFS);
+ outputStream = FileFactory.getDataOutputStreamUsingAppend(filePath, FileFactory.FileType.VIEWFS);
} catch (Exception exception) {
assertEquals("Not supported", exception.getMessage());
+ } finally {
+ if (null != outputStream) {
+ outputStream.close();
+ }
}
}
@Test public void getDataOutputStreamForVIEWFSType() throws IOException {
- assertNotNull(FileFactory.getDataOutputStream(filePath, FileFactory.FileType.VIEWFS));
+ DataOutputStream outputStream = FileFactory.getDataOutputStream(filePath, FileFactory.FileType.VIEWFS);
+ assertNotNull(outputStream);
+ outputStream.close();
}
@Test public void getDataOutputStreamForLocalType() throws IOException {
- assertNotNull(FileFactory.getDataOutputStream(filePath, FileFactory.FileType.LOCAL));
+ DataOutputStream outputStream = FileFactory.getDataOutputStream(filePath, FileFactory.FileType.LOCAL);
+ assertNotNull(outputStream);
+ outputStream.close();
}
@Test public void testGetCarbonFile() throws IOException {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/37080e9a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
index 4018123..42d4afa 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
@@ -58,7 +58,7 @@ public class HDFSCarbonFileTest {
@BeforeClass
static public void setUp() throws IOException {
Configuration config = new Configuration();
-//adding local hadoop configuration
+ // adding local hadoop configuration
config.addResource(new Path("core-site.xml"));
config.addResource(new Path("hdfs-site.xml"));
fileName = "Test.carbondata"; //this path is HDFS path
@@ -75,6 +75,7 @@ public class HDFSCarbonFileTest {
BufferedWriter br = new BufferedWriter(new OutputStreamWriter(os, "UTF-8"));
br.write("Hello World");
br.close();
+ os.close();
fs.close();
fileStatus = new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName));
http://git-wip-us.apache.org/repos/asf/carbondata/blob/37080e9a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java
index 96ef106..14f9fe2 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java
@@ -29,6 +29,7 @@ import sun.nio.ch.FileChannelImpl;
import java.io.*;
import java.nio.channels.ReadableByteChannel;
import java.util.Objects;
+import java.util.UUID;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
@@ -44,8 +45,8 @@ public class LocalCarbonFileTest {
@BeforeClass
static public void setUp() {
- file = new File("Test.carbondata");
- dir = new File("Testdir.carbondata");
+ file = new File("TestLocalCarbonFile");
+ dir = new File("TestLocalCarbonDir");
if (!file.exists())
try {
file.createNewFile();
@@ -60,6 +61,7 @@ public class LocalCarbonFileTest {
byte[] bytes = "core java api".getBytes();
oFile.write(bytes);
+ oFile.close();
} catch (FileNotFoundException e) {
e.printStackTrace();
localCarbonFile = new LocalCarbonFile(file);
@@ -121,8 +123,9 @@ public class LocalCarbonFileTest {
@Test
public void testRenameForce() {
localCarbonFile = new LocalCarbonFile(file);
- assertTrue(localCarbonFile.renameForce("Testdb.carbon"));
- File file1 = new File("Testdb.carbon");
+ String destFile = "TestRename" + UUID.randomUUID().toString();
+ assertTrue(localCarbonFile.renameForce(destFile));
+ File file1 = new File(destFile);
if (file1.exists()) {
file1.delete();
}
@@ -131,7 +134,12 @@ public class LocalCarbonFileTest {
@Test
public void testRenameTo() {
localCarbonFile = new LocalCarbonFile(file);
- assertTrue(!localCarbonFile.renameTo("Testdb.carbon"));
+ String destFile = "TestRename" + UUID.randomUUID().toString();
+ assertTrue(!localCarbonFile.renameTo(destFile));
+ File file1 = new File(destFile);
+ if (file1.exists()) {
+ file1.delete();
+ }
}
@Test
@@ -463,6 +471,6 @@ public class LocalCarbonFileTest {
localCarbonFile = new LocalCarbonFile("demo.txt");
- assertEquals(localCarbonFile.renameForce("Test.carbondata"), true);
+ assertEquals(localCarbonFile.renameForce("renameToFile"), true);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/37080e9a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesInsertTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesInsertTest.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesInsertTest.scala
index f8cfa6b..45edd3d 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesInsertTest.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesInsertTest.scala
@@ -23,6 +23,7 @@ import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.util.CarbonProperties
class BooleanDataTypesInsertTest extends QueryTest with BeforeAndAfterEach with BeforeAndAfterAll {
@@ -555,7 +556,7 @@ class BooleanDataTypesInsertTest extends QueryTest with BeforeAndAfterEach with
sql(
s"""
- | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | LOAD DATA LOCAL INPATH '${FileFactory.getUpdatedFilePath(storeLocation)}'
| INTO TABLE hive_table
""".stripMargin)
@@ -923,7 +924,7 @@ class BooleanDataTypesInsertTest extends QueryTest with BeforeAndAfterEach with
sql(
s"""
- | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | LOAD DATA LOCAL INPATH '${FileFactory.getUpdatedFilePath(storeLocation)}'
| INTO TABLE hive_table
""".stripMargin)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/37080e9a/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java b/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java
index 812db5c..8d5f3d4 100644
--- a/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java
+++ b/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java
@@ -65,7 +65,7 @@ public class LocalFileLockTest {
Assert.assertTrue(localLock1.unlock());
Assert.assertTrue(localLock2.lock());
-
+ Assert.assertTrue(localLock2.unlock());
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/37080e9a/processing/src/test/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormatTest.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormatTest.java b/processing/src/test/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormatTest.java
index 14c680e..9a9c662 100644
--- a/processing/src/test/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormatTest.java
+++ b/processing/src/test/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormatTest.java
@@ -147,6 +147,7 @@ public class CSVInputFormatTest extends TestCase {
FileOutputFormat.setOutputPath(job, new Path(output.getCanonicalPath()));
Assert.assertTrue(job.waitForCompletion(true));
+ deleteOutput(output);
}
private void prepareConf(Configuration conf) {