You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2018/03/04 12:25:09 UTC
[49/50] [abbrv] carbondata git commit: [CARBONDATA-1114][Tests] Fix
bugs in tests in windows env
[CARBONDATA-1114][Tests] Fix bugs in tests in windows env
Fix bugs in tests that will cause failure under windows env
This closes #1994
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/87bda960
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/87bda960
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/87bda960
Branch: refs/heads/carbonstore-rebase5
Commit: 87bda9601ed23871454640432025858799bff1f6
Parents: 49d06c2
Author: xuchuanyin <xu...@hust.edu.cn>
Authored: Sat Feb 24 21:18:17 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Sun Mar 4 20:08:31 2018 +0800
----------------------------------------------------------------------
.../carbondata/core/locks/LocalFileLock.java | 30 +++++++++++---------
.../store/impl/DFSFileReaderImplUnitTest.java | 11 ++++---
.../store/impl/FileFactoryImplUnitTest.java | 28 +++++++++++++-----
.../filesystem/HDFSCarbonFileTest.java | 3 +-
.../filesystem/LocalCarbonFileTest.java | 20 +++++++++----
.../BooleanDataTypesInsertTest.scala | 5 ++--
.../carbondata/lcm/locks/LocalFileLockTest.java | 2 +-
.../loading/csvinput/CSVInputFormatTest.java | 1 +
8 files changed, 64 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/87bda960/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
index 75ea074..cb80877 100644
--- a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
+++ b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
@@ -17,17 +17,20 @@
package org.apache.carbondata.core.locks;
-import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.channels.FileChannel;
import java.nio.channels.FileLock;
import java.nio.channels.OverlappingFileLockException;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.util.CarbonUtil;
/**
* This class handles the file locking in the local file system.
@@ -40,11 +43,6 @@ public class LocalFileLock extends AbstractCarbonLock {
private String location;
/**
- * fileOutputStream of the local lock file
- */
- private FileOutputStream fileOutputStream;
-
- /**
* channel is the FileChannel of the lock file.
*/
private FileChannel channel;
@@ -104,8 +102,8 @@ public class LocalFileLock extends AbstractCarbonLock {
FileFactory.createNewLockFile(lockFilePath, FileFactory.getFileType(location));
}
- fileOutputStream = new FileOutputStream(lockFilePath);
- channel = fileOutputStream.getChannel();
+ channel = FileChannel.open(Paths.get(lockFilePath), StandardOpenOption.WRITE,
+ StandardOpenOption.APPEND);
try {
fileLock = channel.tryLock();
} catch (OverlappingFileLockException e) {
@@ -137,11 +135,17 @@ public class LocalFileLock extends AbstractCarbonLock {
} catch (IOException e) {
status = false;
} finally {
- if (null != fileOutputStream) {
- try {
- fileOutputStream.close();
- } catch (IOException e) {
- LOGGER.error(e.getMessage());
+ CarbonUtil.closeStreams(channel);
+
+ // deleting the lock file after releasing the lock.
+ if (null != lockFilePath) {
+ CarbonFile lockFile = FileFactory.getCarbonFile(lockFilePath,
+ FileFactory.getFileType(lockFilePath));
+ if (!lockFile.exists() || lockFile.delete()) {
+ LOGGER.info("Successfully deleted the lock file " + lockFilePath);
+ } else {
+ LOGGER.error("Not able to delete the lock file " + lockFilePath);
+ status = false;
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/87bda960/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
index da61a94..30144c1 100644
--- a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
@@ -38,7 +38,6 @@ import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertThat;
public class DFSFileReaderImplUnitTest {
-
private static DFSFileReaderImpl dfsFileHolder;
private static String fileName;
private static String fileNameWithEmptyContent;
@@ -50,10 +49,8 @@ public class DFSFileReaderImplUnitTest {
file = new File("Test.carbondata");
fileWithEmptyContent = new File("TestEXception.carbondata");
- if (!file.exists()) try {
- file.createNewFile();
- } catch (IOException e) {
- e.printStackTrace();
+ if (file.exists()) {
+ file.delete();
}
if (!fileWithEmptyContent.exists()) try {
fileWithEmptyContent.createNewFile();
@@ -61,10 +58,12 @@ public class DFSFileReaderImplUnitTest {
e.printStackTrace();
}
try {
- FileOutputStream of = new FileOutputStream(file, true);
+ FileOutputStream of = new FileOutputStream(file, false);
BufferedWriter br = new BufferedWriter(new OutputStreamWriter(of, "UTF-8"));
br.write("Hello World");
br.close();
+ of.flush();
+ of.close();
} catch (Exception e) {
e.getMessage();
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/87bda960/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileFactoryImplUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileFactoryImplUnitTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileFactoryImplUnitTest.java
index 65590d6..0e7d1c9 100644
--- a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileFactoryImplUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileFactoryImplUnitTest.java
@@ -17,6 +17,7 @@
package org.apache.carbondata.core.carbon.datastorage.filesystem.store.impl;
+import java.io.DataOutputStream;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
@@ -43,6 +44,10 @@ public class FileFactoryImplUnitTest {
@AfterClass
public static void tearDown() {
+ cleanUp();
+ }
+
+ private static void cleanUp() {
File file = new File(filePath);
if (file.exists()) {
file.delete();
@@ -83,17 +88,17 @@ public class FileFactoryImplUnitTest {
}
@Test public void testCreateNewFileWithDefaultFileType() throws IOException {
- tearDown();
+ cleanUp();
assertTrue(FileFactory.createNewFile(filePath, FileFactory.FileType.LOCAL));
}
@Test public void testCreateNewLockFileWithDefaultFileType() throws IOException {
- tearDown();
+ cleanUp();
assertTrue(FileFactory.createNewLockFile(filePath, FileFactory.FileType.LOCAL));
}
@Test public void testCreateNewLockFileWithViewFsFileType() throws IOException {
- tearDown();
+ cleanUp();
assertTrue(FileFactory.createNewLockFile(filePath, FileFactory.FileType.VIEWFS));
}
@@ -129,20 +134,29 @@ public class FileFactoryImplUnitTest {
assertTrue(FileFactory.mkdirs(filePath, FileFactory.FileType.VIEWFS));
}
- @Test public void testGetDataOutputStreamUsingAppendeForException() {
+ @Test public void testGetDataOutputStreamUsingAppendeForException() throws IOException {
+ DataOutputStream outputStream = null;
try {
- FileFactory.getDataOutputStreamUsingAppend(filePath, FileFactory.FileType.VIEWFS);
+ outputStream = FileFactory.getDataOutputStreamUsingAppend(filePath, FileFactory.FileType.VIEWFS);
} catch (Exception exception) {
assertEquals("Not supported", exception.getMessage());
+ } finally {
+ if (null != outputStream) {
+ outputStream.close();
+ }
}
}
@Test public void getDataOutputStreamForVIEWFSType() throws IOException {
- assertNotNull(FileFactory.getDataOutputStream(filePath, FileFactory.FileType.VIEWFS));
+ DataOutputStream outputStream = FileFactory.getDataOutputStream(filePath, FileFactory.FileType.VIEWFS);
+ assertNotNull(outputStream);
+ outputStream.close();
}
@Test public void getDataOutputStreamForLocalType() throws IOException {
- assertNotNull(FileFactory.getDataOutputStream(filePath, FileFactory.FileType.LOCAL));
+ DataOutputStream outputStream = FileFactory.getDataOutputStream(filePath, FileFactory.FileType.LOCAL);
+ assertNotNull(outputStream);
+ outputStream.close();
}
@Test public void testGetCarbonFile() throws IOException {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/87bda960/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
index 4018123..42d4afa 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
@@ -58,7 +58,7 @@ public class HDFSCarbonFileTest {
@BeforeClass
static public void setUp() throws IOException {
Configuration config = new Configuration();
-//adding local hadoop configuration
+ // adding local hadoop configuration
config.addResource(new Path("core-site.xml"));
config.addResource(new Path("hdfs-site.xml"));
fileName = "Test.carbondata"; //this path is HDFS path
@@ -75,6 +75,7 @@ public class HDFSCarbonFileTest {
BufferedWriter br = new BufferedWriter(new OutputStreamWriter(os, "UTF-8"));
br.write("Hello World");
br.close();
+ os.close();
fs.close();
fileStatus = new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName));
http://git-wip-us.apache.org/repos/asf/carbondata/blob/87bda960/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java
index 96ef106..14f9fe2 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java
@@ -29,6 +29,7 @@ import sun.nio.ch.FileChannelImpl;
import java.io.*;
import java.nio.channels.ReadableByteChannel;
import java.util.Objects;
+import java.util.UUID;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
@@ -44,8 +45,8 @@ public class LocalCarbonFileTest {
@BeforeClass
static public void setUp() {
- file = new File("Test.carbondata");
- dir = new File("Testdir.carbondata");
+ file = new File("TestLocalCarbonFile");
+ dir = new File("TestLocalCarbonDir");
if (!file.exists())
try {
file.createNewFile();
@@ -60,6 +61,7 @@ public class LocalCarbonFileTest {
byte[] bytes = "core java api".getBytes();
oFile.write(bytes);
+ oFile.close();
} catch (FileNotFoundException e) {
e.printStackTrace();
localCarbonFile = new LocalCarbonFile(file);
@@ -121,8 +123,9 @@ public class LocalCarbonFileTest {
@Test
public void testRenameForce() {
localCarbonFile = new LocalCarbonFile(file);
- assertTrue(localCarbonFile.renameForce("Testdb.carbon"));
- File file1 = new File("Testdb.carbon");
+ String destFile = "TestRename" + UUID.randomUUID().toString();
+ assertTrue(localCarbonFile.renameForce(destFile));
+ File file1 = new File(destFile);
if (file1.exists()) {
file1.delete();
}
@@ -131,7 +134,12 @@ public class LocalCarbonFileTest {
@Test
public void testRenameTo() {
localCarbonFile = new LocalCarbonFile(file);
- assertTrue(!localCarbonFile.renameTo("Testdb.carbon"));
+ String destFile = "TestRename" + UUID.randomUUID().toString();
+ assertTrue(!localCarbonFile.renameTo(destFile));
+ File file1 = new File(destFile);
+ if (file1.exists()) {
+ file1.delete();
+ }
}
@Test
@@ -463,6 +471,6 @@ public class LocalCarbonFileTest {
localCarbonFile = new LocalCarbonFile("demo.txt");
- assertEquals(localCarbonFile.renameForce("Test.carbondata"), true);
+ assertEquals(localCarbonFile.renameForce("renameToFile"), true);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/87bda960/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesInsertTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesInsertTest.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesInsertTest.scala
index f8cfa6b..45edd3d 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesInsertTest.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesInsertTest.scala
@@ -23,6 +23,7 @@ import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.util.CarbonProperties
class BooleanDataTypesInsertTest extends QueryTest with BeforeAndAfterEach with BeforeAndAfterAll {
@@ -555,7 +556,7 @@ class BooleanDataTypesInsertTest extends QueryTest with BeforeAndAfterEach with
sql(
s"""
- | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | LOAD DATA LOCAL INPATH '${FileFactory.getUpdatedFilePath(storeLocation)}'
| INTO TABLE hive_table
""".stripMargin)
@@ -923,7 +924,7 @@ class BooleanDataTypesInsertTest extends QueryTest with BeforeAndAfterEach with
sql(
s"""
- | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | LOAD DATA LOCAL INPATH '${FileFactory.getUpdatedFilePath(storeLocation)}'
| INTO TABLE hive_table
""".stripMargin)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/87bda960/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java b/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java
index 812db5c..8d5f3d4 100644
--- a/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java
+++ b/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java
@@ -65,7 +65,7 @@ public class LocalFileLockTest {
Assert.assertTrue(localLock1.unlock());
Assert.assertTrue(localLock2.lock());
-
+ Assert.assertTrue(localLock2.unlock());
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/87bda960/processing/src/test/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormatTest.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormatTest.java b/processing/src/test/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormatTest.java
index 14c680e..9a9c662 100644
--- a/processing/src/test/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormatTest.java
+++ b/processing/src/test/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormatTest.java
@@ -147,6 +147,7 @@ public class CSVInputFormatTest extends TestCase {
FileOutputFormat.setOutputPath(job, new Path(output.getCanonicalPath()));
Assert.assertTrue(job.waitForCompletion(true));
+ deleteOutput(output);
}
private void prepareConf(Configuration conf) {