You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2018/11/01 07:54:47 UTC
carbondata git commit: [CARBONDATA-3053][Cli] Fix bugs for carbon-cli
found in windows env
Repository: carbondata
Updated Branches:
refs/heads/master 82eec10cb -> 26d1cdd9c
[CARBONDATA-3053][Cli] Fix bugs for carbon-cli found in windows env
Tests run failed in windows env due to:
1.unclosed file stream causing failure in clearing files in windows
2.the line separator is not n in windows
In this commit, We fix the bugs.
This closes #2874
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/26d1cdd9
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/26d1cdd9
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/26d1cdd9
Branch: refs/heads/master
Commit: 26d1cdd9cac47cf4caa41611413fe5d18e896d17
Parents: 82eec10
Author: xuchuanyin <xu...@hust.edu.cn>
Authored: Mon Oct 29 19:49:25 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Thu Nov 1 15:54:25 2018 +0800
----------------------------------------------------------------------
.../org/apache/carbondata/tool/CarbonCli.java | 2 +
.../org/apache/carbondata/tool/DataFile.java | 10 +-
.../org/apache/carbondata/tool/DataSummary.java | 5 +
.../apache/carbondata/tool/FileCollector.java | 6 +
.../apache/carbondata/tool/ScanBenchmark.java | 10 +-
.../apache/carbondata/tool/CarbonCliTest.java | 184 +++++++++++--------
6 files changed, 133 insertions(+), 84 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/26d1cdd9/tools/cli/src/main/java/org/apache/carbondata/tool/CarbonCli.java
----------------------------------------------------------------------
diff --git a/tools/cli/src/main/java/org/apache/carbondata/tool/CarbonCli.java b/tools/cli/src/main/java/org/apache/carbondata/tool/CarbonCli.java
index 11553a6..bb2260b 100644
--- a/tools/cli/src/main/java/org/apache/carbondata/tool/CarbonCli.java
+++ b/tools/cli/src/main/java/org/apache/carbondata/tool/CarbonCli.java
@@ -181,6 +181,8 @@ public class CarbonCli {
out.flush();
} catch (IOException | MemoryException e) {
e.printStackTrace();
+ } finally {
+ out.close();
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/26d1cdd9/tools/cli/src/main/java/org/apache/carbondata/tool/DataFile.java
----------------------------------------------------------------------
diff --git a/tools/cli/src/main/java/org/apache/carbondata/tool/DataFile.java b/tools/cli/src/main/java/org/apache/carbondata/tool/DataFile.java
index c8fdc9e..457ef0c 100644
--- a/tools/cli/src/main/java/org/apache/carbondata/tool/DataFile.java
+++ b/tools/cli/src/main/java/org/apache/carbondata/tool/DataFile.java
@@ -114,7 +114,8 @@ class DataFile {
this.footer = footer;
String filePath = dataFile.getPath();
// folder path that contains this file
- String fileName = filePath.substring(filePath.lastIndexOf(FILE_SEPARATOR));
+ String fileName =
+ filePath.substring(filePath.replaceAll("\\\\", FILE_SEPARATOR).lastIndexOf(FILE_SEPARATOR));
this.shardName = CarbonTablePath.getShardName(fileName);
this.partNo = CarbonTablePath.DataFileUtil.getPartNo(fileName);
@@ -160,7 +161,9 @@ class DataFile {
}
FileFooter3 readFooter() throws IOException {
- this.fileReader = FileFactory.getFileHolder(FileFactory.getFileType(dataFile.getPath()));
+ if (this.fileReader == null) {
+ this.fileReader = FileFactory.getFileHolder(FileFactory.getFileType(dataFile.getPath()));
+ }
ByteBuffer buffer = fileReader.readByteBuffer(FileFactory.getUpdatedFilePath(
dataFile.getPath()), dataFile.getSize() - 8, 8);
this.footerOffset = buffer.getLong();
@@ -500,4 +503,7 @@ class DataFile {
}
}
+ public void close() throws IOException {
+ this.fileReader.finish();
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/26d1cdd9/tools/cli/src/main/java/org/apache/carbondata/tool/DataSummary.java
----------------------------------------------------------------------
diff --git a/tools/cli/src/main/java/org/apache/carbondata/tool/DataSummary.java b/tools/cli/src/main/java/org/apache/carbondata/tool/DataSummary.java
index 4a8c85e..cd4c040 100644
--- a/tools/cli/src/main/java/org/apache/carbondata/tool/DataSummary.java
+++ b/tools/cli/src/main/java/org/apache/carbondata/tool/DataSummary.java
@@ -111,6 +111,11 @@ class DataSummary implements Command {
collectColumnChunkMeta(columName);
}
}
+
+ collector.close();
+ for (DataFile file : dataFiles.values()) {
+ file.close();
+ }
}
private void collectSchemaDetails(DataFile dataFile) throws IOException {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/26d1cdd9/tools/cli/src/main/java/org/apache/carbondata/tool/FileCollector.java
----------------------------------------------------------------------
diff --git a/tools/cli/src/main/java/org/apache/carbondata/tool/FileCollector.java b/tools/cli/src/main/java/org/apache/carbondata/tool/FileCollector.java
index b2ff061..9cd743a 100644
--- a/tools/cli/src/main/java/org/apache/carbondata/tool/FileCollector.java
+++ b/tools/cli/src/main/java/org/apache/carbondata/tool/FileCollector.java
@@ -145,4 +145,10 @@ class FileCollector {
numRow / numBlocklet);
outPuts.add(format1);
}
+
+ public void close() throws IOException {
+ for (DataFile file : dataFiles.values()) {
+ file.close();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/26d1cdd9/tools/cli/src/main/java/org/apache/carbondata/tool/ScanBenchmark.java
----------------------------------------------------------------------
diff --git a/tools/cli/src/main/java/org/apache/carbondata/tool/ScanBenchmark.java b/tools/cli/src/main/java/org/apache/carbondata/tool/ScanBenchmark.java
index ddb9652..12384fc 100644
--- a/tools/cli/src/main/java/org/apache/carbondata/tool/ScanBenchmark.java
+++ b/tools/cli/src/main/java/org/apache/carbondata/tool/ScanBenchmark.java
@@ -18,6 +18,7 @@
package org.apache.carbondata.tool;
import java.io.IOException;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;
@@ -68,7 +69,12 @@ class ScanBenchmark implements Command {
return;
}
Map<String, DataFile> dataFiles = collector.getDataFiles();
- file = dataFiles.entrySet().iterator().next().getValue();
+ Iterator<DataFile> iterator = dataFiles.values().iterator();
+ // use the first file and close the rest
+ file = iterator.next();
+ while (iterator.hasNext()) {
+ iterator.next().close();
+ }
}
outPuts.add("\n## Benchmark");
@@ -139,7 +145,7 @@ class ScanBenchmark implements Command {
}
}
}
-
+ file.close();
}
interface Operation {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/26d1cdd9/tools/cli/src/test/java/org/apache/carbondata/tool/CarbonCliTest.java
----------------------------------------------------------------------
diff --git a/tools/cli/src/test/java/org/apache/carbondata/tool/CarbonCliTest.java b/tools/cli/src/test/java/org/apache/carbondata/tool/CarbonCliTest.java
index 002bc8d..e2db442 100644
--- a/tools/cli/src/test/java/org/apache/carbondata/tool/CarbonCliTest.java
+++ b/tools/cli/src/test/java/org/apache/carbondata/tool/CarbonCliTest.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.io.PrintStream;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.sdk.file.Field;
import org.apache.carbondata.sdk.file.Schema;
import org.apache.carbondata.sdk.file.TestUtil;
@@ -37,6 +38,22 @@ public class CarbonCliTest {
private String path = "./CarbonCliTest";
+ private String buildLines(String... lines) {
+ ByteArrayOutputStream expectedOut = null;
+ PrintStream expectedStream = null;
+ try {
+ expectedOut = new ByteArrayOutputStream();
+ expectedStream = new PrintStream(expectedOut);
+ for (String line : lines) {
+ expectedStream.println(line);
+ }
+
+ return new String(expectedOut.toByteArray());
+ } finally {
+ CarbonUtil.closeStreams(expectedStream, expectedOut);
+ }
+ }
+
@Before
public void before() throws IOException {
FileUtils.deleteDirectory(new File(path));
@@ -73,23 +90,25 @@ public class CarbonCliTest {
PrintStream stream = new PrintStream(out);
CarbonCli.run(args, stream);
String output = new String(out.toByteArray());
- Assert.assertTrue(
- output.contains(
- "Input Folder: ./CarbonCliTest\n"
- + "## Summary\n"
- + "total: 6 blocks, 2 shards, 14 blocklets, 314 pages, 10,000,000 rows, 32.27MB\n"
- + "avg: 5.38MB/block, 2.30MB/blocklet, 1,666,666 rows/block, 714,285 rows/blocklet"));
+
+ String expectedOutput = buildLines(
+ "Input Folder: ./CarbonCliTest",
+ "## Summary",
+ "total: 6 blocks, 2 shards, 14 blocklets, 314 pages, 10,000,000 rows, 32.27MB",
+ "avg: 5.38MB/block, 2.30MB/blocklet, 1,666,666 rows/block, 714,285 rows/blocklet");
+ Assert.assertTrue(output.contains(expectedOutput));
String[] args2 = {"-cmd", "summary", "-p", path, "-s"};
out = new ByteArrayOutputStream();
stream = new PrintStream(out);
CarbonCli.run(args2, stream);
output = new String(out.toByteArray());
- Assert.assertTrue(
- output.contains(
- "Column Name Data Type Column Type SortColumn Encoding Ordinal Id \n"
- + "name STRING dimension true [INVERTED_INDEX] 0 NA \n"
- + "age INT measure false [] 1 NA "));
+
+ expectedOutput = buildLines(
+ "Column Name Data Type Column Type SortColumn Encoding Ordinal Id ",
+ "name STRING dimension true [INVERTED_INDEX] 0 NA ",
+ "age INT measure false [] 1 NA ");
+ Assert.assertTrue(output.contains(expectedOutput));
String[] args3 = {"-cmd", "summary", "-p", path, "-t"};
out = new ByteArrayOutputStream();
@@ -97,42 +116,44 @@ public class CarbonCliTest {
CarbonCli.run(args3, stream);
output = new String(out.toByteArray());
- Assert.assertTrue(
- output.contains(
- "## Table Properties\n"
- + "schema file not found"));
+ expectedOutput = buildLines(
+ "## Table Properties",
+ "schema file not found");
+ Assert.assertTrue(output.contains(expectedOutput));
String[] args4 = {"-cmd", "summary", "-p", path, "-b", "7"};
out = new ByteArrayOutputStream();
stream = new PrintStream(out);
CarbonCli.run(args4, stream);
output = new String(out.toByteArray());
- Assert.assertTrue(
- output.contains(
- "BLK BLKLT NumPages NumRows Size \n"
- + "0 0 25 800,000 2.58MB \n"
- + "0 1 25 800,000 2.58MB \n"
- + "1 0 25 800,000 2.58MB \n"
- + "1 1 25 800,000 2.58MB \n"
- + "2 0 25 800,000 2.58MB \n"
- + "2 1 25 800,000 2.58MB \n"
- + "2 2 7 200,000 660.79KB "));
+
+ expectedOutput = buildLines(
+ "BLK BLKLT NumPages NumRows Size ",
+ "0 0 25 800,000 2.58MB ",
+ "0 1 25 800,000 2.58MB ",
+ "1 0 25 800,000 2.58MB ",
+ "1 1 25 800,000 2.58MB ",
+ "2 0 25 800,000 2.58MB ",
+ "2 1 25 800,000 2.58MB ",
+ "2 2 7 200,000 660.79KB ");
+ Assert.assertTrue(output.contains(expectedOutput));
String[] args5 = {"-cmd", "summary", "-p", path, "-c", "name"};
out = new ByteArrayOutputStream();
stream = new PrintStream(out);
CarbonCli.run(args5, stream);
output = new String(out.toByteArray());
- Assert.assertTrue(
- output.contains(
- "BLK BLKLT Meta Size Data Size LocalDict DictEntries DictSize AvgPageSize Min% Max% Min Max \n"
- + "0 0 1.81KB 295.98KB false 0 0.0B 11.77KB NA NA robot0 robot1 \n"
- + "0 1 1.81KB 295.99KB false 0 0.0B 11.77KB NA NA robot1 robot3 \n"
- + "1 0 1.81KB 295.98KB false 0 0.0B 11.77KB NA NA robot3 robot4 \n"
- + "1 1 1.81KB 295.99KB false 0 0.0B 11.77KB NA NA robot4 robot6 \n"
- + "2 0 1.81KB 295.98KB false 0 0.0B 11.77KB NA NA robot6 robot7 \n"
- + "2 1 1.81KB 295.98KB false 0 0.0B 11.77KB NA NA robot8 robot9 \n"
- + "2 2 519.0B 74.06KB false 0 0.0B 10.51KB NA NA robot9 robot9 "));
+
+ expectedOutput = buildLines(
+ "BLK BLKLT Meta Size Data Size LocalDict DictEntries DictSize AvgPageSize Min% Max% Min Max ",
+ "0 0 1.81KB 295.98KB false 0 0.0B 11.77KB NA NA robot0 robot1 ",
+ "0 1 1.81KB 295.99KB false 0 0.0B 11.77KB NA NA robot1 robot3 ",
+ "1 0 1.81KB 295.98KB false 0 0.0B 11.77KB NA NA robot3 robot4 ",
+ "1 1 1.81KB 295.99KB false 0 0.0B 11.77KB NA NA robot4 robot6 ",
+ "2 0 1.81KB 295.98KB false 0 0.0B 11.77KB NA NA robot6 robot7 ",
+ "2 1 1.81KB 295.98KB false 0 0.0B 11.77KB NA NA robot8 robot9 ",
+ "2 2 519.0B 74.06KB false 0 0.0B 10.51KB NA NA robot9 robot9 ");
+ Assert.assertTrue(output.contains(expectedOutput));
}
@Test
@@ -142,47 +163,50 @@ public class CarbonCliTest {
PrintStream stream = new PrintStream(out);
CarbonCli.run(args, stream);
String output = new String(out.toByteArray());
- Assert.assertTrue(
- output.contains(
- "Input Folder: ./CarbonCliTest\n"
- + "## Summary\n"
- + "total: 6 blocks, 2 shards, 14 blocklets, 314 pages, 10,000,000 rows, 32.27MB\n"
- + "avg: 5.38MB/block, 2.30MB/blocklet, 1,666,666 rows/block, 714,285 rows/blocklet\n"));
-
- Assert.assertTrue(
- output.contains(
- "Column Name Data Type Column Type SortColumn Encoding Ordinal Id \n"
- + "name STRING dimension true [INVERTED_INDEX] 0 NA \n"
- + "age INT measure false [] 1 NA \n"));
-
- Assert.assertTrue(
- output.contains(
- "## Table Properties\n"
- + "schema file not found"));
-
- Assert.assertTrue(
- output.contains(
- "BLK BLKLT NumPages NumRows Size \n"
- + "0 0 25 800,000 2.58MB \n"
- + "0 1 25 800,000 2.58MB \n"
- + "1 0 25 800,000 2.58MB \n"
- + "1 1 25 800,000 2.58MB"));
-
- Assert.assertTrue(
- output.contains(
- "BLK BLKLT Meta Size Data Size LocalDict DictEntries DictSize AvgPageSize Min% Max% Min Max \n"
- + "0 0 3.00KB 4.87MB false 0 0.0B 93.76KB 0.0 100.0 0 2999990 \n"
- + "0 1 3.00KB 2.29MB false 0 0.0B 93.76KB 0.0 100.0 1 2999992 \n"
- + "1 0 3.00KB 4.87MB false 0 0.0B 93.76KB 0.0 100.0 3 2999993 \n"
- + "1 1 3.00KB 2.29MB false 0 0.0B 93.76KB 0.0 100.0 4 2999995 \n"
- + "2 0 3.00KB 5.52MB false 0 0.0B 93.76KB 0.0 100.0 6 2999997 \n"
- + "2 1 3.00KB 2.94MB false 0 0.0B 93.76KB 0.0 100.0 8 2999998 \n"
- + "2 2 858.0B 586.84KB false 0 0.0B 83.71KB 0.0 100.0 9 2999999 "));
-
- Assert.assertTrue(output.contains(
- "## version Details\n"
- + "written_by Version \n"
- + "TestUtil 1.6.0-SNAPSHOT"));
+
+ String expectedOutput = buildLines(
+ "Input Folder: ./CarbonCliTest",
+ "## Summary",
+ "total: 6 blocks, 2 shards, 14 blocklets, 314 pages, 10,000,000 rows, 32.27MB",
+ "avg: 5.38MB/block, 2.30MB/blocklet, 1,666,666 rows/block, 714,285 rows/blocklet");
+
+ Assert.assertTrue(output.contains(expectedOutput));
+
+ expectedOutput = buildLines(
+ "Column Name Data Type Column Type SortColumn Encoding Ordinal Id ",
+ "name STRING dimension true [INVERTED_INDEX] 0 NA ",
+ "age INT measure false [] 1 NA ");
+ Assert.assertTrue(output.contains(expectedOutput));
+
+ expectedOutput = buildLines(
+ "## Table Properties",
+ "schema file not found");
+ Assert.assertTrue(output.contains(expectedOutput));
+
+ expectedOutput = buildLines(
+ "BLK BLKLT NumPages NumRows Size ",
+ "0 0 25 800,000 2.58MB ",
+ "0 1 25 800,000 2.58MB ",
+ "1 0 25 800,000 2.58MB ",
+ "1 1 25 800,000 2.58MB ");
+ Assert.assertTrue(output.contains(expectedOutput));
+
+ expectedOutput = buildLines(
+ "BLK BLKLT Meta Size Data Size LocalDict DictEntries DictSize AvgPageSize Min% Max% Min Max ",
+ "0 0 3.00KB 4.87MB false 0 0.0B 93.76KB 0.0 100.0 0 2999990 ",
+ "0 1 3.00KB 2.29MB false 0 0.0B 93.76KB 0.0 100.0 1 2999992 ",
+ "1 0 3.00KB 4.87MB false 0 0.0B 93.76KB 0.0 100.0 3 2999993 ",
+ "1 1 3.00KB 2.29MB false 0 0.0B 93.76KB 0.0 100.0 4 2999995 ",
+ "2 0 3.00KB 5.52MB false 0 0.0B 93.76KB 0.0 100.0 6 2999997 ",
+ "2 1 3.00KB 2.94MB false 0 0.0B 93.76KB 0.0 100.0 8 2999998 ",
+ "2 2 858.0B 586.84KB false 0 0.0B 83.71KB 0.0 100.0 9 2999999 ");
+ Assert.assertTrue(output.contains(expectedOutput));
+
+ expectedOutput = buildLines(
+ "## version Details",
+ "written_by Version ",
+ "TestUtil 1.6.0-SNAPSHOT ");
+ Assert.assertTrue(output.contains(expectedOutput));
}
@Test
@@ -193,10 +217,10 @@ public class CarbonCliTest {
CarbonCli.run(args, stream);
String output = new String(out.toByteArray());
System.out.println(output);
- Assert.assertTrue(
- output.contains(
- "Blocklet 0:\n"
- + "Page 0 (offset 0, length 12049): DataChunk2(chunk_meta:ChunkCompressionMeta(compression_codec:DEPRECATED, total_uncompressed_size:256000, total_compressed_size:12049, compressor_name:snappy), rowMajor:false, data_page_length:12039, rowid_page_length:10, presence:PresenceMeta(represents_presence:false, present_bit_stream:00), sort_state:SORT_EXPLICIT, encoders:[INVERTED_INDEX], encoder_meta:[], min_max:BlockletMinMaxIndex(min_values:[72 6F 62 6F 74 30], max_values:[72 6F 62 6F 74 30], min_max_presence:[true]), numberOfRowsInpage:32000)"));
+ String expectedOutput = buildLines(
+ "Blocklet 0:",
+ "Page 0 (offset 0, length 12049): DataChunk2(chunk_meta:ChunkCompressionMeta(compression_codec:DEPRECATED, total_uncompressed_size:256000, total_compressed_size:12049, compressor_name:snappy), rowMajor:false, data_page_length:12039, rowid_page_length:10, presence:PresenceMeta(represents_presence:false, present_bit_stream:00), sort_state:SORT_EXPLICIT, encoders:[INVERTED_INDEX], encoder_meta:[], min_max:BlockletMinMaxIndex(min_values:[72 6F 62 6F 74 30], max_values:[72 6F 62 6F 74 30], min_max_presence:[true]), numberOfRowsInpage:32000)");
+ Assert.assertTrue(output.contains(expectedOutput));
}
@Test