You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by pa...@apache.org on 2016/05/20 12:31:57 UTC
ambari git commit: AMBARI-16418. Upload Table- marks column as INT
since first 6 rows have INT values. (Nitiraj Rathore via pallavkul)
Repository: ambari
Updated Branches:
refs/heads/trunk 248239dfb -> 85798b397
AMBARI-16418. Upload Table- marks column as INT since first 6 rows have INT values. (Nitiraj Rathore via pallavkul)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/85798b39
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/85798b39
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/85798b39
Branch: refs/heads/trunk
Commit: 85798b397e9c551d18a685a4d67bfc5d36aed20a
Parents: 248239d
Author: Pallav Kulshreshtha <pa...@gmail.com>
Authored: Fri May 20 18:01:39 2016 +0530
Committer: Pallav Kulshreshtha <pa...@gmail.com>
Committed: Fri May 20 18:01:39 2016 +0530
----------------------------------------------------------------------
.../hive/resources/uploads/UploadService.java | 4 +-
.../resources/uploads/parsers/DataParser.java | 4 +-
.../hive/resources/uploads/parsers/IParser.java | 5 +-
.../resources/uploads/parsers/ParseUtils.java | 77 ++++++-
.../hive/resources/uploads/parsers/Parser.java | 67 ++++---
.../uploads/parsers/csv/CSVParser.java | 2 +-
.../uploads/parsers/json/JSONParser.java | 2 +-
.../uploads/parsers/xml/XMLParser.java | 2 +-
.../hive/resources/upload/CSVParserTest.java | 72 ++-----
.../resources/upload/DataParserCSVTest.java | 199 ++++++++++++-------
.../resources/upload/DataParserJSONTest.java | 106 ++++------
.../resources/upload/DataParserXMLTest.java | 109 ++++------
.../hive/resources/upload/JsonParserTest.java | 87 +++-----
.../hive/resources/upload/XMLParserTest.java | 94 +++------
14 files changed, 389 insertions(+), 441 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/85798b39/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/UploadService.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/UploadService.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/UploadService.java
index b6b000a..7a46fd0 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/UploadService.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/UploadService.java
@@ -390,7 +390,7 @@ public class UploadService extends BaseService {
out.close();
}
- private PreviewData generatePreview(Boolean isFirstRowHeader, String inputFileType, InputStream uploadedInputStream) throws IOException {
+ private PreviewData generatePreview(Boolean isFirstRowHeader, String inputFileType, InputStream uploadedInputStream) throws Exception {
ParseOptions parseOptions = new ParseOptions();
parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, inputFileType);
if (inputFileType.equals(ParseOptions.InputFileType.CSV.toString())){
@@ -432,7 +432,7 @@ public class UploadService extends BaseService {
String tableName,
String databaseName
- ) throws IOException {
+ ) throws Exception {
LOG.info(" uploading file into databaseName {}, tableName {}", databaseName, tableName);
ParseOptions parseOptions = new ParseOptions();
parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, inputFileType);
http://git-wip-us.apache.org/repos/asf/ambari/blob/85798b39/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/DataParser.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/DataParser.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/DataParser.java
index 7eae679..d03dd7e 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/DataParser.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/DataParser.java
@@ -35,7 +35,7 @@ public class DataParser implements IParser {
private IParser parser;
- public DataParser(Reader reader, ParseOptions parseOptions) throws IOException {
+ public DataParser(Reader reader, ParseOptions parseOptions) throws Exception {
if (parseOptions.getOption(ParseOptions.OPTIONS_FILE_TYPE).equals(ParseOptions.InputFileType.CSV.toString())) {
parser = new CSVParser(reader, parseOptions);
} else if (parseOptions.getOption(ParseOptions.OPTIONS_FILE_TYPE).equals(ParseOptions.InputFileType.JSON.toString())) {
@@ -61,7 +61,7 @@ public class DataParser implements IParser {
}
@Override
- public void close() throws IOException {
+ public void close() throws Exception {
parser.close();
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/85798b39/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/IParser.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/IParser.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/IParser.java
index 6056e73..8b75c04 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/IParser.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/IParser.java
@@ -31,7 +31,7 @@ import java.util.List;
* Interface defining methods for Parsers that can used for generating preview
* and uploading table into hive.
*/
-public interface IParser extends Iterable<Row> {
+public interface IParser extends Iterable<Row>, AutoCloseable{
/**
* @return returns the Reader that can be read to get the table data as CSV Text Data that can be uploaded directly
@@ -42,7 +42,4 @@ public interface IParser extends Iterable<Row> {
PreviewData parsePreview();
Row extractHeader();
-
- void close() throws IOException;
-
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/85798b39/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/ParseUtils.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/ParseUtils.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/ParseUtils.java
index a17346f..3261bfa 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/ParseUtils.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/ParseUtils.java
@@ -18,16 +18,24 @@
package org.apache.ambari.view.hive.resources.uploads.parsers;
-import org.apache.ambari.view.hive.client.ColumnDescription;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
+import java.util.List;
+
+import static org.apache.ambari.view.hive.client.ColumnDescription.*;
public class ParseUtils {
+ protected final static Logger LOG =
+ LoggerFactory.getLogger(ParseUtils.class);
+
final public static String[] DATE_FORMATS = {"mm/dd/yyyy", "dd/mm/yyyy", "mm-dd-yyyy" /*add more formatss*/};
+ final public static DataTypes [] dataTypeList = {DataTypes.BOOLEAN,DataTypes.INT,DataTypes.BIGINT,DataTypes.DOUBLE,DataTypes.CHAR,DataTypes.DATE,DataTypes.STRING};
+
public static boolean isInteger(Object object) {
if (object == null)
return false;
@@ -57,6 +65,12 @@ public class ParseUtils {
return false;
}
+ public static boolean isString(Object object) {
+ if (object == null)
+ return false;
+ else return true; // any non null can always be interpreted as a string
+ }
+
public static boolean isLong(Object object) {
if (object == null)
return false;
@@ -120,15 +134,56 @@ public class ParseUtils {
return false;
}
- public static ColumnDescription.DataTypes detectHiveDataType(Object object) {
+ public static DataTypes detectHiveDataType(Object object) {
// detect Integer
- if (isInteger(object)) return ColumnDescription.DataTypes.INT;
- if (isLong(object)) return ColumnDescription.DataTypes.BIGINT;
- if (isBoolean(object)) return ColumnDescription.DataTypes.BOOLEAN;
- if (isDouble(object)) return ColumnDescription.DataTypes.DOUBLE;
- if (isDate(object)) return ColumnDescription.DataTypes.DATE;
- if (isChar(object)) return ColumnDescription.DataTypes.CHAR;
-
- return ColumnDescription.DataTypes.STRING;
+ if (isBoolean(object)) return DataTypes.BOOLEAN;
+ if (isInteger(object)) return DataTypes.INT;
+ if (isLong(object)) return DataTypes.BIGINT;
+ if (isDouble(object)) return DataTypes.DOUBLE;
+ if (isChar(object)) return DataTypes.CHAR;
+ if (isDate(object)) return DataTypes.DATE;
+
+ return DataTypes.STRING;
+ }
+
+ public static boolean checkDatatype( Object object, DataTypes datatype){
+ switch(datatype){
+
+ case BOOLEAN :
+ return isBoolean(object);
+ case INT :
+ return isInteger(object);
+ case BIGINT :
+ return isLong(object);
+ case DOUBLE:
+ return isDouble(object);
+ case CHAR:
+ return isChar(object);
+ case DATE:
+ return isDate(object);
+ case STRING:
+ return isString(object);
+
+ default:
+ LOG.error("this datatype detection is not supported : {}", datatype);
+ return false;
+ }
+ }
+
+ public static DataTypes detectHiveColumnDataType(List<Object> colValues) {
+ boolean found = true;
+ for(DataTypes datatype : dataTypeList){
+ found = true;
+ for(Object object : colValues){
+ if(!checkDatatype(object,datatype)){
+ found = false;
+ break;
+ }
+ }
+
+ if(found) return datatype;
+ }
+
+ return DataTypes.STRING; //default
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/85798b39/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/Parser.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/Parser.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/Parser.java
index 1644c13..450d896 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/Parser.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/Parser.java
@@ -22,6 +22,8 @@ import org.apache.ambari.view.hive.client.ColumnDescription;
import org.apache.ambari.view.hive.client.Row;
import org.apache.ambari.view.hive.resources.uploads.ColumnDescriptionImpl;
import org.apache.ambari.view.hive.resources.uploads.TableDataReader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.Reader;
import java.util.ArrayList;
@@ -35,6 +37,9 @@ import java.util.NoSuchElementException;
*/
public abstract class Parser implements IParser {
+ protected final static Logger LOG =
+ LoggerFactory.getLogger(Parser.class);
+
protected Reader reader; // same as CSV reader in this case
protected ParseOptions parseOptions;
private int numberOfPreviewRows = 10;
@@ -45,22 +50,23 @@ public abstract class Parser implements IParser {
}
/**
- * returns which datatype was detected for the maximum number of times in the given column
+ * returns which datatype is valid for all the values
+ */
+
+ /**
*
- * @param typeCounts
- * @param colNum
- * @return
+ * @param rows : non empty list of rows
+ * @param colNum : to detect datatype for this column number.
+ * @return data type for that column
*/
- private int getLikelyDataType(int[][] typeCounts, int colNum) {
- int[] colArray = typeCounts[colNum];
- int maxIndex = 0;
- int i = 1;
- for (; i < colArray.length; i++) {
- if (colArray[i] > colArray[maxIndex])
- maxIndex = i;
+ private ColumnDescription.DataTypes getLikelyDataType(List<Row> rows, int colNum) {
+ // order of detection BOOLEAN,INT,BIGINT,DOUBLE,DATE,CHAR,STRING
+ List<Object> colValues = new ArrayList<>(rows.size());
+ for( Row row : rows ){
+ colValues.add(row.getRow()[colNum]);
}
- return maxIndex;
+ return ParseUtils.detectHiveColumnDataType(colValues);
}
@Override
@@ -70,12 +76,15 @@ public abstract class Parser implements IParser {
@Override
public PreviewData parsePreview() {
- List<Row> previewRows;
+ LOG.info("generating preview for : {}", this.parseOptions );
+
+ ArrayList<Row> previewRows;
List<ColumnDescription> header;
try {
numberOfPreviewRows = (Integer) parseOptions.getOption(ParseOptions.OPTIONS_NUMBER_OF_PREVIEW_ROWS);
} catch (Exception e) {
+ LOG.debug("Illegal number of preview columns supplied {}",parseOptions.getOption(ParseOptions.OPTIONS_NUMBER_OF_PREVIEW_ROWS) );
}
int numberOfRows = numberOfPreviewRows;
@@ -83,7 +92,6 @@ public abstract class Parser implements IParser {
Row headerRow = null;
Integer numOfCols = null;
- int[][] typeCounts = null;
if (parseOptions.getOption(ParseOptions.OPTIONS_HEADER) != null &&
( parseOptions.getOption(ParseOptions.OPTIONS_HEADER).equals(ParseOptions.HEADER.FIRST_RECORD.toString()) ||
@@ -91,33 +99,28 @@ public abstract class Parser implements IParser {
)) {
headerRow = extractHeader();
numOfCols = headerRow.getRow().length;
- typeCounts = new int[numOfCols][ColumnDescription.DataTypes.values().length];
}
- // find data types.
-
Row r;
if (iterator().hasNext()) {
r = iterator().next();
if( null == numOfCols ) {
numOfCols = r.getRow().length;
- typeCounts = new int[numOfCols][ColumnDescription.DataTypes.values().length];
}
} else {
- throw new NoSuchElementException("No rows in the file.");
+ LOG.error("No rows found in the file. returning error.");
+ throw new NoSuchElementException("No rows in the file.");
}
while (true) {
// create Header definition from row
Object[] values = r.getRow();
-
Object[] newValues= new Object[numOfCols]; // adds null if less columns detected and removes extra columns if any
for (int colNum = 0; colNum < numOfCols; colNum++) {
if(colNum < values.length) {
// detect type
ColumnDescription.DataTypes type = ParseUtils.detectHiveDataType(values[colNum]);
- typeCounts[colNum][type.ordinal()]++;
newValues[colNum] = values[colNum];
}else{
newValues[colNum] = null;
@@ -133,13 +136,24 @@ public abstract class Parser implements IParser {
r = iterator().next();
}
- if (previewRows.size() <= 0)
+ if (previewRows.size() <= 0) {
+ LOG.error("No rows found in the file. returning error.");
throw new NoSuchElementException("Does not contain any rows.");
+ }
+
+ // find data types.
+ header = generateHeader(headerRow,previewRows,numOfCols);
+
+ return new PreviewData(header,previewRows);
+ }
+
+ private List<ColumnDescription> generateHeader(Row headerRow,List<Row> previewRows, int numOfCols) {
+ List<ColumnDescription> header = new ArrayList<>();
- header = new ArrayList<>(numOfCols);
for (int colNum = 0; colNum < numOfCols; colNum++) {
- int dataTypeId = getLikelyDataType(typeCounts, colNum);
- ColumnDescription.DataTypes type = ColumnDescription.DataTypes.values()[dataTypeId];
+ ColumnDescription.DataTypes type = getLikelyDataType(previewRows,colNum);
+ LOG.info("datatype detected for column {} : {}", colNum, type);
+
String colName = "Column" + (colNum + 1);
if (null != headerRow)
colName = (String) headerRow.getRow()[colNum];
@@ -148,6 +162,7 @@ public abstract class Parser implements IParser {
header.add(cd);
}
- return new PreviewData(header,previewRows);
+ LOG.debug("return headers : {} ", header);
+ return header;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/85798b39/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/csv/CSVParser.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/csv/CSVParser.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/csv/CSVParser.java
index 56cff58..a48041c 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/csv/CSVParser.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/csv/CSVParser.java
@@ -45,7 +45,7 @@ public class CSVParser extends Parser {
}
@Override
- public void close() throws IOException {
+ public void close() throws Exception {
this.parser.close();
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/85798b39/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/json/JSONParser.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/json/JSONParser.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/json/JSONParser.java
index 1d61798..9ca89a7 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/json/JSONParser.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/json/JSONParser.java
@@ -74,7 +74,7 @@ public class JSONParser extends Parser {
}
@Override
- public void close() throws IOException {
+ public void close() throws Exception {
this.jsonReader.close();
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/85798b39/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/xml/XMLParser.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/xml/XMLParser.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/xml/XMLParser.java
index 74bf272..4edc82c 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/xml/XMLParser.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/xml/XMLParser.java
@@ -85,7 +85,7 @@ public class XMLParser extends Parser {
}
@Override
- public void close() throws IOException {
+ public void close() throws Exception {
try {
this.xmlReader.close();
} catch (XMLStreamException e) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/85798b39/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/CSVParserTest.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/CSVParserTest.java b/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/CSVParserTest.java
index fb4c4da..c548d23 100644
--- a/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/CSVParserTest.java
+++ b/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/CSVParserTest.java
@@ -37,24 +37,14 @@ public class CSVParserTest {
* @throws IOException
*/
@Test
- public void testEmptyStream() throws IOException {
+ public void testEmptyStream() throws Exception {
String csv = "";
- StringReader sr = new StringReader(csv);
-
- CSVParser jp = null;
-
- try {
-
- jp = new CSVParser(sr, null);
-
+ try(
+ StringReader sr = new StringReader(csv);
+ CSVParser jp = new CSVParser(sr, null);
+ ) {
Assert.assertEquals("There should not be any rows.",false, jp.iterator().hasNext());
-
- }finally{
- if( null != jp )
- jp.close();
-
- sr.close();
}
}
@@ -63,38 +53,28 @@ public class CSVParserTest {
* @throws IOException
*/
@Test
- public void testEmptyRow() throws IOException {
+ public void testEmptyRow() throws Exception {
String csv = " ";
- StringReader sr = new StringReader(csv);
-
- CSVParser jp = null;
-
- try {
- jp = new CSVParser(sr, null);
+ try(
+ StringReader sr = new StringReader(csv);
+ CSVParser jp = new CSVParser(sr, null);
+ ) {
Iterator<Row> iterator = jp.iterator();
Assert.assertEquals("Iterator should be Empty", true, iterator.hasNext());
Assert.assertArrayEquals("Row should not be empty",new Object[]{" "},iterator.next().getRow());
- }finally{
- if( null != jp )
- jp.close();
-
- sr.close();
}
}
@Test
- public void testParse1Row() throws IOException {
+ public void testParse1Row() throws Exception {
String csv = "value1,c,10,10.1";
- StringReader sr = new StringReader(csv);
-
- CSVParser jp = null;
-
- try {
- jp = new CSVParser(sr, null);
-
+ try(
+ StringReader sr = new StringReader(csv);
+ CSVParser jp = new CSVParser(sr, null);
+ ) {
Iterator<Row> iterator = jp.iterator();
Assert.assertEquals("Iterator Empty!", true, iterator.hasNext());
@@ -103,26 +83,19 @@ public class CSVParserTest {
Assert.assertEquals("Row not equal!", expected, row);
Assert.assertEquals("Should report no more rows!", false, iterator.hasNext());
- }finally{
- if( null != jp )
- jp.close();
-
- sr.close();
}
}
@Test
- public void testParseMultipleRow() throws IOException {
+ public void testParseMultipleRow() throws Exception {
String csv = "value1,c,10,10.1\n" +
"value2,c2,102,true";
- StringReader sr = new StringReader(csv);
-
- CSVParser jp = null;
-
- try {
- jp = new CSVParser(sr, null);
+ try(
+ StringReader sr = new StringReader(csv);
+ CSVParser jp = new CSVParser(sr, null);
+ ) {
Iterator<Row> iterator = jp.iterator();
@@ -134,11 +107,6 @@ public class CSVParserTest {
Assert.assertEquals("Failed to detect end of rows!", false, iterator.hasNext());
Assert.assertEquals("Failed to detect end of rows 2nd time!", false, iterator.hasNext());
- }finally{
- if( null != jp )
- jp.close();
-
- sr.close();
}
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/85798b39/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/DataParserCSVTest.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/DataParserCSVTest.java b/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/DataParserCSVTest.java
index 99f758b..a367375 100644
--- a/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/DataParserCSVTest.java
+++ b/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/DataParserCSVTest.java
@@ -38,19 +38,21 @@ import java.io.StringReader;
public class DataParserCSVTest {
@Test
- public void testParsePreviewCSV() throws IOException {
+ public void testParsePreviewCSV() throws Exception {
String str = "1,a\n" +
"2,b\n" +
"3,c\n";
- StringReader sr = new StringReader(str);
+
ParseOptions parseOptions = new ParseOptions();
parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.CSV.toString());
parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.FIRST_RECORD.toString());
- DataParser dp = null;
- try {
- dp = new DataParser(sr, parseOptions);
+
+ try (
+ StringReader sr = new StringReader(str);
+ DataParser dp = new DataParser(sr, parseOptions);
+ ){
PreviewData pd = dp.parsePreview();
Assert.assertNotNull(pd.getPreviewRows());
@@ -74,11 +76,93 @@ public class DataParserCSVTest {
Assert.assertArrayEquals("Header Not Correct.", cd, pd.getHeader().toArray());
Assert.assertArrayEquals("Rows Not Correct.", rows, pd.getPreviewRows().toArray());
- } finally {
- if (null != dp)
- dp.close();
+ }
+ }
+
+ /**
+ * even if in one of the preview rows, datatype is not correct, then it should be assigned that datatype.
+ * but if first row is header then first row should not be acconted for detecting datatype
+ * @throws IOException
+ */
+ @Test
+ public void testParsePreviewDataTypeDetectionCSV() throws Exception {
+ String str = "1,a,10,k\n" +
+ "2,b,6,8\n" +
+ "2.2,b,7,9\n" +
+ "2,b,abc,1\n" +
+ "2,b,9,3\n" +
+ "2,b,8,5\n" +
+ "2,b,7,3\n" +
+ "2,b,6,3\n" +
+ "3,c,c,3\n";
+
+
+ ParseOptions parseOptions = new ParseOptions();
+ parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.CSV.toString());
+ parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.FIRST_RECORD.toString());
+
+ try(StringReader sr = new StringReader(str);
+ DataParser dp= new DataParser(sr, parseOptions)) {
- sr.close();
+ PreviewData pd = dp.parsePreview();
+ Assert.assertNotNull(pd.getHeader());
+ Assert.assertEquals(4, pd.getHeader().size());
+ ColumnDescription[] cd = {
+ // as row 3 contains 2.2
+ new ColumnDescriptionImpl("1", ColumnDescriptionShort.DataTypes.DOUBLE.toString(), 0),
+ // as all are chars
+ new ColumnDescriptionImpl("a", ColumnDescriptionShort.DataTypes.CHAR.toString(), 1),
+ // as row 4 contains abc
+ new ColumnDescriptionImpl("10", ColumnDescriptionShort.DataTypes.STRING.toString(), 2),
+ // although row 1 contains k but it is in header and not counted in detecting datatype
+ new ColumnDescriptionImpl("k", ColumnDescriptionShort.DataTypes.INT.toString(), 3)};
+
+ Assert.assertArrayEquals("Header Not Correct.", cd, pd.getHeader().toArray());
+ }
+ }
+
+ /**
+ * even if in one of the preview rows, datatype is not correct, then it should be assigned that datatype.
+ * but if first row is header then first row should not be acconted for detecting datatype
+ * @throws IOException
+ */
+ @Test
+ public void testParsePreviewDataTypeDetection2CSV() throws Exception {
+ String str = "1,a,10,k\n" +
+ "2,b,6,p\n" +
+ "2.2,b,7,9\n" +
+ "2,b,2.2,1\n" +
+ "2,b,9,3\n" +
+ "2,b,8,5\n" +
+ "2,b,7,3\n" +
+ "2,b,6,3\n" +
+ "3,c,c,3\n";
+
+
+ ParseOptions parseOptions = new ParseOptions();
+ parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.CSV.toString());
+ parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.FIRST_RECORD.toString());
+
+
+ try(StringReader sr = new StringReader(str);
+ DataParser dp = new DataParser(sr, parseOptions)) {
+
+
+ PreviewData pd = dp.parsePreview();
+ Assert.assertNotNull(pd.getHeader());
+ Assert.assertEquals(4, pd.getHeader().size());
+ ColumnDescription[] cd = {
+ // as row 3 contains 2.2
+ new ColumnDescriptionImpl("1", ColumnDescriptionShort.DataTypes.DOUBLE.toString(), 0),
+ // as all are chars
+ new ColumnDescriptionImpl("a", ColumnDescriptionShort.DataTypes.CHAR.toString(), 1),
+ // some are int, char and some double .. nothing other than 'string' satisfies all the rows
+ new ColumnDescriptionImpl("10", ColumnDescriptionShort.DataTypes.STRING.toString(), 2),
+ // although row 1 contains k but it is in header and not counted in detecting datatype
+ // but row 2 also has a char p which will be acconted for datatype detection
+ new ColumnDescriptionImpl("k", ColumnDescriptionShort.DataTypes.CHAR.toString(), 3)};
+
+ Assert.assertArrayEquals("Header Not Correct.", cd, pd.getHeader().toArray());
}
}
@@ -87,17 +171,18 @@ public class DataParserCSVTest {
* @throws IOException
*/
@Test
- public void testParsePreview1RowCSV() throws IOException {
+ public void testParsePreview1RowCSV() throws Exception {
String str = "1,a\n" ;
- StringReader sr = new StringReader(str);
ParseOptions parseOptions = new ParseOptions();
parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.CSV.toString());
parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.PROVIDED_BY_USER.toString());
- DataParser dp = null;
- try {
- dp = new DataParser(sr, parseOptions);
+ try(
+ StringReader sr = new StringReader(str);
+ DataParser dp = new DataParser(sr, parseOptions)
+ ) {
+
PreviewData pd = dp.parsePreview();
Assert.assertNotNull(pd.getPreviewRows());
@@ -116,11 +201,6 @@ public class DataParserCSVTest {
Assert.assertArrayEquals("Header Not Correct.", cd, pd.getHeader().toArray());
Assert.assertArrayEquals("Rows Not Correct.", rows, pd.getPreviewRows().toArray());
- } finally {
- if (null != dp)
- dp.close();
-
- sr.close();
}
}
@@ -129,24 +209,22 @@ public class DataParserCSVTest {
* @throws IOException
*/
@Test(expected = java.util.NoSuchElementException.class)
- public void testParsePreview1RowCSVFirstRowHeader() throws IOException {
+ public void testParsePreview1RowCSVFirstRowHeader() throws Exception {
String str = "col1,col2\n" ;
- StringReader sr = new StringReader(str);
+
ParseOptions parseOptions = new ParseOptions();
parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.CSV.toString());
parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.FIRST_RECORD.toString());
- DataParser dp = null;
- try {
- dp = new DataParser(sr, parseOptions);
- PreviewData pd = dp.parsePreview();
- } finally {
- if (null != dp)
- dp.close();
+ try(
+ StringReader sr = new StringReader(str);
+ DataParser dp = new DataParser(sr, parseOptions)
+ ) {
+
- sr.close();
+ PreviewData pd = dp.parsePreview();
}
}
@@ -157,30 +235,25 @@ public class DataParserCSVTest {
* @throws IOException
*/
@Test
- public void testParsePreviewCSVMoreColumns() throws IOException {
+ public void testParsePreviewCSVMoreColumns() throws Exception {
String str = "1,a\n" +
"2,b,x\n" + // contains 3 cols, more number of columns
"3,c\n";
- StringReader sr = new StringReader(str);
ParseOptions parseOptions = new ParseOptions();
parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.CSV.toString());
parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.FIRST_RECORD.toString());
- DataParser dp = null;
- try {
- dp = new DataParser(sr, parseOptions);
+
+ try(
+ StringReader sr = new StringReader(str);
+ DataParser dp = new DataParser(sr, parseOptions)
+ ) {
PreviewData pd = dp.parsePreview();
Row row = new Row(new Object[]{"2","b"});
Assert.assertArrayEquals("Additional columns not properly handled.", row.getRow(),pd.getPreviewRows().get(0).getRow());
- } finally {
- if (null != dp) {
- dp.close();
- }
-
- sr.close();
}
}
@@ -190,26 +263,21 @@ public class DataParserCSVTest {
* @throws IOException
*/
@Test
- public void testParsePreviewCSVLessColumns() throws IOException {
+ public void testParsePreviewCSVLessColumns() throws Exception {
String str = "1,a\n" +
"2\n" + // contains 1 col, less number of columns
"3,c\n";
- StringReader sr = new StringReader(str);
ParseOptions parseOptions = new ParseOptions();
parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.CSV.toString());
- DataParser dp = null;
- try {
- dp = new DataParser(sr, parseOptions);
+ try(
+ StringReader sr = new StringReader(str);
+ DataParser dp = new DataParser(sr, parseOptions)
+ ) {
PreviewData pd = dp.parsePreview();
Assert.assertEquals("Missing value not detected as null.",pd.getPreviewRows().get(1).getRow()[1],null);
- } finally {
- if (null != dp)
- dp.close();
-
- sr.close();
}
}
@@ -218,27 +286,21 @@ public class DataParserCSVTest {
* @throws IOException
*/
@Test
- public void testEmptyColumn() throws IOException {
+ public void testEmptyColumn() throws Exception {
String str = "1,a,x\n" +
"2,,y\n" + // contains 1 col, less number of columns
"3,c,z\n";
- StringReader sr = new StringReader(str);
-
ParseOptions parseOptions = new ParseOptions();
parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.CSV.toString());
parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.FIRST_RECORD.toString());
- DataParser dp = null;
- try {
- dp = new DataParser(sr, parseOptions);
+ try(
+ StringReader sr = new StringReader(str);
+ DataParser dp = new DataParser(sr, parseOptions)
+ ) {
PreviewData pd = dp.parsePreview();
Assert.assertEquals("Empty column not detected properly.",pd.getPreviewRows().get(0).getRow()[1],"");
- } finally {
- if (null != dp)
- dp.close();
-
- sr.close();
}
}
@@ -247,28 +309,23 @@ public class DataParserCSVTest {
* @throws IOException
*/
@Test
- public void testLastEmptyColumn() throws IOException {
+ public void testLastEmptyColumn() throws Exception {
String str = "1,a,x\n" +
"2,,\n" + // contains 1 col, less number of columns
"3,c,z\n";
- StringReader sr = new StringReader(str);
ParseOptions parseOptions = new ParseOptions();
parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.CSV.toString());
parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.FIRST_RECORD.toString());
- DataParser dp = null;
- try {
- dp = new DataParser(sr, parseOptions);
+ try(
+ StringReader sr = new StringReader(str);
+ DataParser dp = new DataParser(sr, parseOptions)
+ ) {
PreviewData pd = dp.parsePreview();
Assert.assertEquals("Empty column not detected properly.",pd.getPreviewRows().get(0).getRow()[1],"");
Assert.assertEquals("Empty column not detected properly.",pd.getPreviewRows().get(0).getRow()[2],"");
- } finally {
- if (null != dp)
- dp.close();
-
- sr.close();
}
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/85798b39/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/DataParserJSONTest.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/DataParserJSONTest.java b/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/DataParserJSONTest.java
index 79166bb..6b2f6a33 100644
--- a/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/DataParserJSONTest.java
+++ b/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/DataParserJSONTest.java
@@ -34,7 +34,7 @@ import java.io.StringReader;
public class DataParserJSONTest {
@Test
- public void testParsePreviewJSON() throws IOException {
+ public void testParsePreviewJSON() throws Exception {
String str = "[ {\"col1\" : \"a\", \n\"col2\": \"abcd\" ,\"col3\": \"abcd\" ,\"col4\": \"abcd\" ,\"col5\": \"abcd\" ,\"col6\": \"abcd\" ,\"col7\": \"abcd\" ,\"col8\": \"abcd\" ,\"col9\": \"abcd\" ,\"col10\": \"abcd\" ,\"col11\": \"abcd\" ,\"col12\" : \"abcd\" ,\"col13\" : \"abcd\" ,\"col14\" : \"4.4\" },"
+ "{\"col1\": \"b\", \n\"col2\": \"abcd\" ,\"col3\": \"abcd\" ,\"col4\": \"abcd\" ,\"col5\": \"abcd\" ,\"col6\": \"abcd\" ,\"col7\": \"abcd\" ,\"col8\": \"abcd\" ,\"col9\": \"abcd\" ,\"col10\": \"abcd\" ,\"col11\": \"abcd\" ,\"col12\" : \"abcd\" ,\"col13\" : \"abcd\" ,\"col14\" : \"5.4\" },"
+ "{\"col1\": \"c\", \n\"col2\": \"abcd\" ,\"col3\": \"abcd\" ,\"col4\": \"abcd\" ,\"col5\": \"abcd\" ,\"col6\": \"abcd\" ,\"col7\": \"abcd\" ,\"col8\": \"abcd\" ,\"col9\": \"abcd\" ,\"col10\": \"abcd\" ,\"col11\": \"abcd\" ,\"col12\" : \"abcd\" ,\"col13\" : \"abcd\" ,\"col14\" : \"6.4\" },"
@@ -55,16 +55,16 @@ public class DataParserJSONTest {
+ "{\"col1\": \"r\", \n\"col2\": \"abcd\" ,\"col3\": \"abcd\" ,\"col4\": \"abcd\" ,\"col5\": \"abcd\" ,\"col6\": \"abcd\" ,\"col7\": \"abcd\" ,\"col8\": \"abcd\" ,\"col9\": \"abcd\" ,\"col10\": \"abcd\" ,\"col11\": \"abcd\" ,\"col12\" : \"abcd\" ,\"col13\" : \"abcd\" ,\"col14\" : \"64.4\" }"
+ "]";
- StringReader sr = new StringReader(str);
ParseOptions parseOptions = new ParseOptions();
parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.JSON.toString());
parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.EMBEDDED.toString());
parseOptions.setOption(ParseOptions.OPTIONS_NUMBER_OF_PREVIEW_ROWS, 7);
- DataParser dp = null;
- try {
- dp = new DataParser(sr, parseOptions);
+ try(
+ StringReader sr = new StringReader(str);
+ DataParser dp = new DataParser(sr, parseOptions)
+ ) {
PreviewData pd = dp.parsePreview();
Assert.assertNotNull(pd.getPreviewRows());
@@ -98,11 +98,6 @@ public class DataParserJSONTest {
Assert.assertArrayEquals("Header Not Correct.", cd, pd.getHeader().toArray());
Assert.assertArrayEquals("Rows Not Correct.", rows, pd.getPreviewRows().toArray());
- } finally {
- if (null != dp)
- dp.close();
-
- sr.close();
}
}
@@ -112,31 +107,26 @@ public class DataParserJSONTest {
* @throws IOException
*/
@Test
- public void testParsePreviewCSVMoreColumns() throws IOException {
+ public void testParsePreviewCSVMoreColumns() throws Exception {
String str = "[ {\"col1\" : \"a\", \n\"col2\": \"abcd\" ,\"col3\": \"abcd\" ,\"col4\": \"abcd\" ,\"col5\": \"abcd\" ,\"col6\": \"abcd\" ,\"col7\": \"abcd\" ,\"col8\": \"abcd\" ,\"col9\": \"abcd\" ,\"col10\": \"abcd\" ,\"col11\": \"abcd\" ,\"col12\" : \"abcd\" ,\"col13\" : \"abcd\" ,\"col14\" : \"4.4\" },"
+ "{\"col1\": \"b\", \n\"col2\": \"abcd\" ,\"col3\": \"abcd\" ,\"col4\": \"abcd\" ,\"col5\": \"abcd\" ,\"col6\": \"abcd\" ,\"col7\": \"abcd\" ,\"col8\": \"abcd\" ,\"col9\": \"abcd\" ,\"col10\": \"abcd\" ,\"col11\": \"abcd\" ,\"col12\" : \"abcd\" ,\"col13\" : \"abcd\" , \"col14\" : \"43.4\" ,\"col15\" : \"asafsfa\" },"
+ "{\"col1\": \"c\", \n\"col2\": \"abcd\" ,\"col3\": \"abcd\" ,\"col4\": \"abcd\" ,\"col5\": \"abcd\" ,\"col6\": \"abcd\" ,\"col7\": \"abcd\" ,\"col8\": \"abcd\" ,\"col9\": \"abcd\" ,\"col10\": \"abcd\" ,\"col11\": \"abcd\" ,\"col12\" : \"abcd\" ,\"col13\" : \"abcd\" ,\"col14\" : \"6.4\" },"
+ "{\"col1\": \"d\", \n\"col2\": \"abcd\" ,\"col3\": \"abcd\" ,\"col4\": \"abcd\" ,\"col5\": \"abcd\" ,\"col6\": \"abcd\" ,\"col7\": \"abcd\" ,\"col8\": \"abcd\" ,\"col9\": \"abcd\" ,\"col10\": \"abcd\" ,\"col11\": \"abcd\" ,\"col12\" : \"abcd\" ,\"col13\" : \"abcd\" ,\"col14\" : \"7.4\" }"
+ "]";
- StringReader sr = new StringReader(str);
ParseOptions parseOptions = new ParseOptions();
parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.JSON.toString());
- DataParser dp = null;
- try {
- dp = new DataParser(sr, parseOptions);
+ try(
+ StringReader sr = new StringReader(str);
+ DataParser dp = new DataParser(sr, parseOptions)
+ ) {
PreviewData pd = dp.parsePreview();
Row row2 = new Row(new Object[]{"b", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "43.4"});
Assert.assertArrayEquals("More number of columns do not give correct result.", row2.getRow(), pd.getPreviewRows().get(1).getRow());
- } finally {
- if (null != dp)
- dp.close();
-
- sr.close();
}
}
@@ -146,7 +136,7 @@ public class DataParserJSONTest {
* @throws IOException
*/
@Test
- public void testParsePreviewCSVLessColumns() throws IOException {
+ public void testParsePreviewCSVLessColumns() throws Exception {
String str = "[ " +
"{\"col1\" : \"a\", \n\"col2\": \"abcd\" ,\"col3\": \"abcd\" ,\"col4\": \"abcd\" ,\"col5\": \"abcd\" ,\"col6\": \"abcd\" ,\"col7\": \"abcd\" ,\"col8\": \"abcd\" ,\"col9\": \"abcd\" ,\"col10\": \"abcd\" ,\"col11\": \"abcd\" ,\"col12\" : \"abcd\" ,\"col13\" : \"abcd\" ,\"col14\" : \"4.4\" },"
+ "{\"col1\": \"b\", \n\"col2\": \"abcd\" ,\"col3\": \"abcd\" ,\"col4\": \"abcd\" ,\"col5\": \"abcd\" ,\"col6\": \"abcd\" ,\"col7\": \"abcd\" ,\"col8\": \"abcd\" ,\"col9\": \"abcd\" ,\"col10\": \"abcd\" ,\"col11\": \"abcd\" ,\"col12\" : \"abcd\" ,\"col13\" : \"abcd\" },"
@@ -154,23 +144,17 @@ public class DataParserJSONTest {
+ "{\"col1\": \"d\", \n\"col2\": \"abcd\" ,\"col3\": \"abcd\" ,\"col4\": \"abcd\" ,\"col5\": \"abcd\" ,\"col6\": \"abcd\" ,\"col7\": \"abcd\" ,\"col8\": \"abcd\" ,\"col9\": \"abcd\" ,\"col10\": \"abcd\" ,\"col11\": \"abcd\" ,\"col12\" : \"abcd\" ,\"col13\" : \"abcd\" ,\"col14\" : \"7.4\" }"
+ "]";
- StringReader sr = new StringReader(str);
- DataParser dp = null;
- try {
- ParseOptions parseOptions = new ParseOptions();
- parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.JSON.toString());
- parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.EMBEDDED.toString());
-
- dp = new DataParser(sr, parseOptions);
+ ParseOptions parseOptions = new ParseOptions();
+ parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.JSON.toString());
+ parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.EMBEDDED.toString());
+ try(
+ StringReader sr = new StringReader(str);
+ DataParser dp = new DataParser(sr, parseOptions)
+ ) {
PreviewData pd = dp.parsePreview();
Assert.assertNull(pd.getPreviewRows().get(1).getRow()[13]);
- } finally {
- if (null != dp)
- dp.close();
-
- sr.close();
}
}
@@ -180,29 +164,22 @@ public class DataParserJSONTest {
* @throws IOException
*/
@Test(expected = IllegalArgumentException.class)
- public void testWrongJsonFormat() throws IOException {
+ public void testWrongJsonFormat() throws Exception {
String str = "[ " +
"{\"col1\" : \"a\", \n\"col2\": \"abcd\" },"
+ "{\"col1\": \"b\", \n\"col2\": \"abcd\" },"
+ "{\"col1\": \"c\", \n\"col2\": \"abcd\" },"
+ "{\"col1\": \"d\",, \n\"col2\": \"abcd\" }" // extra comma in this line
+ "]";
- DataParser dp = null;
- StringReader sr = new StringReader(str);
-
- try {
- ParseOptions parseOptions = new ParseOptions();
- parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.JSON.toString());
- parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.EMBEDDED.toString());
-
- dp = new DataParser(sr, parseOptions);
+ ParseOptions parseOptions = new ParseOptions();
+ parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.JSON.toString());
+ parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.EMBEDDED.toString());
+ try(
+ StringReader sr = new StringReader(str);
+ DataParser dp = new DataParser(sr, parseOptions);
+ ) {
PreviewData pd = dp.parsePreview();
- } finally {
- if (null != dp)
- dp.close();
-
- sr.close();
}
}
@@ -211,19 +188,19 @@ public class DataParserJSONTest {
* @throws IOException
*/
@Test
- public void testParsePreview1RowJSON() throws IOException {
+ public void testParsePreview1RowJSON() throws Exception {
String str = "[ "
+ "{\"col1\": \"d\", \n\"col2\": \"abcd\" }" // extra comma in this line
+ "]";
- StringReader sr = new StringReader(str);
ParseOptions parseOptions = new ParseOptions();
parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.JSON.toString());
parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.EMBEDDED.toString());
- DataParser dp = null;
- try {
- dp = new DataParser(sr, parseOptions);
+ try(
+ StringReader sr = new StringReader(str);
+ DataParser dp = new DataParser(sr, parseOptions);
+ ) {
PreviewData pd = dp.parsePreview();
Assert.assertNotNull(pd.getPreviewRows());
@@ -242,11 +219,6 @@ public class DataParserJSONTest {
Assert.assertArrayEquals("Header Not Correct.", cd, pd.getHeader().toArray());
Assert.assertArrayEquals("Rows Not Correct.", rows, pd.getPreviewRows().toArray());
- } finally {
- if (null != dp)
- dp.close();
-
- sr.close();
}
}
@@ -255,19 +227,19 @@ public class DataParserJSONTest {
* @throws IOException
*/
@Test
- public void testParsePreview1RowJSONHeaderProvided() throws IOException {
+ public void testParsePreview1RowJSONHeaderProvided() throws Exception {
String str = "[ "
+ "{\"col1\": \"d\", \n\"col2\": \"abcd\" }" // extra comma in this line
+ "]";
- StringReader sr = new StringReader(str);
ParseOptions parseOptions = new ParseOptions();
parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.JSON.toString());
parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.PROVIDED_BY_USER.toString());
- DataParser dp = null;
- try {
- dp = new DataParser(sr, parseOptions);
+ try(
+ StringReader sr = new StringReader(str);
+ DataParser dp = new DataParser(sr, parseOptions);
+ ) {
PreviewData pd = dp.parsePreview();
Assert.assertNotNull(pd.getPreviewRows());
@@ -286,12 +258,6 @@ public class DataParserJSONTest {
Assert.assertArrayEquals("Header Not Correct.", cd, pd.getHeader().toArray());
Assert.assertArrayEquals("Rows Not Correct.", rows, pd.getPreviewRows().toArray());
- } finally {
- if (null != dp)
- dp.close();
-
- sr.close();
}
}
-
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/85798b39/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/DataParserXMLTest.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/DataParserXMLTest.java b/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/DataParserXMLTest.java
index 4199257..e5fddc7 100644
--- a/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/DataParserXMLTest.java
+++ b/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/DataParserXMLTest.java
@@ -34,7 +34,7 @@ import java.io.StringReader;
public class DataParserXMLTest {
@Test
- public void testParsePreviewXML() throws IOException {
+ public void testParsePreviewXML() throws Exception {
String str = "<table>" +
"<row>" +
"<col name=\"col1\">row1-col1-Value</col>" +
@@ -52,15 +52,16 @@ public class DataParserXMLTest {
"</row>" +
"</table>";
- StringReader sr = new StringReader(str);
ParseOptions parseOptions = new ParseOptions();
parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.XML.toString());
parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.FIRST_RECORD.toString());
- DataParser dp = null;
- try {
- dp = new DataParser(sr, parseOptions);
+
+ try(
+ StringReader sr = new StringReader(str);
+ DataParser dp = new DataParser(sr, parseOptions);
+ ) {
PreviewData pd = dp.parsePreview();
Assert.assertNotNull(pd.getPreviewRows());
@@ -81,11 +82,6 @@ public class DataParserXMLTest {
Assert.assertArrayEquals("Header Not Correct.", cd, pd.getHeader().toArray());
Assert.assertArrayEquals("Rows Not Correct.", rows, pd.getPreviewRows().toArray());
- } finally {
- if (null != dp)
- dp.close();
-
- sr.close();
}
}
@@ -96,7 +92,7 @@ public class DataParserXMLTest {
* @throws IOException
*/
@Test
- public void testParsePreviewCSVMoreColumns() throws IOException {
+ public void testParsePreviewCSVMoreColumns() throws Exception {
String str ="<table>" +
"<row>" +
"<col name=\"col1\">row1-col1-Value</col>" +
@@ -116,24 +112,19 @@ public class DataParserXMLTest {
"</row>" +
"</table>";
- StringReader sr = new StringReader(str);
-
ParseOptions parseOptions = new ParseOptions();
parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.XML.toString());
- DataParser dp = null;
- try {
- dp = new DataParser(sr, parseOptions);
+
+ try( StringReader sr = new StringReader(str);
+ DataParser dp = new DataParser(sr, parseOptions);
+ ) {
+
PreviewData pd = dp.parsePreview();
Row row2 = new Row(new Object[]{"row2-col1-Value","row2-col2-Value","row2-col3-Value","20","21"});
Assert.assertArrayEquals("More number of columns do not give correct result.", row2.getRow(), pd.getPreviewRows().get(1).getRow());
- } finally {
- if (null != dp)
- dp.close();
-
- sr.close();
}
}
@@ -144,7 +135,7 @@ public class DataParserXMLTest {
* @throws IOException
*/
@Test
- public void testParsePreviewCSVLessColumns() throws IOException {
+ public void testParsePreviewCSVLessColumns() throws Exception {
String str = "<table>" +
"<row>" +
"<col name=\"col1\">row1-col1-Value</col>" +
@@ -164,24 +155,18 @@ public class DataParserXMLTest {
"</row>" +
"</table>";
- StringReader sr = new StringReader(str);
- DataParser dp = null;
- try {
- ParseOptions parseOptions = new ParseOptions();
- parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.XML.toString());
- parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.FIRST_RECORD.toString());
-
- dp = new DataParser(sr, parseOptions);
+ ParseOptions parseOptions = new ParseOptions();
+ parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.XML.toString());
+ parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.FIRST_RECORD.toString());
+ try(
+ StringReader sr = new StringReader(str);
+ DataParser dp = new DataParser(sr, parseOptions);
+ ) {
PreviewData pd = dp.parsePreview();
Row row2 = new Row(new Object[]{"row2-col1-Value","row2-col2-Value","row2-col3-Value",null,null,"20","21"});
Assert.assertArrayEquals("Less number of columns do not give correct result.", row2.getRow(), pd.getPreviewRows().get(1).getRow());
- } finally {
- if (null != dp)
- dp.close();
-
- sr.close();
}
}
@@ -191,7 +176,7 @@ public class DataParserXMLTest {
* @throws IOException
*/
@Test(expected = IllegalArgumentException.class)
- public void testWrongXMLFormat() throws IOException {
+ public void testWrongXMLFormat() throws Exception {
String str = "<table>" +
"<row>" +
"<ccc></ccc>" + // illegal tag.
@@ -211,22 +196,15 @@ public class DataParserXMLTest {
"<col name=\"col5\">21</col>" +
"</row>" +
"</table>";
- DataParser dp = null;
- StringReader sr = new StringReader(str);
-
- try {
- ParseOptions parseOptions = new ParseOptions();
- parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.XML.toString());
- parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.FIRST_RECORD.toString());
-
- dp = new DataParser(sr, parseOptions);
+ ParseOptions parseOptions = new ParseOptions();
+ parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.XML.toString());
+ parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.FIRST_RECORD.toString());
+ try(
+ StringReader sr = new StringReader(str);
+ DataParser dp = new DataParser(sr, parseOptions);
+ ) {
PreviewData pd = dp.parsePreview();
- } finally {
- if (null != dp)
- dp.close();
-
- sr.close();
}
}
@@ -235,22 +213,23 @@ public class DataParserXMLTest {
* @throws IOException
*/
@Test
- public void testParsePreview1RowXML() throws IOException {
+ public void testParsePreview1RowXML() throws Exception {
String str = "<table>" +
"<row>" +
"<col name=\"col1\">row1-col1-Value</col>" +
"<col name=\"col2\">11</col>" +
"</row>" +
"</table>";
- StringReader sr = new StringReader(str);
+
ParseOptions parseOptions = new ParseOptions();
parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.XML.toString());
parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.EMBEDDED.toString());
- DataParser dp = null;
- try {
- dp = new DataParser(sr, parseOptions);
+ try(
+ StringReader sr = new StringReader(str);
+ DataParser dp = new DataParser(sr, parseOptions);
+ ) {
PreviewData pd = dp.parsePreview();
Assert.assertNotNull(pd.getPreviewRows());
@@ -269,11 +248,6 @@ public class DataParserXMLTest {
Assert.assertArrayEquals("Header Not Correct.", cd, pd.getHeader().toArray());
Assert.assertArrayEquals("Rows Not Correct.", rows, pd.getPreviewRows().toArray());
- } finally {
- if (null != dp)
- dp.close();
-
- sr.close();
}
}
@@ -282,22 +256,22 @@ public class DataParserXMLTest {
* @throws IOException
*/
@Test
- public void testParsePreview1RowXMLHeaderProvided() throws IOException {
+ public void testParsePreview1RowXMLHeaderProvided() throws Exception {
String str = "<table>" +
"<row>" +
"<col name=\"col1\">row1-col1-Value</col>" +
"<col name=\"col2\">11</col>" +
"</row>" +
"</table>";
- StringReader sr = new StringReader(str);
ParseOptions parseOptions = new ParseOptions();
parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.InputFileType.XML.toString());
parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER.PROVIDED_BY_USER.toString());
- DataParser dp = null;
- try {
- dp = new DataParser(sr, parseOptions);
+ try(
+ StringReader sr = new StringReader(str);
+ DataParser dp = new DataParser(sr, parseOptions)
+ ) {
PreviewData pd = dp.parsePreview();
Assert.assertNotNull(pd.getPreviewRows());
@@ -316,11 +290,6 @@ public class DataParserXMLTest {
Assert.assertArrayEquals("Header Not Correct.", cd, pd.getHeader().toArray());
Assert.assertArrayEquals("Rows Not Correct.", rows, pd.getPreviewRows().toArray());
- } finally {
- if (null != dp)
- dp.close();
-
- sr.close();
}
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/85798b39/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/JsonParserTest.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/JsonParserTest.java b/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/JsonParserTest.java
index 38ed225..cd571b3 100644
--- a/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/JsonParserTest.java
+++ b/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/JsonParserTest.java
@@ -33,77 +33,53 @@ import java.util.Iterator;
public class JsonParserTest {
@Test(expected = IOException.class)
- public void testEmptyStream() throws IOException {
+ public void testEmptyStream() throws Exception {
String json = "";
- StringReader sr = new StringReader(json);
-
- JSONParser jp = null;
-
- try {
-
- jp = new JSONParser(sr, null);
-
- }finally{
- if( null != jp )
- jp.close();
-
- sr.close();
+ try(
+ StringReader sr = new StringReader(json);
+ JSONParser jp = new JSONParser(sr, null);
+ ) {
+ // PARSING WILL THROW ERROR
}
}
@Test
- public void testEmptyRow() throws IOException {
+ public void testEmptyRow() throws Exception {
JsonObject jo = new JsonObject();
JsonArray ja = new JsonArray();
ja.add(jo);
String json = ja.toString();
- StringReader sr = new StringReader(json);
-
- JSONParser jp = null;
-
- try {
- jp = new JSONParser(sr, null);
+ try(
+ StringReader sr = new StringReader(json);
+ JSONParser jp = new JSONParser(sr, null)
+ ) {
Iterator<Row> iterator = jp.iterator();
Assert.assertEquals("Iterator should not be Empty", true, iterator.hasNext());
Assert.assertArrayEquals("Row should be empty",new Object[]{},iterator.next().getRow());
- }finally{
- if( null != jp )
- jp.close();
-
- sr.close();
}
}
@Test
- public void testEmptyTable() throws IOException {
+ public void testEmptyTable() throws Exception {
JsonArray ja = new JsonArray();
String json = ja.toString();
- StringReader sr = new StringReader(json);
-
- JSONParser jp = null;
-
- try {
- jp = new JSONParser(sr, null);
-
+ try(
+ StringReader sr = new StringReader(json);
+ JSONParser jp = new JSONParser(sr, null);
+ ) {
Iterator<Row> iterator = jp.iterator();
-
Assert.assertEquals("Iterator Empty!", false, iterator.hasNext());
- }finally{
- if( null != jp )
- jp.close();
-
- sr.close();
}
}
@Test
- public void testParse1Row() throws IOException {
+ public void testParse1Row() throws Exception {
JsonObject jo = new JsonObject();
jo.addProperty("key1","value1");
jo.addProperty("key2",'c');
@@ -114,13 +90,10 @@ public class JsonParserTest {
ja.add(jo);
String json = ja.toString();
- StringReader sr = new StringReader(json);
-
- JSONParser jp = null;
-
- try {
- jp = new JSONParser(sr, null);
+ try(StringReader sr = new StringReader(json);
+ JSONParser jp = new JSONParser(sr, null)
+ ) {
Iterator<Row> iterator = jp.iterator();
Assert.assertEquals("Iterator Empty!", true, iterator.hasNext());
@@ -129,16 +102,11 @@ public class JsonParserTest {
Assert.assertEquals("Row not equal!", expected, row);
Assert.assertEquals("Should report no more rows!", false, iterator.hasNext());
- }finally{
- if( null != jp )
- jp.close();
-
- sr.close();
}
}
@Test
- public void testParseMultipleRow() throws IOException {
+ public void testParseMultipleRow() throws Exception {
JsonObject jo1 = new JsonObject();
jo1.addProperty("key1","value1");
jo1.addProperty("key2","c");
@@ -157,13 +125,13 @@ public class JsonParserTest {
ja.add(jo2);
String json = ja.toString();
- StringReader sr = new StringReader(json);
- JSONParser jp = null;
- try {
- jp = new JSONParser(sr, null);
+ try(
+ StringReader sr = new StringReader(json);
+ JSONParser jp = new JSONParser(sr, null)
+ ) {
Iterator<Row> iterator = jp.iterator();
Assert.assertEquals("Failed to detect 1st row!", true, iterator.hasNext());
@@ -174,11 +142,6 @@ public class JsonParserTest {
Assert.assertEquals("Failed to detect end of rows!", false, iterator.hasNext());
Assert.assertEquals("Failed to detect end of rows 2nd time!", false, iterator.hasNext());
- }finally{
- if( null != jp )
- jp.close();
-
- sr.close();
}
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/85798b39/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/XMLParserTest.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/XMLParserTest.java b/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/XMLParserTest.java
index d048a62..1bdf031 100644
--- a/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/XMLParserTest.java
+++ b/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/XMLParserTest.java
@@ -33,72 +33,48 @@ import java.util.Iterator;
public class XMLParserTest {
@Test(expected = IOException.class)
- public void testEmptyStream() throws IOException {
+ public void testEmptyStream() throws Exception {
String xml = "";
- StringReader sr = new StringReader(xml);
-
- XMLParser jp = null;
-
- try {
-
- jp = new XMLParser(sr, null);
-
- }finally{
- if( null != jp )
- jp.close();
-
- sr.close();
+ try(
+ StringReader sr = new StringReader(xml);
+ XMLParser jp = new XMLParser(sr, null);
+ ) {
+ // creation of XMLParser will throw exception.
}
}
@Test
- public void testEmptyRow() throws IOException {
+ public void testEmptyRow() throws Exception {
String xml = "<table><row></row></table>";
-
- StringReader sr = new StringReader(xml);
-
- XMLParser jp = null;
-
- try {
- jp = new XMLParser(sr, null);
-
+ try(
+ StringReader sr = new StringReader(xml);
+ XMLParser jp = new XMLParser(sr, null);
+ ) {
Iterator<Row> iterator = jp.iterator();
Assert.assertEquals("Iterator should not be Empty", true, iterator.hasNext());
Assert.assertArrayEquals("Row should be empty",new Object[]{},iterator.next().getRow());
- }finally{
- if( null != jp )
- jp.close();
-
- sr.close();
}
}
@Test
- public void testEmptyTable() throws IOException {
+ public void testEmptyTable() throws Exception {
String xml = "<table></table>";
- StringReader sr = new StringReader(xml);
-
- XMLParser jp = null;
-
- try {
- jp = new XMLParser(sr, null);
+ try(
+ StringReader sr = new StringReader(xml);
+ XMLParser jp = new XMLParser(sr, null);
+ ) {
Iterator<Row> iterator = jp.iterator();
Assert.assertEquals("Iterator Empty!", false, iterator.hasNext());
- }finally{
- if( null != jp )
- jp.close();
-
- sr.close();
}
}
@Test
- public void testParse1Row() throws IOException {
+ public void testParse1Row() throws Exception {
String xml =
"<table>"
@@ -110,13 +86,10 @@ public class XMLParserTest {
+ "</row>"
+ "</table>" ;
- StringReader sr = new StringReader(xml);
-
- XMLParser jp = null;
-
- try {
- jp = new XMLParser(sr, null);
-
+ try(
+ StringReader sr = new StringReader(xml);
+ XMLParser jp = new XMLParser(sr, null)
+ ) {
Iterator<Row> iterator = jp.iterator();
Assert.assertEquals("Iterator Empty!", true, iterator.hasNext());
@@ -125,16 +98,11 @@ public class XMLParserTest {
Assert.assertEquals("Row not equal!", expected, row);
Assert.assertEquals("Should report no more rows!", false, iterator.hasNext());
- }finally{
- if( null != jp )
- jp.close();
-
- sr.close();
}
}
@Test
- public void testParseMultipleRow() throws IOException {
+ public void testParseMultipleRow() throws Exception {
String xml =
"<table>"
+ "<row>"
@@ -151,15 +119,10 @@ public class XMLParserTest {
+ "</row>"
+ "</table>" ;
-
-
- StringReader sr = new StringReader(xml);
-
- XMLParser jp = null;
-
- try {
- jp = new XMLParser(sr, null);
-
+ try(
+ StringReader sr = new StringReader(xml);
+ XMLParser jp = new XMLParser(sr, null)
+ ) {
Iterator<Row> iterator = jp.iterator();
Assert.assertEquals("Failed to detect 1st row!", true, iterator.hasNext());
@@ -170,11 +133,6 @@ public class XMLParserTest {
Assert.assertEquals("Failed to detect end of rows!", false, iterator.hasNext());
Assert.assertEquals("Failed to detect end of rows 2nd time!", false, iterator.hasNext());
- }finally{
- if( null != jp )
- jp.close();
-
- sr.close();
}
}
}