You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by yu...@apache.org on 2015/11/07 01:50:44 UTC
ambari git commit: AMBARI-13747. Hive View : Add Database/Table
creation from File. (Nitiraj Singh Rathore via yusaku)
Repository: ambari
Updated Branches:
refs/heads/trunk 810a32abb -> 079293935
AMBARI-13747. Hive View : Add Database/Table creation from File. (Nitiraj Singh Rathore via yusaku)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/07929393
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/07929393
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/07929393
Branch: refs/heads/trunk
Commit: 079293935aed3650f5240c1b2dc2975d2875b82f
Parents: 810a32a
Author: Yusaku Sako <yu...@hortonworks.com>
Authored: Fri Nov 6 16:50:15 2015 -0800
Committer: Yusaku Sako <yu...@hortonworks.com>
Committed: Fri Nov 6 16:50:15 2015 -0800
----------------------------------------------------------------------
.../view/hive/client/ColumnDescription.java | 18 ++
.../view/hive/client/ConnectionFactory.java | 19 +-
.../org/apache/ambari/view/hive/client/Row.java | 24 ++
.../view/hive/resources/uploads/CSVParser.java | 185 ++++++++++++
.../uploads/ColumnDescriptionImpl.java | 95 ++++++
.../view/hive/resources/uploads/DataParser.java | 63 ++++
.../view/hive/resources/uploads/IParser.java | 37 +++
.../hive/resources/uploads/ParseOptions.java | 46 +++
.../view/hive/resources/uploads/ParseUtils.java | 103 +++++++
.../hive/resources/uploads/QueryGenerator.java | 66 +++++
.../view/hive/resources/uploads/TableInfo.java | 62 ++++
.../hive/resources/uploads/UploadService.java | 296 +++++++++++++++++++
.../ui/hive-web/app/adapters/file-upload.js | 31 ++
.../ui/hive-web/app/adapters/upload-table.js | 76 +++++
.../ui/hive-web/app/components/file-upload.js | 25 ++
.../ui/hive-web/app/components/navbar-widget.js | 5 +-
.../ui/hive-web/app/controllers/upload-table.js | 261 ++++++++++++++++
.../ui/hive-web/app/initializers/i18n.js | 7 +-
.../main/resources/ui/hive-web/app/router.js | 1 +
.../resources/ui/hive-web/app/styles/app.scss | 3 +-
.../ui/hive-web/app/templates/upload-table.hbs | 92 ++++++
.../ui/hive-web/app/utils/constants.js | 1 +
.../src/main/resources/ui/hive-web/bower.json | 1 +
.../src/main/resources/ui/hive-web/package.json | 1 +
contrib/views/hive/src/main/resources/view.xml | 6 +
.../hive/resources/upload/DataParserTest.java | 65 ++++
26 files changed, 1584 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/ColumnDescription.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/ColumnDescription.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/ColumnDescription.java
index d7ea560..a25571f 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/ColumnDescription.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/ColumnDescription.java
@@ -19,6 +19,24 @@
package org.apache.ambari.view.hive.client;
public interface ColumnDescription {
+ enum DataTypes {
+ TINYINT, //
+ SMALLINT, //
+ INT, //
+ BIGINT, //
+ BOOLEAN, //
+ FLOAT, //
+ DOUBLE, //
+ STRING, //
+ BINARY, // -- (Note: Available in Hive 0.8.0 and later)
+ TIMESTAMP, // -- (Note: Available in Hive 0.8.0 and later)
+ DECIMAL, // -- (Note: Available in Hive 0.11.0 and later)
+ // DECIMAL,(precision, scale) -- (Note: Available in Hive 0.13.0 and later) Not included.
+ DATE, // -- (Note: Available in Hive 0.12.0 and later)
+ VARCHAR, // -- (Note: Available in Hive 0.12.0 and later)
+ CHAR, // -- (Note: Available in Hive 0.13.0 and later)
+ }
+
public abstract String getName();
public abstract void setName(String name);
http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/ConnectionFactory.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/ConnectionFactory.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/ConnectionFactory.java
index 82ac1f5..1442748 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/ConnectionFactory.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/ConnectionFactory.java
@@ -24,10 +24,11 @@ import org.apache.ambari.view.hive.utils.ServiceFormattedException;
import org.apache.ambari.view.utils.UserLocalFactory;
import org.apache.ambari.view.utils.ambari.AmbariApi;
import org.apache.ambari.view.utils.ambari.AmbariApiException;
+import org.apache.ambari.view.utils.hdfs.HdfsApi;
+import org.apache.ambari.view.utils.hdfs.HdfsUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -38,6 +39,7 @@ public class ConnectionFactory implements UserLocalFactory<Connection> {
private ViewContext context;
private HiveAuthCredentials credentials;
private AmbariApi ambariApi;
+ private HdfsApi hdfsApi = null;
public ConnectionFactory(ViewContext context, HiveAuthCredentials credentials) {
this.context = context;
@@ -45,6 +47,21 @@ public class ConnectionFactory implements UserLocalFactory<Connection> {
this.ambariApi = new AmbariApi(context);
}
+ /**
+ * Get HdfsApi instance
+ * @return HdfsApi business delegate
+ */
+ public synchronized HdfsApi getHDFSApi() {
+ if (hdfsApi == null) {
+ try {
+ hdfsApi = HdfsUtil.connectToHDFSApi(context);
+ } catch (Exception ex) {
+ throw new ServiceFormattedException("HdfsApi connection failed. Check \"webhdfs.url\" property", ex);
+ }
+ }
+ return hdfsApi;
+ }
+
@Override
public Connection create() {
try {
http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/Row.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/Row.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/Row.java
index 306530a..35f216b 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/Row.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/Row.java
@@ -18,6 +18,7 @@
package org.apache.ambari.view.hive.client;
+import java.util.Arrays;
import java.util.HashSet;
public class Row {
@@ -47,4 +48,27 @@ public class Row {
public void setRow(Object[] row) {
this.row = row;
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ Row row1 = (Row) o;
+
+ // Probably incorrect - comparing Object[] arrays with Arrays.equals
+ return Arrays.equals(row, row1.row);
+ }
+
+ @Override
+ public int hashCode() {
+ return Arrays.hashCode(row);
+ }
+
+ @Override
+ public String toString() {
+ return "Row{" +
+ "row=" + Arrays.toString(row) +
+ '}';
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/CSVParser.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/CSVParser.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/CSVParser.java
new file mode 100644
index 0000000..388cf53
--- /dev/null
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/CSVParser.java
@@ -0,0 +1,185 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.view.hive.resources.uploads;
+
+import org.apache.ambari.view.hive.client.ColumnDescription;
+import org.apache.ambari.view.hive.client.ColumnDescriptionShort;
+import org.apache.ambari.view.hive.client.Row;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVRecord;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * Parses the given Reader and extracts headers and rows, and detect datatypes of columns
+ */
+public class CSVParser implements IParser {
+
+ static class CSVIterator implements Iterator<Row> {
+
+ private Iterator<CSVRecord> iterator;
+
+ public CSVIterator(Iterator<CSVRecord> iterator) {
+ this.iterator = iterator;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return iterator.hasNext();
+ }
+
+ @Override
+ public Row next() {
+ CSVRecord row = iterator.next();
+ Object[] values = new Object[row.size()];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = row.get(i);
+ }
+ Row r = new Row(values);
+ return r;
+ }
+
+ @Override
+ public void remove() {
+ this.iterator.remove();
+ }
+ }
+
+ private Reader originalReader; // same as CSV reader in this case
+ private ParseOptions parseOptions;
+ private CSVIterator iterator;
+ private List<Row> previewRows;
+ private List<ColumnDescription> header;
+ private boolean isHeaderFirstRow = false;
+ private int numberOfPreviewRows = 10;
+ private org.apache.commons.csv.CSVParser parser;
+
+ public CSVParser(Reader reader, ParseOptions parseOptions) throws IOException {
+ this.originalReader = reader;
+ this.parseOptions = parseOptions;
+ // always create without headers
+ parser = new org.apache.commons.csv.CSVParser(reader, CSVFormat.EXCEL);
+ iterator = new CSVIterator(parser.iterator());
+ }
+
+ public void parsePreview() {
+ try {
+ numberOfPreviewRows = (Integer) parseOptions.getOption(ParseOptions.OPTIONS_NUMBER_OF_PREVIEW_ROWS);
+ } catch (Exception e) {
+ }
+
+ int numberOfRows = numberOfPreviewRows;
+ previewRows = new ArrayList<Row>(numberOfPreviewRows); // size including the header.
+
+ Row headerRow = null;
+ if (parseOptions.getOption(ParseOptions.OPTIONS_HEADER).equals(ParseOptions.HEADER_FIRST_RECORD)) {
+ if (!this.iterator().hasNext()) {
+ throw new NoSuchElementException("Cannot parse Header");
+ }
+ isHeaderFirstRow = true;
+ headerRow = iterator().next();
+ previewRows.add(headerRow);
+ }
+
+ // find data types.
+ int[][] typeCounts = null;
+ Row r = null;
+ int numOfCols = 0;
+ if (iterator().hasNext()) {
+ r = iterator().next();
+ numOfCols = r.getRow().length;
+ typeCounts = new int[numOfCols][ColumnDescription.DataTypes.values().length];
+ } else {
+ throw new NoSuchElementException("No rows in the csv.");
+ }
+
+ while (true) {
+ // create Header definition from row
+ Object[] values = r.getRow();
+ previewRows.add(r);
+
+ if (values.length != numOfCols)
+ throw new IllegalArgumentException("Illegal number of cols for row : " + r);
+
+ for (int colNum = 0; colNum < values.length; colNum++) {
+ // detect type
+ ColumnDescription.DataTypes type = ParseUtils.detectHiveDataType(values[colNum]);
+ typeCounts[colNum][type.ordinal()]++;
+ }
+ numberOfRows--;
+ if (numberOfRows <= 0 || !iterator().hasNext())
+ break;
+
+ r = iterator().next();
+ }
+ ;
+
+ if (previewRows.size() <= 0)
+ throw new NoSuchElementException("Does not contain any rows.");
+
+ header = new ArrayList<ColumnDescription>(numOfCols);
+ for (int colNum = 0; colNum < numOfCols; colNum++) {
+ int dataTypeId = getLikelyDataType(typeCounts, colNum);
+ ColumnDescription.DataTypes type = ColumnDescription.DataTypes.values()[dataTypeId];
+ String colName = "Column" + colNum;
+ if (null != headerRow)
+ colName = (String) headerRow.getRow()[colNum];
+
+ ColumnDescription cd = new ColumnDescriptionImpl(colName, type.toString(), colNum);
+ header.add(cd);
+ }
+ }
+
+ /**
+ * returns which datatype was detected for the maximum number of times in the given column
+ * @param typeCounts
+ * @param colNum
+ * @return
+ */
+ private int getLikelyDataType(int[][] typeCounts, int colNum) {
+ int[] colArray = typeCounts[colNum];
+ int maxIndex = 0;
+ int i = 1;
+ for (; i < colArray.length; i++) {
+ if (colArray[i] > colArray[maxIndex])
+ maxIndex = i;
+ }
+
+ return maxIndex;
+ }
+
+ @Override
+ public Reader getCSVReader() {
+ return originalReader;
+ }
+
+ @Override
+ public List<ColumnDescription> getHeader() {
+ return header;
+ }
+
+ @Override
+ public List<Row> getPreviewRows() {
+ return this.previewRows;
+ }
+
+ public Iterator<Row> iterator() {
+ return iterator; // only one iterator per parser.
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/ColumnDescriptionImpl.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/ColumnDescriptionImpl.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/ColumnDescriptionImpl.java
new file mode 100644
index 0000000..50f5036
--- /dev/null
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/ColumnDescriptionImpl.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.hive.resources.uploads;
+
+import org.apache.ambari.view.hive.client.ColumnDescription;
+
+import java.io.Serializable;
+
+public class ColumnDescriptionImpl implements ColumnDescription, Serializable {
+ private String name;
+ private String type;
+ private int position;
+
+ public ColumnDescriptionImpl(String name, String type, int position) {
+ this.name = name;
+ this.type = type;
+ this.position = position;
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public String getType() {
+ return type;
+ }
+
+ @Override
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ @Override
+ public int getPosition() {
+ return this.position;
+ }
+
+ @Override
+ public void setPosition(int position) {
+ this.position = position;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ ColumnDescriptionImpl that = (ColumnDescriptionImpl) o;
+
+ if (position != that.position) return false;
+ if (!name.equals(that.name)) return false;
+ return type.equals(that.type);
+
+ }
+
+ @Override
+ public int hashCode() {
+ int result = name.hashCode();
+ result = 31 * result + type.hashCode();
+ result = 31 * result + position;
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return new StringBuilder().append("ColumnDescriptionImpl[")
+ .append("name : ").append(name)
+ .append("type : " + type)
+ .append("position : " + position)
+ .append("]").toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/DataParser.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/DataParser.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/DataParser.java
new file mode 100644
index 0000000..5f2db55
--- /dev/null
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/DataParser.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.hive.resources.uploads;
+
+import org.apache.ambari.view.hive.client.ColumnDescription;
+import org.apache.ambari.view.hive.client.Row;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.util.Iterator;
+import java.util.List;
+
+public class DataParser implements IParser {
+
+ private IParser parser;
+
+ public DataParser(Reader reader, ParseOptions parseOptions) throws IOException {
+ if (parseOptions.getOption(ParseOptions.OPTIONS_FILE_TYPE).equals(ParseOptions.FILE_TYPE_CSV)) {
+ parser = new CSVParser(reader, parseOptions);
+ }
+ }
+
+ @Override
+ public Reader getCSVReader() {
+ return parser.getCSVReader();
+ }
+
+ @Override
+ public List<ColumnDescription> getHeader() {
+ return parser.getHeader();
+ }
+
+ @Override
+ public List<Row> getPreviewRows() {
+ return parser.getPreviewRows();
+ }
+
+ @Override
+ public void parsePreview() {
+ parser.parsePreview();
+ }
+
+ @Override
+ public Iterator<Row> iterator() {
+ return parser.iterator();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/IParser.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/IParser.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/IParser.java
new file mode 100644
index 0000000..c478b70
--- /dev/null
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/IParser.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.hive.resources.uploads;
+
+import org.apache.ambari.view.hive.client.ColumnDescription;
+import org.apache.ambari.view.hive.client.Row;
+
+import java.io.File;
+import java.io.InputStream;
+import java.io.Reader;
+import java.util.List;
+
+public interface IParser extends Iterable<Row> {
+ public Reader getCSVReader();
+
+ public List<ColumnDescription> getHeader();
+
+ public List<Row> getPreviewRows();
+
+ public void parsePreview();
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/ParseOptions.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/ParseOptions.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/ParseOptions.java
new file mode 100644
index 0000000..2ec3b1b
--- /dev/null
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/ParseOptions.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.hive.resources.uploads;
+
+import java.util.HashMap;
+
+public class ParseOptions {
+ final public static String OPTIONS_FILE_TYPE = "FILE_TYPE";
+ final public static String OPTIONS_HEADER = "HEADER";
+ final public static String OPTIONS_NUMBER_OF_PREVIEW_ROWS = "NUMBER_OF_PREVIEW_ROWS";
+
+ final public static String FILE_TYPE_CSV = "CSV";
+ final public static String FILE_TYPE_JSON = "JSON";
+ final public static String XML = "XML";
+
+ final public static String HEADER_FIRST_RECORD = "FIRST_RECORD";
+ final public static String HEADER_PROVIDED_BY_USER = "PROVIDED_BY_USER";
+
+ final public static String HEADERS = "HEADERS";
+
+ private HashMap<String, Object> options = new HashMap<String, Object>();
+
+ public void setOption(String key, Object value) {
+ this.options.put(key, value);
+ }
+
+ public Object getOption(String key) {
+ return this.options.get(key);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/ParseUtils.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/ParseUtils.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/ParseUtils.java
new file mode 100644
index 0000000..aea370e
--- /dev/null
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/ParseUtils.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.hive.resources.uploads;
+
+import org.apache.ambari.view.hive.client.ColumnDescription;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+public class ParseUtils {
+
+ final public static String[] DATE_FORMATS = {"mm/dd/yyyy", "dd/mm/yyyy", "mm-dd-yyyy" /*add more formatss*/};
+
+ public static boolean isInteger(Object object) {
+ if (object == null)
+ return false;
+
+ if (object instanceof Integer)
+ return true;
+
+ try {
+ Integer i = Integer.parseInt(object.toString());
+ return true;
+ } catch (NumberFormatException nfe) {
+ return false;
+ }
+ }
+
+ public static boolean isDouble(Object object) {
+ if (object == null)
+ return false;
+
+ if (object instanceof Double)
+ return true;
+
+ try {
+ Double i = Double.parseDouble(object.toString());
+ return true;
+ } catch (NumberFormatException nfe) {
+ return false;
+ }
+ }
+
+ public static boolean isChar(Object object) {
+ if (object == null)
+ return false;
+
+ if (object instanceof Character)
+ return true;
+
+ String str = object.toString().trim();
+ if (str.length() == 1)
+ return true;
+
+ return false;
+ }
+
+ public static boolean isDate(Object object) {
+ if (object == null)
+ return false;
+
+ if (object instanceof Date)
+ return true;
+
+ String str = object.toString();
+ for (String format : DATE_FORMATS) {
+ try {
+ Date i = new SimpleDateFormat(format).parse(str);
+ return true;
+ } catch (Exception e) {
+ }
+ }
+
+ return false;
+ }
+
+ public static ColumnDescription.DataTypes detectHiveDataType(Object object) {
+ // detect Integer
+ if (isInteger(object)) return ColumnDescription.DataTypes.INT;
+ if (isDouble(object)) return ColumnDescription.DataTypes.DOUBLE;
+ if (isDate(object)) return ColumnDescription.DataTypes.DATE;
+ if (isChar(object)) return ColumnDescription.DataTypes.CHAR;
+
+ return ColumnDescription.DataTypes.STRING;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/QueryGenerator.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/QueryGenerator.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/QueryGenerator.java
new file mode 100644
index 0000000..98616cf
--- /dev/null
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/QueryGenerator.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.hive.resources.uploads;
+
+import org.apache.ambari.view.hive.client.ColumnDescription;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
+/**
+ * generates the sql query from given data
+ */
+public class QueryGenerator {
+ protected final static Logger LOG =
+ LoggerFactory.getLogger(QueryGenerator.class);
+
+ public String generateCreateQuery(TableInfo tableInfo) {
+ String tableName = tableInfo.getTableName();
+ List<ColumnDescription> cdList = tableInfo.getColumns();
+
+ StringBuilder query = new StringBuilder();
+ query.append("create table " + tableName + " (");
+ Collections.sort(cdList, new Comparator<ColumnDescription>() {
+ @Override
+ public int compare(ColumnDescription o1, ColumnDescription o2) {
+ return o1.getPosition() - o2.getPosition();
+ }
+ });
+
+ boolean first = true;
+ for (ColumnDescription cd : cdList) {
+ if (first) {
+ first = false;
+ } else {
+ query.append(", ");
+ }
+
+ query.append(cd.getName() + " " + cd.getType());
+ }
+
+ query.append(") ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE;");
+
+ String queryString = query.toString();
+ LOG.info("Query : %S", queryString);
+ return queryString;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/TableInfo.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/TableInfo.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/TableInfo.java
new file mode 100644
index 0000000..ed4943d
--- /dev/null
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/TableInfo.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.hive.resources.uploads;
+
+import org.apache.ambari.view.hive.client.ColumnDescription;
+
+import java.util.List;
+
+public class TableInfo {
+ private String tableName;
+ private String databaseName;
+ private List<ColumnDescription> columns;
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public void setTableName(String tableName) {
+ this.tableName = tableName;
+ }
+
+ public String getDatabaseName() {
+ return databaseName;
+ }
+
+ public void setDatabaseName(String databaseName) {
+ this.databaseName = databaseName;
+ }
+
+ public List<ColumnDescription> getColumns() {
+ return columns;
+ }
+
+ public void setColumns(List<ColumnDescription> columns) {
+ this.columns = columns;
+ }
+
+ public TableInfo(String databaseName, String tableName, List<ColumnDescription> columns) {
+ this.tableName = tableName;
+ this.databaseName = databaseName;
+ this.columns = columns;
+ }
+
+ public TableInfo() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/UploadService.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/UploadService.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/UploadService.java
new file mode 100644
index 0000000..2098a5f
--- /dev/null
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/UploadService.java
@@ -0,0 +1,296 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.hive.resources.uploads;
+
+import com.sun.jersey.core.header.FormDataContentDisposition;
+import com.sun.jersey.multipart.FormDataParam;
+import org.apache.ambari.view.hive.BaseService;
+import org.apache.ambari.view.hive.client.ColumnDescription;
+import org.apache.ambari.view.hive.client.HiveClientException;
+import org.apache.ambari.view.hive.persistence.utils.ItemNotFound;
+import org.apache.ambari.view.hive.resources.jobs.NoOperationStatusSetException;
+import org.apache.ambari.view.hive.resources.jobs.viewJobs.Job;
+import org.apache.ambari.view.hive.resources.jobs.viewJobs.JobController;
+import org.apache.ambari.view.hive.resources.jobs.viewJobs.JobImpl;
+import org.apache.ambari.view.hive.resources.jobs.viewJobs.JobResourceManager;
+import org.apache.ambari.view.hive.utils.ServiceFormattedException;
+import org.apache.ambari.view.hive.utils.SharedObjectsFactory;
+import org.apache.ambari.view.utils.ambari.AmbariApi;
+import org.apache.commons.io.input.ReaderInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+import org.json.simple.JSONValue;
+
+import javax.ws.rs.*;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import java.io.*;
+import java.lang.reflect.InvocationTargetException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Servlet for queries
+ * API:
+ * POST /preview
+ * POST /upload
+ * POST /createTable
+ * GET /createTable/status
+ */
+public class UploadService extends BaseService {
+
+ private AmbariApi ambariApi;
+
+ protected JobResourceManager resourceManager;
+
+ final private String HIVE_META_STORE_LOCATION_KEY = "hive.metastore.warehouse.dir";
+ final private String HIVE_SITE = "hive-site";
+
+ @POST
+ @Path("/preview")
+ @Consumes(MediaType.MULTIPART_FORM_DATA)
+ public Response uploadForPreview(
+ @FormDataParam("file") InputStream uploadedInputStream,
+ @FormDataParam("file") FormDataContentDisposition fileDetail) {
+
+ ParseOptions parseOptions = new ParseOptions();
+ parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.FILE_TYPE_CSV);
+ parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER_FIRST_RECORD);
+
+ try {
+ DataParser dataParser = new DataParser(new InputStreamReader(uploadedInputStream), parseOptions);
+
+ dataParser.parsePreview();
+
+ Map<String, Object> retData = new HashMap<String, Object>();
+ retData.put("header", dataParser.getHeader());
+ retData.put("rows", dataParser.getPreviewRows());
+ retData.put("isFirstRowHeader", true);
+
+ JSONObject jsonObject = new JSONObject(retData);
+ return Response.ok(jsonObject).build();
+ } catch (IOException e) {
+ throw new ServiceFormattedException(e.getMessage(), e);
+ }
+ }
+
+ public static class TableInput {
+ public Boolean isFirstRowHeader;
+ public String header;
+ public String tableName;
+ public String databaseName;
+
+ public TableInput() {
+ }
+
+ public Boolean getIsFirstRowHeader() {
+ return isFirstRowHeader;
+ }
+
+ public void setIsFirstRowHeader(Boolean isFirstRowHeader) {
+ this.isFirstRowHeader = isFirstRowHeader;
+ }
+
+ public String getHeader() {
+ return header;
+ }
+
+ public void setHeader(String header) {
+ this.header = header;
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public void setTableName(String tableName) {
+ this.tableName = tableName;
+ }
+
+ public String getDatabaseName() {
+ return databaseName;
+ }
+
+ public void setDatabaseName(String databaseName) {
+ this.databaseName = databaseName;
+ }
+ }
+
+ @Path("/createTable")
+ @POST
+ @Consumes(MediaType.APPLICATION_JSON)
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response createTable(TableInput tableInput) throws IllegalAccessException, InvocationTargetException, ItemNotFound, NoSuchMethodException {
+ String header = tableInput.getHeader();
+ String databaseName = tableInput.getDatabaseName();
+ String tableName = tableInput.getTableName();
+ Boolean isFirstRowHeader = (Boolean) tableInput.getIsFirstRowHeader();
+
+ Object headerObj = JSONValue.parse(header);
+ JSONArray array = (JSONArray) headerObj;
+ List<ColumnDescription> cdList = new ArrayList<ColumnDescription>(array.size());
+ for (Object o : array) {
+ JSONObject jo = (JSONObject) o;
+ String name = (String) jo.get("name");
+ String type = (String) jo.get("type");
+ Long p = (Long) jo.get("position");
+ Integer position = p != null ? p.intValue() : 0;
+
+ ColumnDescriptionImpl cdi = new ColumnDescriptionImpl(name, type, position);
+ cdList.add(cdi);
+ }
+
+ Map jobInfo = new HashMap<String, String>();//PropertyUtils.describe(request.job);
+ jobInfo.put("title", "Internal Table Creation");
+ jobInfo.put("forcedContent", generateCreateQuery(databaseName, tableName, cdList));
+ jobInfo.put("dataBase", databaseName);
+
+ LOG.info("jobInfo : " + jobInfo);
+ Job job = new JobImpl(jobInfo);
+ LOG.info("job : " + job);
+ getResourceManager().create(job);
+
+ JobController createdJobController = getResourceManager().readController(job.getId());
+ createdJobController.submit();
+ getResourceManager().saveIfModified(createdJobController);
+
+ String filePath = databaseName + ".db/" + tableName + "/" + tableName + ".csv";
+
+ JSONObject jobObject = new JSONObject();
+ jobObject.put("jobId", job.getId());
+ jobObject.put("filePath", filePath);
+
+ LOG.info("Create table query submitted : file should be uploaded at location : {}", filePath);
+ return Response.ok(jobObject).status(201).build();
+ }
+
+ @Path("/createTable/status")
+ @GET
+ @Consumes(MediaType.APPLICATION_JSON)
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response isTableCreated(@QueryParam("jobId") int jobId) throws IllegalAccessException, NoSuchMethodException, InvocationTargetException, ItemNotFound, HiveClientException, NoOperationStatusSetException {
+ JobController jobController = getResourceManager().readController(jobId + "");
+ LOG.info("jobController.getStatus().status : {} for job : {}", jobController.getStatus().status, jobController.getJob().getId());
+ JSONObject jsonObject = new JSONObject();
+ jsonObject.put("status", jobController.getStatus().status);
+ return Response.ok(jsonObject).build();
+ }
+
+ @Path("/upload")
+ @POST
+ @Consumes(MediaType.MULTIPART_FORM_DATA)
+ public Response uploadFile(
+ @FormDataParam("file") InputStream uploadedInputStream,
+ @FormDataParam("file") FormDataContentDisposition fileDetail,
+ @FormDataParam("isFirstRowHeader") Boolean isFirstRowHeader,
+ @FormDataParam("filePath") String filePath
+
+ ) throws IOException, InterruptedException {
+ LOG.info("inside uploadFile : isFirstRowHeader : {} , filePath : {}", isFirstRowHeader, filePath);
+/* This is not working as expected.
+ ParseOptions parseOptions = new ParseOptions();
+ parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.FILE_TYPE_CSV);
+ parseOptions.setOption(ParseOptions.HEADERS,cdList);
+
+ if(isFirstRowHeader)
+ parseOptions.setOption(ParseOptions.OPTIONS_HEADER,ParseOptions.HEADER_FIRST_RECORD);
+ else
+ parseOptions.setOption(ParseOptions.OPTIONS_HEADER,ParseOptions.HEADER_PROVIDED_BY_USER);
+
+ DataParser dataParser = new DataParser(new InputStreamReader(uploadedInputStream),parseOptions);
+
+ // remove first row if it is header and send the rest to HDFS
+ if(isFirstRowHeader){
+ if( dataParser.iterator().hasNext() ){
+ dataParser.iterator().next();
+ }
+ }
+
+ Reader csvReader = dataParser.getCSVReader();
+*/
+
+ // TODO : workaround alert as above method is not working properly
+ // remove first row if it is header and send the rest to HDFS
+ Reader r = new InputStreamReader(uploadedInputStream);
+ if (isFirstRowHeader) {
+ BufferedReader br = new BufferedReader(r, 1); //
+ br.readLine(); // TODO : remove the header line. Wrong if first record is beyond first endline
+ }
+
+
+ String basePath = getHiveMetaStoreLocation();
+ if (null == basePath)
+ basePath = "/apps/hive/warehouse";
+
+ if (!basePath.endsWith("/"))
+ basePath = basePath + "/";
+
+ String fullPath = basePath + filePath;
+
+ uploadTable(new ReaderInputStream(r), fullPath);
+
+ LOG.info("create the table successfully at : {}", fullPath);
+ return Response.ok().build();
+ }
+
+ private String getHiveMetaStoreLocation() {
+ return this.getAmbariApi().getCluster().getConfigurationValue(HIVE_SITE, HIVE_META_STORE_LOCATION_KEY);
+ }
+
+ private void uploadTable(InputStream is, String path) throws IOException, InterruptedException {
+ if (!path.endsWith("/")) {
+ path = path + "/";
+ }
+
+ uploadFile(path, is);
+ }
+
+ private void uploadFile(final String filePath, InputStream uploadedInputStream)
+ throws IOException, InterruptedException {
+ byte[] chunk = new byte[1024];
+ FSDataOutputStream out = getSharedObjectsFactory().getHdfsApi().create(filePath, false);
+ while (uploadedInputStream.read(chunk) != -1) {
+ out.write(chunk);
+ }
+ out.close();
+ }
+
+
+ protected synchronized JobResourceManager getResourceManager() {
+ if (resourceManager == null) {
+ SharedObjectsFactory connectionsFactory = getSharedObjectsFactory();
+ resourceManager = new JobResourceManager(connectionsFactory, context);
+ }
+ return resourceManager;
+ }
+
+ protected synchronized AmbariApi getAmbariApi() {
+ if (null == ambariApi) {
+ ambariApi = new AmbariApi(this.context);
+ }
+ return ambariApi;
+ }
+
+ private String generateCreateQuery(String databaseName, String tableName, List<ColumnDescription> cdList) {
+ return new QueryGenerator().generateCreateQuery(new TableInfo(databaseName, tableName, cdList));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/resources/ui/hive-web/app/adapters/file-upload.js
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/resources/ui/hive-web/app/adapters/file-upload.js b/contrib/views/hive/src/main/resources/ui/hive-web/app/adapters/file-upload.js
new file mode 100644
index 0000000..1bd8eee
--- /dev/null
+++ b/contrib/views/hive/src/main/resources/ui/hive-web/app/adapters/file-upload.js
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import EmberUploader from 'ember-uploader';
+import Ember from 'ember';
+
+export default EmberUploader.Uploader.extend({
+ headers: {},
+
+ // Override
+ _ajax: function(settings) {
+ settings = Ember.merge(settings, this.getProperties('headers'));
+ console.log("_ajax : settings: " + JSON.stringify(settings));
+ return this._super(settings);
+ }
+});
http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/resources/ui/hive-web/app/adapters/upload-table.js
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/resources/ui/hive-web/app/adapters/upload-table.js b/contrib/views/hive/src/main/resources/ui/hive-web/app/adapters/upload-table.js
new file mode 100644
index 0000000..6a9c54b
--- /dev/null
+++ b/contrib/views/hive/src/main/resources/ui/hive-web/app/adapters/upload-table.js
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import EmberUploader from 'ember-uploader';
+import Ember from 'ember';
+import application from './application';
+import FileUploader from './file-upload';
+
+export default application.extend({
+ hdrs : function(){
+ console.log("inside hdrs : headers : ",this.get('headers'));
+ var h = Ember.$.extend(true, {},this.get('headers'));
+ delete h['Content-Type'];
+ return h;
+ }.property('headers'),
+
+ buildUploadURL: function (path) {
+ return this.buildURL() + "/resources/upload/" + path;
+ },
+
+ uploadFiles: function (path, files, extras) {
+ var uploadUrl = this.buildUploadURL(path);
+
+ console.log("uplaoder : uploadURL : ", uploadUrl);
+ console.log("uploader : extras : ", extras);
+ console.log("uploader : files : ", files);
+
+ console.log("hdrs : ", this.get('hdrs'));
+ var uploader = FileUploader.create({
+ headers: this.get('hdrs'),
+ url: uploadUrl
+ });
+
+ if (!Ember.isEmpty(files)) {
+ var promise = uploader.upload(files[0], extras);
+ return promise;
+ }
+ },
+
+ createTable: function (tableData) {
+ var _this = this;
+ var postHeader = JSON.parse(JSON.stringify(this.get('headers')));
+ console.log("headers postHeadesfsfdfsfsfss : : " , postHeader);
+ return Ember.$.ajax( {
+ url : this.buildUploadURL("createTable"),
+ type : 'post',
+ data: JSON.stringify(tableData),
+ headers: this.get('headers'),
+ dataType : 'json'
+ }
+ );
+ },
+
+ getCreateTableResult : function(jobId){
+ return Ember.$.ajax(this.buildUploadURL("createTable/status"),{
+ data : {"jobId":jobId},
+ type: "get",
+ headers: this.get('headers')
+ });
+ }
+});
http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/resources/ui/hive-web/app/components/file-upload.js
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/resources/ui/hive-web/app/components/file-upload.js b/contrib/views/hive/src/main/resources/ui/hive-web/app/components/file-upload.js
new file mode 100644
index 0000000..1cd05ae
--- /dev/null
+++ b/contrib/views/hive/src/main/resources/ui/hive-web/app/components/file-upload.js
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import EmberUploader from 'ember-uploader';
+
+export default EmberUploader.FileField.extend({
+ filesDidChange: function(files) {
+ this.sendAction('filesUploaded',files); // sends this action to controller.
+ }
+});
http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/resources/ui/hive-web/app/components/navbar-widget.js
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/resources/ui/hive-web/app/components/navbar-widget.js b/contrib/views/hive/src/main/resources/ui/hive-web/app/components/navbar-widget.js
index c3659cf..11333d0 100644
--- a/contrib/views/hive/src/main/resources/ui/hive-web/app/components/navbar-widget.js
+++ b/contrib/views/hive/src/main/resources/ui/hive-web/app/components/navbar-widget.js
@@ -34,6 +34,9 @@ export default Ember.Component.extend({
path: constants.namingConventions.routes.history}),
Ember.Object.create({text: 'menus.udfs',
- path: constants.namingConventions.routes.udfs})
+ path: constants.namingConventions.routes.udfs}),
+
+ Ember.Object.create({text: 'menus.uploadTable',
+ path: constants.namingConventions.routes.uploadTable})
])
});
http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/resources/ui/hive-web/app/controllers/upload-table.js
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/resources/ui/hive-web/app/controllers/upload-table.js b/contrib/views/hive/src/main/resources/ui/hive-web/app/controllers/upload-table.js
new file mode 100644
index 0000000..ab7b934
--- /dev/null
+++ b/contrib/views/hive/src/main/resources/ui/hive-web/app/controllers/upload-table.js
@@ -0,0 +1,261 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Ember from 'ember';
+import Uploader from 'hive/adapters/upload-table'
+import constants from 'hive/utils/constants';
+
+
+export default Ember.Controller.extend({
+ notifyService: Ember.inject.service(constants.namingConventions.notify),
+ needs : ['databases'],
+ showErrors : false,
+ uploader: Uploader.create(),
+ baseUrl: "/resources/upload",
+ isFirstRowHeader: null, // is first row header
+ header: null, // header received from server
+ files: null, // files that need to be uploaded only file[0] is relevant
+ firstRow: [], // the actual first row of the table.
+ rows: null, // preview rows received from server
+ databaseName:null,
+ selectedDatabase : null,
+ filePath : null,
+ tableName: null,
+ dataTypes : [
+ "TINYINT", //
+ "SMALLINT", //
+ "INT", //
+ "BIGINT", //
+ "BOOLEAN", //
+ "FLOAT", //
+ "DOUBLE", //
+ "STRING", //
+ "BINARY", // -- (Note: Available in Hive 0.8.0 and later)
+ "TIMESTAMP", // -- (Note: Available in Hive 0.8.0 and later)
+ "DECIMAL", // -- (Note: Available in Hive 0.11.0 and later)
+ "DATE", // -- (Note: Available in Hive 0.12.0 and later)
+ "VARCHAR", // -- (Note: Available in Hive 0.12.0 and later)
+ "CHAR" // -- (Note: Available in Hive 0.13.0 and later)
+ ],
+ isFirstRowHeaderDidChange: function () {
+ console.log("inside onFirstRowHeader : isFirstRowHeader : " + this.get('isFirstRowHeader'));
+ if (this.get('isFirstRowHeader') != null && typeof this.get('isFirstRowHeader') !== 'undefined') {
+ if (this.get('isFirstRowHeader') == false) {
+ if (this.get('rows')) {
+ this.get('rows').unshiftObject({row: this.get('firstRow')});
+ }
+ } else {
+ // take first row of
+ this.get('header').forEach(function (item, index) {
+ console.log("item : ", item);
+ console.log("this.get('firstRow').objectAt(index) : ", this.get('firstRow').objectAt(index));
+ Ember.set(item, 'name', this.get('firstRow')[index]);
+ }, this);
+
+ this.get('rows').removeAt(0);
+ }
+
+ this.printValues();
+ }
+ }.observes('isFirstRowHeader'),
+
+ uploadForPreview: function (files) {
+ console.log("uploaderForPreview called.");
+ return this.get('uploader').uploadFiles('preview', files);
+ },
+
+ clearFields: function () {
+ this.set("header");
+ this.set("rows");
+ this.set("error");
+ this.set('isFirstRowHeader');
+ this.set('files');
+ this.set("firstRow");
+ this.set("selectedDatabase");
+ this.set("databaseName");
+ this.set("filePath");
+ this.set('tableName');
+
+ this.printValues();
+ },
+
+ printValues: function () {
+ console.log("printing all values : ");
+ console.log("header : ", this.get('header'));
+ console.log("rows : ", this.get('rows'));
+ console.log("error : ", this.get('error'));
+ console.log("isFirstRowHeader : ", this.get('isFirstRowHeader'));
+ console.log("files : ", this.get('files'));
+ console.log("firstRow : ", this.get('firstRow'));
+ },
+ previewTable: function (data) {
+ console.log('inside previewTable');
+ this.set("header", data.header);
+ this.set("rows", data.rows);
+ this.set("firstRow", data.rows[0].row);
+ console.log("firstRow : ", this.get('firstRow'));
+ this.set('isFirstRowHeader', data.isFirstRowHeader);
+ },
+
+ fetchCreateTableStatus: function (jobId, resolve, reject) {
+ var self = this;
+ this.get('uploader').getCreateTableResult(jobId).then(function (data) {
+ console.log("fetchCreateTableStatus : data : ", data);
+ var status = data.status;
+ if (status == "Succeeded") {
+ console.log("resolving fetchCreateTableStatus with : " + data);
+ resolve(status);
+ } else if (status == "Canceled" || status == "Closed" || status == "Error") {
+ console.log("rejecting fetchCreateTableStatus with : " + status);
+ reject(new Error(status));
+ } else {
+ console.log("retrying fetchCreateTableStatus : ");
+ self.fetchCreateTableStatus(jobId, resolve, reject);
+ }
+ }, function (error) {
+ console.log("rejecting fetchCreateTableStatus with : " + error);
+ reject(error);
+ })
+ },
+
+ waitForResult: function (jobId) {
+ var self = this;
+ return new Ember.RSVP.Promise(function (resolve, reject) {
+ self.fetchCreateTableStatus(jobId,resolve,reject);
+ });
+ },
+
+ createTable: function () {
+ var headers = JSON.stringify(this.get('header'));
+
+ var selectedDatabase = this.get('selectedDatabase');
+ if( null == selectedDatabase || typeof selectedDatabase === 'undefined'){
+ throw new Error(constants.hive.errors.emptyDatabase);
+ }
+
+ this.set('databaseName',this.get('selectedDatabase').get('name'));
+ var databaseName = this.get('databaseName');
+ var tableName = this.get('tableName');
+ var isFirstRowHeader = this.get('isFirstRowHeader');
+ console.log("databaseName : " , databaseName, ", tableName : ", tableName, ", isFirstRowHeader : " , isFirstRowHeader , ", headers : ", headers);
+
+ if( null == databaseName || typeof databaseName === 'undefined'){
+ throw new Error(constants.hive.errors.emptyDatabase);
+ }
+ if( null == tableName || typeof tableName === 'undefined'){
+ throw new Error(constants.hive.errors.emptyTableName);
+ }
+ if( null == isFirstRowHeader || typeof isFirstRowHeader === 'undefined'){
+ throw new Error(constants.hive.errors.emptyIsFirstRow);
+ }
+
+ this.validateColumns();
+
+ return this.get('uploader').createTable({
+ "isFirstRowHeader": isFirstRowHeader,
+ "header": headers,
+ "tableName": tableName,
+ "databaseName": databaseName
+ });
+ },
+
+ validateColumns: function(){
+ // TODO :check validation of columnames.
+ // throw exception if invalid.
+ },
+ setError: function(error){
+ this.set('error',JSON.stringify(error));
+ console.log("upload table error : ",error);
+ this.get('notifyService').error(error);
+ },
+
+ previewError: function (error) {
+ this.setError(error);
+ },
+
+ uploadTable: function () {
+ this.printValues();
+ return this.get('uploader').uploadFiles('upload', this.get('files'), {
+ "isFirstRowHeader": this.get("isFirstRowHeader"),
+ "filePath": this.get('filePath')
+ });
+ },
+
+ onUploadSuccessfull: function (data) {
+ console.log("onUploadSuccessfull : ", data);
+ this.get('notifyService').success( "Uploaded Successfully", "Table " + this.get('tableName') + " created in database " + this.get("databaseName"));
+ this.clearFields();
+ },
+
+ onUploadError: function (error) {
+ console.log("onUploadError : ", error);
+ this.setError(error);
+ },
+
+ actions: {
+ toggleErrors : function(){
+ this.toggleProperty('showErrors');
+ },
+ filesUploaded: function (files) {
+ console.log("upload-table.js : uploaded new files : ", files);
+
+ this.clearFields();
+
+ this.set('files', files);
+ var name = files[0].name;
+ var i = name.indexOf(".");
+ var tableName = name.substr(0, i);
+ this.set('tableName', tableName);
+ var self = this;
+ return this.uploadForPreview(files).then(function (data) {
+ self.previewTable(data);
+ }, function (error) {
+ self.previewError(error);
+ });
+ },
+
+ createTableAndUploadFile: function () {
+ var self = this;
+
+ try {
+ this.createTable()
+ .then(function (jobData) {
+ console.log("jobData : ", jobData);
+ self.set('filePath', jobData.filePath);
+ self.waitForResult(jobData.jobId)
+ .then(function (successStatus) {
+ console.log("successStatus : ", successStatus);
+ self.uploadTable().then(function (operationData) {
+ console.log("operation successfull operationData : ", operationData);
+ self.onUploadSuccessfull(operationData);
+ }, function (error) {
+ self.onUploadError(error);
+ });
+ }, function (error) {
+ self.onUploadError(error);
+ })
+ }, function (error) {
+ self.onUploadError(error);
+ })
+ }catch(e){
+ self.onUploadError(e);
+ }
+ }
+
+ }
+});
http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/resources/ui/hive-web/app/initializers/i18n.js
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/resources/ui/hive-web/app/initializers/i18n.js b/contrib/views/hive/src/main/resources/ui/hive-web/app/initializers/i18n.js
index 5ae9b7e..bd0e6e6 100644
--- a/contrib/views/hive/src/main/resources/ui/hive-web/app/initializers/i18n.js
+++ b/contrib/views/hive/src/main/resources/ui/hive-web/app/initializers/i18n.js
@@ -158,6 +158,7 @@ TRANSLATIONS = {
savedQueries: 'Saved Queries',
history: 'History',
udfs: 'UDFs',
+ uploadTable: 'Upload Table',
logs: 'Logs',
results: 'Results',
explain: 'Explain'
@@ -199,6 +200,7 @@ TRANSLATIONS = {
expand: 'Expand message',
collapse: 'Collapse message',
previousPage: 'previous',
+ uploadTable: 'Upload Table',
nextPage: 'next',
loadMore: 'Load more...',
saveHdfs: 'Save to HDFS',
@@ -237,7 +239,10 @@ TRANSLATIONS = {
hive: {
errors: {
- 'no.query': "No query to process."
+ 'no.query': "No query to process.",
+ 'emptyDatabase' : "Please select Database.",
+ 'emptyTableName' : "Please enter tableName.",
+ 'emptyIsFirstRow' : "Please select is First Row Header?"
}
},
http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/resources/ui/hive-web/app/router.js
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/resources/ui/hive-web/app/router.js b/contrib/views/hive/src/main/resources/ui/hive-web/app/router.js
index 5a51b11..56e87d9 100644
--- a/contrib/views/hive/src/main/resources/ui/hive-web/app/router.js
+++ b/contrib/views/hive/src/main/resources/ui/hive-web/app/router.js
@@ -31,6 +31,7 @@ Router.map(function () {
this.route(constants.namingConventions.routes.queries);
this.route(constants.namingConventions.routes.history);
this.route(constants.namingConventions.routes.udfs);
+ this.route(constants.namingConventions.routes.uploadTable);
this.resource(constants.namingConventions.routes.index, { path: '/' }, function () {
this.route(constants.namingConventions.routes.savedQuery, { path: savedQueryPath});
http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/resources/ui/hive-web/app/styles/app.scss
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/resources/ui/hive-web/app/styles/app.scss b/contrib/views/hive/src/main/resources/ui/hive-web/app/styles/app.scss
index 7e23d46..f37a057 100644
--- a/contrib/views/hive/src/main/resources/ui/hive-web/app/styles/app.scss
+++ b/contrib/views/hive/src/main/resources/ui/hive-web/app/styles/app.scss
@@ -550,11 +550,10 @@ tree-view ul li {
margin: 0 0 10px;
}
-#query-results {
+#query-results, #upload-table {
.table {
display: inline-block;
overflow: auto;
- max-width: 830px;
}
.query-results-tools {
http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/resources/ui/hive-web/app/templates/upload-table.hbs
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/resources/ui/hive-web/app/templates/upload-table.hbs b/contrib/views/hive/src/main/resources/ui/hive-web/app/templates/upload-table.hbs
new file mode 100644
index 0000000..0ef4a7d
--- /dev/null
+++ b/contrib/views/hive/src/main/resources/ui/hive-web/app/templates/upload-table.hbs
@@ -0,0 +1,92 @@
+{{!
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+}}
+
+<div class="pull-right">
+ <i class="query-menu-tab fa queries-icon query-context-tab fa-envelope" {{ action 'toggleErrors'}}></i>
+</div>
+
+<div style="width : 90%">
+<div class="main-content">
+ {{#if showErrors}}
+ {{render 'messages'}}
+ {{/if}}
+</div>
+</div>
+
+{{#unless showErrors}}
+<div>
+ {{file-upload filesUploaded="filesUploaded"}}
+</div>
+<div id="upload-table">
+ <!--<div class='fa query-menu-tab fa queries-icon query-context-tab fa-envelope'></div>-->
+
+ {{#if rows}}
+ <div class="query-results-tools">
+ <div class="pull-right">
+ <button type="button" {{action
+ "createTableAndUploadFile"}}
+ {{bind-attr class=":btn :btn-sm :btn-default"}}>{{t "buttons.uploadTable"}}</button>
+ </div>
+ </div>
+
+ <div>
+ <div class="col-md-3">Database :
+ {{typeahead-widget
+ content=controllers.databases.databases
+ optionValuePath="id"
+ optionLabelPath="name"
+ selection=selectedDatabase
+ placeholder="Select a Database"
+ }}
+ </div>
+ <div class="col-md-3">Table Name : {{input type="text" class="form-control" placeHolder="Table Name" value=tableName }}
+ </div>
+ <div class="col-md-3">Is First Row Header? :{{input type="checkbox" class="form-control" checked=isFirstRowHeader }}
+ </div>
+ </div>
+
+ <table class="table table-expandable">
+ <thead>
+ <tr>
+ {{#each column in header}}
+ <th> {{input type="text" class="form-control" value=column.name}}</th>
+ {{/each}}
+ </tr>
+ <tr>
+ {{#each column in header}}
+ <th> {{typeahead-widget
+ content=dataTypes
+ selection=column.type
+ }}
+ </th>
+ {{/each}}
+ </tr>
+ </thead>
+ <tbody>
+ {{#each row in rows}}
+ <tr>
+ {{#each item in row.row}}
+ <td>{{item}}</td>
+ {{/each}}
+ </tr>
+ {{/each}}
+ </tbody>
+ </table>
+ {{/if}}
+</div>
+{{/unless}}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/resources/ui/hive-web/app/utils/constants.js
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/resources/ui/hive-web/app/utils/constants.js b/contrib/views/hive/src/main/resources/ui/hive-web/app/utils/constants.js
index a349d51..199677d 100644
--- a/contrib/views/hive/src/main/resources/ui/hive-web/app/utils/constants.js
+++ b/contrib/views/hive/src/main/resources/ui/hive-web/app/utils/constants.js
@@ -39,6 +39,7 @@ export default Ember.Object.create({
logs: 'logs',
results: 'results',
explain: 'explain',
+ uploadTable :'upload-table',
visualization: 'visualization'
},
http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/resources/ui/hive-web/bower.json
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/resources/ui/hive-web/bower.json b/contrib/views/hive/src/main/resources/ui/hive-web/bower.json
index d43881f..72cdf2a 100644
--- a/contrib/views/hive/src/main/resources/ui/hive-web/bower.json
+++ b/contrib/views/hive/src/main/resources/ui/hive-web/bower.json
@@ -18,6 +18,7 @@
"jquery-ui": "~1.11.2",
"selectize": "~0.12.0",
"pretender": "0.1.0",
+ "ember-uploader": "0.3.9",
"polestar": "https://github.com/pallavkul/polestar.git",
"voyager": "https://github.com/pallavkul/voyager.git"
},
http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/resources/ui/hive-web/package.json
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/resources/ui/hive-web/package.json b/contrib/views/hive/src/main/resources/ui/hive-web/package.json
index 6ecdcb6..595b1f2 100644
--- a/contrib/views/hive/src/main/resources/ui/hive-web/package.json
+++ b/contrib/views/hive/src/main/resources/ui/hive-web/package.json
@@ -38,6 +38,7 @@
"ember-cli-qunit": "0.3.14",
"ember-cli-selectize": "0.0.19",
"ember-cli-uglify": "1.0.1",
+ "ember-cli-uploader": "^0.3.9",
"ember-data": "1.0.0-beta.16.1",
"ember-dynamic-component": "0.0.1",
"ember-export-application-global": "^1.0.0",
http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/main/resources/view.xml
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/resources/view.xml b/contrib/views/hive/src/main/resources/view.xml
index fdc32d7..23f901b 100644
--- a/contrib/views/hive/src/main/resources/view.xml
+++ b/contrib/views/hive/src/main/resources/view.xml
@@ -255,6 +255,12 @@
</resource>
<resource>
+ <name>upload</name>
+ <plural-name>uploads</plural-name>
+ <service-class>org.apache.ambari.view.hive.resources.uploads.UploadService</service-class>
+ </resource>
+
+ <resource>
<name>file</name>
<service-class>org.apache.ambari.view.hive.resources.files.FileService</service-class>
</resource>
http://git-wip-us.apache.org/repos/asf/ambari/blob/07929393/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/DataParserTest.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/DataParserTest.java b/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/DataParserTest.java
new file mode 100644
index 0000000..3d77d29
--- /dev/null
+++ b/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/DataParserTest.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.hive.resources.upload;
+
+import org.apache.ambari.view.hive.client.ColumnDescription;
+import org.apache.ambari.view.hive.client.ColumnDescriptionShort;
+import org.apache.ambari.view.hive.client.Row;
+import org.apache.ambari.view.hive.resources.uploads.ColumnDescriptionImpl;
+import org.apache.ambari.view.hive.resources.uploads.DataParser;
+import org.apache.ambari.view.hive.resources.uploads.ParseOptions;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.Arrays;
+
+public class DataParserTest {
+
+ @Test
+ public void testDataParser() throws IOException {
+ String str = "1,a\n" +
+ "2,b\n" +
+ "3,c\n";
+ StringReader sr = new StringReader(str);
+
+ ParseOptions parseOptions = new ParseOptions();
+ parseOptions.setOption(ParseOptions.OPTIONS_FILE_TYPE, ParseOptions.FILE_TYPE_CSV);
+ parseOptions.setOption(ParseOptions.OPTIONS_HEADER, ParseOptions.HEADER_FIRST_RECORD);
+
+ DataParser dp = new DataParser(sr, parseOptions);
+ dp.parsePreview();
+ Assert.assertNotNull(dp.getPreviewRows());
+ Assert.assertNotNull(dp.getHeader());
+ Assert.assertEquals(3, dp.getPreviewRows().size());
+ Assert.assertEquals(2, dp.getHeader().size());
+ ColumnDescription[] cd = {new ColumnDescriptionImpl("1", ColumnDescriptionShort.DataTypes.INT.toString(), 0),
+ new ColumnDescriptionImpl("a", ColumnDescriptionShort.DataTypes.CHAR.toString(), 1)};
+
+ Assert.assertArrayEquals("Header Not Correct.", cd, dp.getHeader().toArray());
+
+ // TODO : include testing of each row element. Below comparison does not work properly.
+ // Object[] rows = {new Row(new Object[]{'1','a'}),new Row(new Object[]{'2','b'}),new Row(new Object[]{'3','c'})};
+ // Assert.assertArrayEquals("Rows Not Correct.", rows, dp.getPreviewRows().toArray());
+
+ sr.close();
+ }
+}