You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@knox.apache.org by lm...@apache.org on 2019/09/04 02:53:46 UTC
[knox] branch master updated: KNOX-2005 - Improvements to
KnoxShellTable
This is an automated email from the ASF dual-hosted git repository.
lmccay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/knox.git
The following commit(s) were added to refs/heads/master by this push:
new 074a035 KNOX-2005 - Improvements to KnoxShellTable
074a035 is described below
commit 074a0354b815968a162790f09189ff47ca04e169
Author: lmccay <lm...@apache.org>
AuthorDate: Tue Sep 3 22:53:34 2019 -0400
KNOX-2005 - Improvements to KnoxShellTable
---
gateway-shell/pom.xml | 9 +-
.../apache/knox/gateway/shell/KnoxShellTable.java | 334 ++++++++++++++++++++-
.../java/org/apache/knox/gateway/shell/Shell.java | 3 +-
.../knox/gateway/shell/KnoxShellTableTest.java | 119 +++++++-
.../gateway/i18n/GatewayUtilCommonMessages.java | 2 +
.../org/apache/knox/gateway/util/JsonUtils.java | 27 ++
6 files changed, 487 insertions(+), 7 deletions(-)
diff --git a/gateway-shell/pom.xml b/gateway-shell/pom.xml
index 7f5f09d..139355b 100644
--- a/gateway-shell/pom.xml
+++ b/gateway-shell/pom.xml
@@ -110,6 +110,13 @@
<groupId>de.thetaphi</groupId>
<artifactId>forbiddenapis</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
</dependencies>
-
</project>
diff --git a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/KnoxShellTable.java b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/KnoxShellTable.java
index 9905c42..6c5e1b1 100644
--- a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/KnoxShellTable.java
+++ b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/KnoxShellTable.java
@@ -17,11 +17,27 @@
*/
package org.apache.knox.gateway.shell;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.URL;
+import java.net.URLConnection;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.regex.Pattern;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.knox.gateway.util.JsonUtils;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
/**
* Simple table representation and text based rendering of a table via toString().
@@ -36,6 +52,12 @@ public class KnoxShellTable {
private List<String> headers = new ArrayList<String>();
private List<List<String>> rows = new ArrayList<List<String>>();
+ private String title;
+
+ public KnoxShellTable title(String title) {
+ this.title = title;
+ return this;
+ }
public KnoxShellTable header(String header) {
headers.add(header);
@@ -58,16 +80,106 @@ public class KnoxShellTable {
return this;
}
+ public KnoxShellTableCell cell(int colIndex, int rowIndex) {
+ return new KnoxShellTableCell(colIndex, rowIndex);
+ }
+
+ public List<String> values(int colIndex) {
+ ArrayList<String> col = new ArrayList<String>();
+ rows.forEach(row -> col.add(row.get(colIndex)));
+ return col;
+ }
+
+ public KnoxShellTable apply(KnoxShellTableCell cell) {
+ if (!headers.isEmpty()) {
+ headers.set(cell.colIndex, cell.header);
+ }
+ if (!rows.isEmpty()) {
+ rows.get(cell.rowIndex).set(cell.colIndex, cell.value);
+ }
+ return this;
+ }
+
+ public class KnoxShellTableCell {
+ private int colIndex;
+ private int rowIndex;
+ private String header;
+ private String value;
+
+ KnoxShellTableCell(int colIndex, int rowIndex) {
+ this.colIndex = colIndex;
+ this.rowIndex = rowIndex;
+ if (!headers.isEmpty()) {
+ this.header = headers.get(colIndex);
+ }
+ if (!rows.isEmpty()) {
+ this.value = rows.get(rowIndex).get(colIndex);
+ }
+ }
+
+ KnoxShellTableCell(String name, int rowIndex) {
+ this.rowIndex = rowIndex;
+ if (!headers.isEmpty()) {
+ this.header = name;
+ this.colIndex = headers.indexOf(name);
+ }
+ if (!rows.isEmpty()) {
+ this.value = rows.get(rowIndex).get(colIndex);
+ }
+ }
+
+ public KnoxShellTableCell value(String value) {
+ this.value = value;
+ return this;
+ }
+
+ public KnoxShellTableCell header(String name) {
+ this.header = name;
+ return this;
+ }
+
+ public String value() {
+ return this.value;
+ }
+
+ public String header() {
+ return this.header;
+ }
+ }
+
+ public List<String> getHeaders() {
+ if (headers.isEmpty()) {
+ return null;
+ }
+ return headers;
+ }
+
+ public List<List<String>> getRows() {
+ return rows;
+ }
+
+ public String getTitle() {
+ return title;
+ }
+
@Override
public String toString() {
- if (!headers.isEmpty() && headers.size() != rows.get(0).size()) {
- throw new IllegalStateException("Number of columns within rows and headers must be the same.");
+ if (!headers.isEmpty() && !rows.isEmpty() && headers.size() != rows.get(0).size()) {
+ throw new IllegalStateException("Number of columns and headers must be the same.");
}
StringBuilder sb = new StringBuilder();
Map<Integer, Integer> widthMap = getWidthMap();
- int colCount = rows.get(0).size();
+ if (title != null && !title.isEmpty()) {
+ sb.append(this.title);
+ newLine(sb, 1);
+ }
+ int colCount = 0;
+ if (!rows.isEmpty()) {
+ colCount = rows.get(0).size();
+ }
if (!headers.isEmpty()) {
+ colCount = headers.size();
createBorder(sb, colCount, widthMap);
newLine(sb, 1);
@@ -145,7 +257,221 @@ public class KnoxShellTable {
}
}
}
-
return map;
}
+
+ public static KnoxShellTableBuilder builder() {
+ return new KnoxShellTableBuilder();
+ }
+
+ public static class KnoxShellTableBuilder {
+ public CSVKnoxShellTableBuilder csv() {
+ return new CSVKnoxShellTableBuilder();
+ }
+
+ public JSONKnoxShellTableBuilder json() {
+ return new JSONKnoxShellTableBuilder();
+ }
+
+ public JoinKnoxShellTableBuilder join() {
+ return new JoinKnoxShellTableBuilder();
+ }
+ }
+
+ public static class JoinKnoxShellTableBuilder {
+ private KnoxShellTable left;
+ private KnoxShellTable right;
+ private int leftIndex = -1;
+ private int rightIndex = -1;
+
+ public JoinKnoxShellTableBuilder() {
+ }
+
+ public JoinKnoxShellTableBuilder left(KnoxShellTable left) {
+ this.left = left;
+ return this;
+ }
+
+ public JoinKnoxShellTableBuilder right(KnoxShellTable right) {
+ this.right = right;
+ return this;
+ }
+
+ public KnoxShellTable on(int leftIndex, int rightIndex) {
+ KnoxShellTable joined = new KnoxShellTable();
+
+ this.leftIndex = leftIndex;
+ this.rightIndex = rightIndex;
+
+ joined.headers.addAll(new ArrayList<String>(left.headers));
+ for (List<String> row : left.rows) {
+ joined.rows.add(new ArrayList<String>(row));
+ }
+ List<String> col = right.values(leftIndex);
+ ArrayList<String> row;
+ String leftKey;
+ int matchedIndex;
+
+ joined.headers.addAll(new ArrayList<String>(right.headers));
+ for (Iterator<List<String>> it = joined.rows.iterator(); it.hasNext();) {
+ row = (ArrayList<String>) it.next();
+ leftKey = row.get(leftIndex);
+ if (leftKey != null) {
+ matchedIndex = col.indexOf(leftKey);
+ if (matchedIndex > -1) {
+ row.addAll(right.rows.get(matchedIndex));
+ }
+ else {
+ it.remove();
+ }
+ }
+ }
+ return joined;
+ }
+ }
+
+ public static class JSONKnoxShellTableBuilder {
+ boolean withHeaders;
+
+ public JSONKnoxShellTableBuilder withHeaders() {
+ withHeaders = true;
+ return this;
+ }
+
+ public KnoxShellTable string(String json) throws IOException {
+ KnoxShellTable table = getKnoxShellTableFromJsonString(json);
+ return table;
+ }
+
+ public KnoxShellTable path(String path) throws IOException {
+ String json = FileUtils.readFileToString(new File(path), StandardCharsets.UTF_8);
+ KnoxShellTable table = getKnoxShellTableFromJsonString(json);
+ return table;
+ }
+
+ public static KnoxShellTable getKnoxShellTableFromJsonString(String json) {
+ KnoxShellTable table = null;
+ JsonFactory factory = new JsonFactory();
+ ObjectMapper mapper = new ObjectMapper(factory);
+ TypeReference<KnoxShellTable> typeRef
+ = new TypeReference<KnoxShellTable>() {};
+ try {
+ table = mapper.readValue(json, typeRef);
+ } catch (IOException e) {
+ //LOG.failedToGetMapFromJsonString( json, e );
+ }
+ return table;
+ }
+ }
+
+ public static class CSVKnoxShellTableBuilder {
+ boolean withHeaders;
+
+ public CSVKnoxShellTableBuilder withHeaders() {
+ withHeaders = true;
+ return this;
+ }
+
+ public KnoxShellTable url(String url) throws IOException {
+ int rowIndex = 0;
+ URLConnection connection;
+ BufferedReader csvReader = null;
+ KnoxShellTable table = null;
+ try {
+ URL urlToCsv = new URL(url);
+ connection = urlToCsv.openConnection();
+ csvReader = new BufferedReader(new InputStreamReader(
+ connection.getInputStream(), StandardCharsets.UTF_8));
+ table = new KnoxShellTable();
+ String row = null;
+ while ((row = csvReader.readLine()) != null) {
+ boolean addingHeaders = (withHeaders && rowIndex == 0);
+ if (!addingHeaders) {
+ table.row();
+ }
+ String[] data = row.split(",");
+
+ for (String value : data) {
+ if (addingHeaders) {
+ table.header(value);
+ }
+ else {
+ table.value(value);
+ }
+ }
+ rowIndex++;
+ }
+ }
+ finally {
+ csvReader.close();
+ }
+ return table;
+ }
+ }
+
+ public String toJSON() {
+ return JsonUtils.renderAsJsonString(this);
+ }
+
+ public String toCSV() {
+ StringBuilder csv = new StringBuilder();
+ String header;
+ for(int i = 0; i < headers.size(); i++) {
+ header = headers.get(i);
+ csv.append(header);
+ if (i < headers.size() - 1) {
+ csv.append(',');
+ }
+ else {
+ csv.append('\n');
+ }
+ }
+ for(List<String> row : rows) {
+ for(int ii = 0; ii < row.size(); ii++) {
+ csv.append(row.get(ii));
+ if (ii < row.size() - 1) {
+ csv.append(',');
+ }
+ else {
+ csv.append('\n');
+ }
+ }
+ }
+
+ return csv.toString();
+ }
+
+ public KnoxShellTableFilter filter() {
+ return new KnoxShellTableFilter();
+ }
+
+ public class KnoxShellTableFilter {
+ String name;
+ int index;
+
+ public KnoxShellTableFilter name(String name) {
+ this.name = name;
+ index = headers.indexOf(name);
+ return this;
+ }
+
+ public KnoxShellTableFilter index(int index) {
+ this.index = index;
+ return this;
+ }
+
+ public KnoxShellTable regex(String regex) {
+ KnoxShellTable table = new KnoxShellTable();
+ table.headers.addAll(headers);
+ for (List<String> row : rows) {
+ if (Pattern.matches(regex, row.get(index))) {
+ table.row();
+ row.forEach(value -> {
+ table.value(value);
+ });
+ }
+ }
+ return table;
+ }
+ }
}
diff --git a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/Shell.java b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/Shell.java
index fa95af7..f0aaf41 100644
--- a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/Shell.java
+++ b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/Shell.java
@@ -43,7 +43,8 @@ public class Shell {
Workflow.class.getName(),
Yarn.class.getName(),
TimeUnit.class.getName(),
- Manager.class.getName()
+ Manager.class.getName(),
+ KnoxShellTable.class.getName()
};
static {
diff --git a/gateway-shell/src/test/java/org/apache/knox/gateway/shell/KnoxShellTableTest.java b/gateway-shell/src/test/java/org/apache/knox/gateway/shell/KnoxShellTableTest.java
index e82fc45..c036c3f 100644
--- a/gateway-shell/src/test/java/org/apache/knox/gateway/shell/KnoxShellTableTest.java
+++ b/gateway-shell/src/test/java/org/apache/knox/gateway/shell/KnoxShellTableTest.java
@@ -20,6 +20,12 @@ package org.apache.knox.gateway.shell;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.knox.gateway.shell.KnoxShellTable.KnoxShellTableCell;
import org.junit.Test;
public class KnoxShellTableTest {
@@ -82,9 +88,11 @@ public class KnoxShellTableTest {
table.row().value("789").value("012").value("844444444");
assertEquals(expectedResult, table.toString());
+
+ assertEquals(expectedResult, table.toString());
}
- @Test (expected = IllegalStateException.class)
+ @Test(expected = IllegalStateException.class)
public void testMultipleRowsTableMismatchedColAndHeadersCountError() {
KnoxShellTable table = new KnoxShellTable();
@@ -94,5 +102,114 @@ public class KnoxShellTableTest {
table.row().value("789").value("012").value("844444444");
assertNull(table.toString());
+ table.toString();
+ }
+
+ @Test
+ public void testLoadCSVToAndFromURL() {
+// table = KnoxShellTable.builder().db().driver("HiveDriver2").connect("sdklfjsdjflsd").sql("select * from test;");
+ KnoxShellTable table = new KnoxShellTable();
+ table.title("From URL");
+
+ table.header("Column A").header("Column B").header("Column C");
+
+ table.row().value("123").value("456").value("344444444");
+ table.row().value("789").value("012").value("844444444");
+
+ String csv = table.toCSV();
+ try {
+ // write file to /tmp to read back in
+ FileUtils.writeStringToFile(new File("/tmp/testtable.csv"), csv, StandardCharsets.UTF_8);
+
+ KnoxShellTable urlTable = KnoxShellTable.builder().csv()
+ .withHeaders()
+ .url("file:///tmp/testtable.csv");
+ //.url("http://samplecsvs.s3.amazonaws.com/Sacramentorealestatetransactions.csv");
+ urlTable.title("From URL");
+ assertEquals(urlTable.toString(), table.toString());
+
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Test
+ public void testToAndFromJSON() throws IOException {
+ KnoxShellTable table = new KnoxShellTable();
+
+ table.header("Column A").header("Column B").header("Column C");
+
+ table.row().value("123").value("456").value("344444444");
+ table.row().value("789").value("012").value("844444444");
+
+ String json = table.toJSON();
+
+ KnoxShellTable table2 = KnoxShellTable.builder().json().string(json);
+ assertEquals(table.toString(), table2.toString());
+ }
+
+ @Test
+ public void testCells() throws IOException {
+ KnoxShellTable table = new KnoxShellTable();
+
+ table.header("Column A").header("Column B").header("Column C");
+
+ table.row().value("123").value("456").value("344444444");
+ table.row().value("789").value("012").value("844444444");
+
+ KnoxShellTableCell cell = table.cell(1, 1);
+ assertEquals(cell.header(), "Column B");
+ assertEquals(cell.value(), "012");
+ cell.header("Column Beeee");
+ cell.value("234");
+ table.apply(cell);
+ assertEquals(table.cell(1, 1).value(), "234");
+ assertEquals(table.cell(1, 1).header(), "Column Beeee");
+ }
+
+ @Test
+ public void testFilterTable() {
+ KnoxShellTable table = new KnoxShellTable();
+
+ table.header("Column A").header("Column B").header("Column C");
+
+ table.row().value("123").value("456").value("344444444");
+ table.row().value("789").value("012").value("844444444");
+
+ KnoxShellTable filtered = table.filter().name("Column A").regex("123");
+
+ assertEquals(filtered.getRows().size(), 1);
+ }
+
+ @Test
+ public void testJoinTables() throws IOException {
+ KnoxShellTable table = new KnoxShellTable();
+
+ table.title("Left Table").header("Column A").header("Column B").header("Column C");
+
+ table.row().value("123").value("456").value("344444444");
+ table.row().value("789").value("012").value("844444444");
+
+ KnoxShellTable table2 = new KnoxShellTable();
+
+ table2.title("Right Table").header("Column D").header("Column E").header("Column F");
+
+ table2.row().value("123").value("367").value("244444444");
+ table2.row().value("780").value("908").value("944444444");
+
+ KnoxShellTable joined = KnoxShellTable.builder().join().left(table).right(table2).on(0, 0);
+ joined.title("Joined Table");
+
+ assertEquals(joined.getRows().size(), 1);
+ assertEquals(joined.getTitle(), "Joined Table");
+ assertEquals(joined.cell(0, 0).value(), "123");
+ String json = joined.toJSON();
+
+ KnoxShellTable zombie = KnoxShellTable.builder().json().string(json);
+ zombie.title("Zombie Table");
+
+ assertEquals(zombie.getRows().size(), 1);
+ assertEquals(zombie.getTitle(), "Zombie Table");
+ assertEquals(zombie.cell(0, 0).value(), "123");
}
}
diff --git a/gateway-util-common/src/main/java/org/apache/knox/gateway/i18n/GatewayUtilCommonMessages.java b/gateway-util-common/src/main/java/org/apache/knox/gateway/i18n/GatewayUtilCommonMessages.java
index 33572f5..96a8e32 100644
--- a/gateway-util-common/src/main/java/org/apache/knox/gateway/i18n/GatewayUtilCommonMessages.java
+++ b/gateway-util-common/src/main/java/org/apache/knox/gateway/i18n/GatewayUtilCommonMessages.java
@@ -36,4 +36,6 @@ public interface GatewayUtilCommonMessages {
@Message( level = MessageLevel.ERROR, text = "Error in generating certificate: {0}" )
void failedToGenerateCertificate( @StackTrace( level = MessageLevel.ERROR ) Exception e );
+ @Message(level = MessageLevel.ERROR, text = "Failed to serialize Object to Json string {0}: {1}" )
+ void failedToSerializeObjectToJSON( Object obj, @StackTrace( level = MessageLevel.DEBUG ) Exception e );
}
diff --git a/gateway-util-common/src/main/java/org/apache/knox/gateway/util/JsonUtils.java b/gateway-util-common/src/main/java/org/apache/knox/gateway/util/JsonUtils.java
index 6662767..28feea2 100644
--- a/gateway-util-common/src/main/java/org/apache/knox/gateway/util/JsonUtils.java
+++ b/gateway-util-common/src/main/java/org/apache/knox/gateway/util/JsonUtils.java
@@ -45,6 +45,33 @@ public class JsonUtils {
return json;
}
+ public static String renderAsJsonString(Object obj) {
+ String json = null;
+ ObjectMapper mapper = new ObjectMapper();
+
+ try {
+ // write JSON to a file
+ json = mapper.writerWithDefaultPrettyPrinter().writeValueAsString(obj);
+ } catch ( JsonProcessingException e ) {
+ LOG.failedToSerializeObjectToJSON( obj, e );
+ }
+ return json;
+ }
+
+ public static Object getObjectFromJsonString(String json) {
+ Map<String, String> obj = null;
+ JsonFactory factory = new JsonFactory();
+ ObjectMapper mapper = new ObjectMapper(factory);
+ TypeReference<Object> typeRef
+ = new TypeReference<Object>() {};
+ try {
+ obj = mapper.readValue(json, typeRef);
+ } catch (IOException e) {
+ LOG.failedToGetMapFromJsonString( json, e );
+ }
+ return obj;
+ }
+
public static Map<String, String> getMapFromJsonString(String json) {
Map<String, String> map = null;
JsonFactory factory = new JsonFactory();