You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fluo.apache.org by kt...@apache.org on 2018/04/04 14:30:43 UTC
[fluo] branch master updated (c35dea1 -> aae4b2b)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/fluo.git.
from c35dea1 Fix Javadoc badge (#1029)
new 0ec21c9 Scan command now can produce results as csv and json. #984 #1018
new f147016 Remove csv option from scan command #984 #1018
new aae4b2b Merge branch 'fluo-984-2'
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/fluo/cluster/runner/AppRunner.java | 16 ++-
.../java/org/apache/fluo/command/FluoScan.java | 39 ++++--
.../java/org/apache/fluo/core/util/ScanUtil.java | 137 +++++++++++++--------
modules/distribution/src/main/lib/fetch.sh | 7 +-
pom.xml | 4 +-
5 files changed, 132 insertions(+), 71 deletions(-)
--
To stop receiving notification emails like this one, please contact
kturner@apache.org.
[fluo] 02/03: Remove csv option from scan command #984 #1018
Posted by kt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/fluo.git
commit f147016e238ecddc30fa770889f75381753abc5e
Author: Keith Turner <kt...@apache.org>
AuthorDate: Thu Mar 29 17:27:58 2018 -0400
Remove csv option from scan command #984 #1018
---
.../org/apache/fluo/cluster/runner/AppRunner.java | 11 +-
modules/command/pom.xml | 4 -
.../java/org/apache/fluo/command/FluoScan.java | 64 +------
modules/core/pom.xml | 8 -
.../java/org/apache/fluo/core/util/ScanUtil.java | 203 +++++----------------
modules/distribution/src/main/lib/fetch.sh | 2 -
pom.xml | 10 -
7 files changed, 52 insertions(+), 250 deletions(-)
diff --git a/modules/cluster/src/main/java/org/apache/fluo/cluster/runner/AppRunner.java b/modules/cluster/src/main/java/org/apache/fluo/cluster/runner/AppRunner.java
index 5be9ef1..3ec090f 100644
--- a/modules/cluster/src/main/java/org/apache/fluo/cluster/runner/AppRunner.java
+++ b/modules/cluster/src/main/java/org/apache/fluo/cluster/runner/AppRunner.java
@@ -209,14 +209,6 @@ public abstract class AppRunner {
+ "internal schema, making it easier to comprehend.")
public boolean scanAccumuloTable = false;
- public boolean exportAsCsv = false;
- public String csvHeader;
- public String csvDelimiter;
- public String csvEscape;
- public String csvQuote;
- public String csvQuoteMode;
- public boolean exportAsJson = false;
-
public String getStartRow() {
return startRow;
}
@@ -242,8 +234,7 @@ public abstract class AppRunner {
public ScanUtil.ScanOpts getScanOpts() {
return new ScanUtil.ScanOpts(startRow, endRow, columns, exactRow, rowPrefix, help,
- hexEncNonAscii, scanAccumuloTable, exportAsCsv, csvDelimiter, csvEscape, csvHeader,
- csvQuote, csvQuoteMode, exportAsJson);
+ hexEncNonAscii, scanAccumuloTable, false);
}
}
}
diff --git a/modules/command/pom.xml b/modules/command/pom.xml
index f9949a2..8c8bf46 100644
--- a/modules/command/pom.xml
+++ b/modules/command/pom.xml
@@ -47,10 +47,6 @@
<artifactId>accumulo-core</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- </dependency>
- <dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-framework</artifactId>
</dependency>
diff --git a/modules/command/src/main/java/org/apache/fluo/command/FluoScan.java b/modules/command/src/main/java/org/apache/fluo/command/FluoScan.java
index 9f4455a..bcd03c0 100644
--- a/modules/command/src/main/java/org/apache/fluo/command/FluoScan.java
+++ b/modules/command/src/main/java/org/apache/fluo/command/FluoScan.java
@@ -4,9 +4,9 @@
* copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License. You may obtain a
* copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied. See the License for the specific language governing permissions and limitations under
@@ -20,7 +20,6 @@ import java.util.Collections;
import java.util.List;
import com.beust.jcommander.Parameter;
-import org.apache.commons.lang3.StringUtils;
import org.apache.fluo.api.config.FluoConfiguration;
import org.apache.fluo.core.client.FluoAdminImpl;
import org.apache.fluo.core.util.ScanUtil;
@@ -56,31 +55,6 @@ public class FluoScan {
+ "internal schema, making it easier to comprehend.")
public boolean scanAccumuloTable = false;
- @Parameter(names = "--csv", help = true,
- description = "Export key/values stored in Accumulo as CSV file. Uses Fluo application "
- + "properties to configure the CSV format.")
- public boolean exportAsCsv = false;
-
- @Parameter(names = "--csv-header", help = true, description = "Set header for \"true\".")
- public String csvHeader;
-
- @Parameter(names = "--csv-delimiter", help = true,
- description = "Configure delimiter to a designeted character.")
- public String csvDelimiter;
-
- @Parameter(names = "--csv-escape", help = true,
- description = "Configure escape to a designeted character.")
- public String csvEscape;
-
- @Parameter(names = "--csv-quote", help = true,
- description = "Configure quote to a designeted character.")
- public String csvQuote;
-
- @Parameter(names = "--csv-quote-mode", help = true,
- description = "Configure quote mode to a designeted mode. The possible "
- + "modes are: ALL, ALL_NON_NULL, MINIMAL, NONE and NON_NUMERIC")
- public String csvQuoteMode;
-
@Parameter(names = "--json", help = true,
description = "Export key/values stored in Accumulo as JSON file.")
public boolean exportAsJson = false;
@@ -101,26 +75,6 @@ public class FluoScan {
return rowPrefix;
}
- public String getCsvHeader() {
- return csvHeader;
- }
-
- public String getCsvDelimiter() {
- return csvDelimiter;
- }
-
- public String getCsvEscape() {
- return csvEscape;
- }
-
- public String getCsvQuote() {
- return csvQuote;
- }
-
- public String getCsvQuoteMode() {
- return csvQuoteMode;
- }
-
public List<String> getColumns() {
if (columns == null) {
return Collections.emptyList();
@@ -130,25 +84,17 @@ public class FluoScan {
/**
* Check if the parameters informed can be used together.
- * @since 1.2
*/
private void checkScanOptions() {
- if (this.exportAsCsv && this.exportAsJson) {
+ if (this.scanAccumuloTable && this.exportAsJson) {
throw new IllegalArgumentException(
- "Both \"--csv\" and \"--json\" can not be set together.");
- }
-
- if (!this.exportAsCsv && (StringUtils.isNotEmpty(this.csvDelimiter)
- | StringUtils.isNotEmpty(this.csvEscape) | StringUtils.isNotEmpty(this.csvHeader)
- | StringUtils.isNotEmpty(this.csvQuote) | StringUtils.isNotEmpty(this.csvQuoteMode))) {
- throw new IllegalArgumentException("No \"--csv\" detected");
+ "Both \"--raw\" and \"--json\" can not be set together.");
}
}
public ScanUtil.ScanOpts getScanOpts() {
return new ScanUtil.ScanOpts(startRow, endRow, columns, exactRow, rowPrefix, help,
- hexEncNonAscii, scanAccumuloTable, exportAsCsv, csvDelimiter, csvEscape, csvHeader,
- csvQuote, csvQuoteMode, exportAsJson);
+ hexEncNonAscii, scanAccumuloTable, exportAsJson);
}
public static ScanOptions parse(String[] args) {
diff --git a/modules/core/pom.xml b/modules/core/pom.xml
index 0cfc6b3..fdd1f69 100644
--- a/modules/core/pom.xml
+++ b/modules/core/pom.xml
@@ -50,14 +50,6 @@
<artifactId>accumulo-core</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-csv</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- </dependency>
- <dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-client</artifactId>
</dependency>
diff --git a/modules/core/src/main/java/org/apache/fluo/core/util/ScanUtil.java b/modules/core/src/main/java/org/apache/fluo/core/util/ScanUtil.java
index 78c3efb..4690fd5 100644
--- a/modules/core/src/main/java/org/apache/fluo/core/util/ScanUtil.java
+++ b/modules/core/src/main/java/org/apache/fluo/core/util/ScanUtil.java
@@ -22,24 +22,13 @@ import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedHashMap;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.function.Function;
-import com.google.common.collect.Iterables;
-import com.google.gson.FieldNamingPolicy;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import com.google.gson.JsonIOException;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.security.Authorizations;
-import org.apache.commons.csv.CSVFormat;
-import org.apache.commons.csv.CSVPrinter;
-import org.apache.commons.csv.QuoteMode;
-import org.apache.commons.lang3.BooleanUtils;
-import org.apache.commons.lang3.ObjectUtils;
-import org.apache.commons.lang3.StringUtils;
import org.apache.fluo.accumulo.format.FluoFormatter;
import org.apache.fluo.api.client.FluoClient;
import org.apache.fluo.api.client.FluoFactory;
@@ -50,14 +39,14 @@ import org.apache.fluo.api.data.Bytes;
import org.apache.fluo.api.data.Column;
import org.apache.fluo.api.data.RowColumnValue;
import org.apache.fluo.api.data.Span;
-import org.apache.fluo.api.exceptions.FluoException;
+
+import com.google.common.collect.Iterables;
+import com.google.gson.FieldNamingPolicy;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonIOException;
public class ScanUtil {
- public static final String CSV_HEADER = "csv.header";
- public static final String CSV_QUOTE_MODE = "csv.quoteMode";
- public static final String CSV_QUOTE = "csv.quote";
- public static final String CSV_ESCAPE = "csv.escape";
- public static final String CSV_DELIMITER = "csv.delimiter";
public static final String FLUO_VALUE = "value";
public static final String FLUO_COLUMN_VISIBILITY = "visibility";
public static final String FLUO_COLUMN_QUALIFIER = "qualifier";
@@ -115,6 +104,15 @@ public class ScanUtil {
return columns;
}
+
+ private static Function<Bytes, String> getEncoder(ScanOpts options) {
+ if (options.hexEncNonAscii) {
+ return Hex::encNonAscii;
+ } else {
+ return Bytes::toString;
+ }
+ }
+
public static void scanFluo(ScanOpts options, FluoConfiguration sConfig, PrintStream out)
throws IOException {
@@ -123,152 +121,56 @@ public class ScanUtil {
Span span = getSpan(options);
Collection<Column> columns = getColumns(options);
+ CellScanner cellScanner = s.scanner().over(span).fetch(columns).build();
+ Function<Bytes, String> encoder = getEncoder(options);
if (options.exportAsJson) {
- generateJson(options, span, columns, s, out);
- } else { // TSV or CSV format
- generateTsvCsv(options, span, columns, s, out);
- }
-
- } catch (FluoException e) {
- throw e;
- }
- }
- }
-
- /**
- * Generate TSV or CSV format as result of the scan.
- *
- * @since 1.2
- */
- private static void generateTsvCsv(ScanOpts options, Span span, Collection<Column> columns,
- final Snapshot snapshot, PrintStream out) throws IOException {
- // CSV Formater
- CSVFormat csvFormat = CSVFormat.DEFAULT;
- csvFormat = csvFormat.withQuoteMode(QuoteMode.ALL);
- csvFormat = csvFormat.withRecordSeparator("\n");
-
- // when "--csv" parameter is passed the "fluo.scan.csv" is analised
- if (options.exportAsCsv) {
- if (StringUtils.isNotEmpty(options.csvDelimiter)) {
- if (options.csvDelimiter.length() > 1) {
- throw new IllegalArgumentException(
- "Invalid character for the \"--csv-delimiter\" parameter.");
- }
- csvFormat = csvFormat.withDelimiter(options.csvDelimiter.charAt(0));
- }
-
- if (StringUtils.isNotEmpty(options.csvEscape)) {
- if (options.csvEscape.length() > 1) {
- throw new IllegalArgumentException(
- "Invalid character for the \"--csv-escape\" parameter.");
- }
- csvFormat = csvFormat.withEscape(options.csvEscape.charAt(0));
- }
-
- if (StringUtils.isNotEmpty(options.csvQuote)) {
- if (options.csvQuote.length() > 1) {
- throw new IllegalArgumentException(
- "Invalid character for the \"--csv-quote\" parameter.");
- }
- csvFormat = csvFormat.withQuote(options.csvQuote.charAt(0));
- }
-
- // It can throw "java.lang.IllegalArgumentException" if the value not exists
- // in "org.apache.commons.csv.QuoteMode"
- if (StringUtils.isNotEmpty(options.csvQuoteMode)) {
- csvFormat = csvFormat.withQuoteMode(QuoteMode.valueOf(options.csvQuoteMode));
- }
-
- if (BooleanUtils.toBooleanObject(
- ObjectUtils.defaultIfNull(options.csvHeader, Boolean.FALSE.toString()))) {
- csvFormat = csvFormat.withHeader(FLUO_ROW, FLUO_COLUMN_FAMILY, FLUO_COLUMN_QUALIFIER,
- FLUO_COLUMN_VISIBILITY, FLUO_VALUE);
- }
- } else {
- // Default TAB separator and NO quotes if possible.
- csvFormat = csvFormat.withDelimiter(CSVFormat.TDF.getDelimiter());
- csvFormat = csvFormat.withQuoteMode(QuoteMode.MINIMAL);
- }
-
- try (CSVPrinter printer = new CSVPrinter(out, csvFormat)) {
- CellScanner cellScanner = snapshot.scanner().over(span).fetch(columns).build();
-
- List<Object> record = new LinkedList<>();
- StringBuilder sb = new StringBuilder();
- int lines2check = 0;
- for (RowColumnValue rcv : cellScanner) {
- record.clear();
- if (options.hexEncNonAscii) {
- sb.setLength(0);
- Hex.encNonAscii(sb, rcv.getRow());
- record.add(sb.toString());
- sb.setLength(0);
- Hex.encNonAscii(sb, rcv.getColumn().getFamily());
- record.add(sb.toString());
- sb.setLength(0);
- Hex.encNonAscii(sb, rcv.getColumn().getQualifier());
- record.add(sb.toString());
- sb.setLength(0);
- Hex.encNonAscii(sb, rcv.getColumn().getVisibility());
- record.add(sb.toString());
- sb.setLength(0);
- Hex.encNonAscii(sb, rcv.getValue());
- record.add(sb.toString());
+ generateJson(cellScanner, encoder, out);
} else {
- record.add(rcv.getsRow());
- record.add(rcv.getColumn().getFamily());
- record.add(rcv.getColumn().getQualifier());
- record.add(rcv.getColumn().getVisibility());
- record.add(rcv.getsValue());
- }
-
- printer.printRecord(record);
- lines2check++;
- if (lines2check == 100) {
- lines2check = 0;
- if (out.checkError()) {
- throw new IOException("Fail to write data to stream.");
+ for (RowColumnValue rcv : cellScanner) {
+ out.print(encoder.apply(rcv.getRow()));
+ out.print(' ');
+ out.print(encoder.apply(rcv.getColumn().getFamily()));
+ out.print(' ');
+ out.print(encoder.apply(rcv.getColumn().getQualifier()));
+ out.print(' ');
+ out.print(encoder.apply(rcv.getColumn().getVisibility()));
+ out.print("\t");
+ out.print(encoder.apply(rcv.getValue()));
+ out.println();
+ if (out.checkError()) {
+ break;
+ }
}
}
}
}
- out.flush();
}
/**
* Generate JSON format as result of the scan.
- *
+ *
* @since 1.2
*/
- private static void generateJson(ScanOpts options, Span span, Collection<Column> columns,
- final Snapshot snapshot, PrintStream out) throws JsonIOException {
+ private static void generateJson(CellScanner cellScanner, Function<Bytes, String> encoder,
+ PrintStream out) throws JsonIOException {
Gson gson = new GsonBuilder().serializeNulls().setDateFormat(DateFormat.LONG)
.setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES).setVersion(1.0)
.create();
- CellScanner cellScanner = snapshot.scanner().over(span).fetch(columns).build();
-
- StringBuilder sb = new StringBuilder();
Map<String, String> json = new LinkedHashMap<>();
for (RowColumnValue rcv : cellScanner) {
- sb.setLength(0);
- Hex.encNonAscii(sb, rcv.getRow());
- json.put(FLUO_ROW, sb.toString());
- sb.setLength(0);
- Hex.encNonAscii(sb, rcv.getColumn().getFamily());
- json.put(FLUO_COLUMN_FAMILY, sb.toString());
- Hex.encNonAscii(sb, rcv.getColumn().getQualifier());
- json.put(FLUO_COLUMN_QUALIFIER, sb.toString());
- sb.setLength(0);
- Hex.encNonAscii(sb, rcv.getColumn().getVisibility());
- json.put(FLUO_COLUMN_VISIBILITY, sb.toString());
- sb.setLength(0);
- Hex.encNonAscii(sb, rcv.getValue());
- json.put(FLUO_VALUE, sb.toString());
-
+ json.put(FLUO_ROW, encoder.apply(rcv.getRow()));
+ json.put(FLUO_COLUMN_FAMILY, encoder.apply(rcv.getColumn().getFamily()));
+ json.put(FLUO_COLUMN_QUALIFIER, encoder.apply(rcv.getColumn().getQualifier()));
+ json.put(FLUO_COLUMN_VISIBILITY, encoder.apply(rcv.getColumn().getVisibility()));
+ json.put(FLUO_VALUE, encoder.apply(rcv.getValue()));
gson.toJson(json, out);
out.append("\n");
+
+ if (out.checkError()) {
+ break;
+ }
}
out.flush();
}
@@ -311,18 +213,11 @@ public class ScanUtil {
public boolean help;
public boolean hexEncNonAscii = true;
public boolean scanAccumuloTable = false;
- public boolean exportAsCsv = false;
public boolean exportAsJson = false;
- public final String csvDelimiter;
- public final String csvEscape;
- public final String csvHeader;
- public final String csvQuote;
- public final String csvQuoteMode;
public ScanOpts(String startRow, String endRow, List<String> columns, String exactRow,
String rowPrefix, boolean help, boolean hexEncNonAscii, boolean scanAccumuloTable,
- boolean exportAsCsv, String csvDelimiter, String csvEscape, String csvHeader,
- String csvQuote, String csvQuoteMode, boolean exportAsJson) {
+ boolean exportAsJson) {
this.startRow = startRow;
this.endRow = endRow;
this.columns = columns;
@@ -331,12 +226,6 @@ public class ScanUtil {
this.help = help;
this.hexEncNonAscii = hexEncNonAscii;
this.scanAccumuloTable = scanAccumuloTable;
- this.exportAsCsv = exportAsCsv;
- this.csvDelimiter = csvDelimiter;
- this.csvEscape = csvEscape;
- this.csvHeader = csvHeader;
- this.csvQuote = csvQuote;
- this.csvQuoteMode = csvQuoteMode;
this.exportAsJson = exportAsJson;
}
diff --git a/modules/distribution/src/main/lib/fetch.sh b/modules/distribution/src/main/lib/fetch.sh
index 51a54f4..2dd523c 100755
--- a/modules/distribution/src/main/lib/fetch.sh
+++ b/modules/distribution/src/main/lib/fetch.sh
@@ -62,8 +62,6 @@ extra)
download commons-collections:commons-collections:jar:3.2.1
download commons-configuration:commons-configuration:jar:1.10
download commons-io:commons-io:jar:2.4
- download org.apache.commons:commons-csv:jar:1.5
- download org.apache.commons:commons-lang3:jar:3.7
download io.dropwizard.metrics:metrics-core:jar:3.1.1
download io.dropwizard.metrics:metrics-graphite:jar:3.1.1
download javax.inject:javax.inject:jar:1
diff --git a/pom.xml b/pom.xml
index 7307daa..c4f8f3e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -140,16 +140,6 @@
<version>${accumulo.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-csv</artifactId>
- <version>1.5</version>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- <version>3.7</version>
- </dependency>
- <dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-client</artifactId>
<version>${curator.version}</version>
--
To stop receiving notification emails like this one, please contact
kturner@apache.org.
[fluo] 03/03: Merge branch 'fluo-984-2'
Posted by kt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/fluo.git
commit aae4b2b3ad954c07bbfd0ea98c3a4975bb019bcb
Merge: c35dea1 f147016
Author: Keith Turner <kt...@apache.org>
AuthorDate: Wed Apr 4 10:28:17 2018 -0400
Merge branch 'fluo-984-2'
.../org/apache/fluo/cluster/runner/AppRunner.java | 16 ++-
.../java/org/apache/fluo/command/FluoScan.java | 39 ++++--
.../java/org/apache/fluo/core/util/ScanUtil.java | 137 +++++++++++++--------
modules/distribution/src/main/lib/fetch.sh | 7 +-
pom.xml | 4 +-
5 files changed, 132 insertions(+), 71 deletions(-)
--
To stop receiving notification emails like this one, please contact
kturner@apache.org.
[fluo] 01/03: Scan command now can produce results as csv and json.
#984 #1018
Posted by kt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/fluo.git
commit 0ec21c92153b916f92ca44b38dca0700ac09d841
Author: Alan Camillo <al...@blueshift.com.br>
AuthorDate: Wed Apr 4 10:23:02 2018 -0400
Scan command now can produce results as csv and json. #984 #1018
---
.../org/apache/fluo/cluster/runner/AppRunner.java | 25 ++-
modules/command/pom.xml | 4 +
.../java/org/apache/fluo/command/FluoScan.java | 89 +++++++-
modules/core/pom.xml | 8 +
.../java/org/apache/fluo/core/util/ScanUtil.java | 244 ++++++++++++++++-----
modules/distribution/src/main/lib/fetch.sh | 9 +-
pom.xml | 14 +-
7 files changed, 326 insertions(+), 67 deletions(-)
diff --git a/modules/cluster/src/main/java/org/apache/fluo/cluster/runner/AppRunner.java b/modules/cluster/src/main/java/org/apache/fluo/cluster/runner/AppRunner.java
index 9de17f4..5be9ef1 100644
--- a/modules/cluster/src/main/java/org/apache/fluo/cluster/runner/AppRunner.java
+++ b/modules/cluster/src/main/java/org/apache/fluo/cluster/runner/AppRunner.java
@@ -15,6 +15,7 @@
package org.apache.fluo.cluster.runner;
+import java.io.IOException;
import java.lang.reflect.Method;
import java.util.Arrays;
import java.util.Collections;
@@ -72,10 +73,15 @@ public abstract class AppRunner {
System.exit(0);
}
- if (options.scanAccumuloTable) {
- ScanUtil.scanAccumulo(options.getScanOpts(), config);
- } else {
- ScanUtil.scanFluo(options.getScanOpts(), config);
+ try {
+ if (options.scanAccumuloTable) {
+ ScanUtil.scanAccumulo(options.getScanOpts(), config, System.out);
+ } else {
+ ScanUtil.scanFluo(options.getScanOpts(), config, System.out);
+ }
+ } catch (IOException e) {
+ System.err.println(e.getMessage());
+ System.exit(-1);
}
}
@@ -203,6 +209,14 @@ public abstract class AppRunner {
+ "internal schema, making it easier to comprehend.")
public boolean scanAccumuloTable = false;
+ public boolean exportAsCsv = false;
+ public String csvHeader;
+ public String csvDelimiter;
+ public String csvEscape;
+ public String csvQuote;
+ public String csvQuoteMode;
+ public boolean exportAsJson = false;
+
public String getStartRow() {
return startRow;
}
@@ -228,7 +242,8 @@ public abstract class AppRunner {
public ScanUtil.ScanOpts getScanOpts() {
return new ScanUtil.ScanOpts(startRow, endRow, columns, exactRow, rowPrefix, help,
- hexEncNonAscii, scanAccumuloTable);
+ hexEncNonAscii, scanAccumuloTable, exportAsCsv, csvDelimiter, csvEscape, csvHeader,
+ csvQuote, csvQuoteMode, exportAsJson);
}
}
}
diff --git a/modules/command/pom.xml b/modules/command/pom.xml
index 8c8bf46..f9949a2 100644
--- a/modules/command/pom.xml
+++ b/modules/command/pom.xml
@@ -47,6 +47,10 @@
<artifactId>accumulo-core</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-framework</artifactId>
</dependency>
diff --git a/modules/command/src/main/java/org/apache/fluo/command/FluoScan.java b/modules/command/src/main/java/org/apache/fluo/command/FluoScan.java
index 4d56194..9f4455a 100644
--- a/modules/command/src/main/java/org/apache/fluo/command/FluoScan.java
+++ b/modules/command/src/main/java/org/apache/fluo/command/FluoScan.java
@@ -15,10 +15,12 @@
package org.apache.fluo.command;
+import java.io.IOException;
import java.util.Collections;
import java.util.List;
import com.beust.jcommander.Parameter;
+import org.apache.commons.lang3.StringUtils;
import org.apache.fluo.api.config.FluoConfiguration;
import org.apache.fluo.core.client.FluoAdminImpl;
import org.apache.fluo.core.util.ScanUtil;
@@ -54,6 +56,35 @@ public class FluoScan {
+ "internal schema, making it easier to comprehend.")
public boolean scanAccumuloTable = false;
+ @Parameter(names = "--csv", help = true,
+ description = "Export key/values stored in Accumulo as CSV file. Uses Fluo application "
+ + "properties to configure the CSV format.")
+ public boolean exportAsCsv = false;
+
+ @Parameter(names = "--csv-header", help = true, description = "Set header for \"true\".")
+ public String csvHeader;
+
+ @Parameter(names = "--csv-delimiter", help = true,
+ description = "Configure delimiter to a designeted character.")
+ public String csvDelimiter;
+
+ @Parameter(names = "--csv-escape", help = true,
+ description = "Configure escape to a designeted character.")
+ public String csvEscape;
+
+ @Parameter(names = "--csv-quote", help = true,
+ description = "Configure quote to a designeted character.")
+ public String csvQuote;
+
+ @Parameter(names = "--csv-quote-mode", help = true,
+ description = "Configure quote mode to a designeted mode. The possible "
+ + "modes are: ALL, ALL_NON_NULL, MINIMAL, NONE and NON_NUMERIC")
+ public String csvQuoteMode;
+
+ @Parameter(names = "--json", help = true,
+ description = "Export key/values stored in Accumulo as JSON file.")
+ public boolean exportAsJson = false;
+
public String getStartRow() {
return startRow;
}
@@ -70,6 +101,26 @@ public class FluoScan {
return rowPrefix;
}
+ public String getCsvHeader() {
+ return csvHeader;
+ }
+
+ public String getCsvDelimiter() {
+ return csvDelimiter;
+ }
+
+ public String getCsvEscape() {
+ return csvEscape;
+ }
+
+ public String getCsvQuote() {
+ return csvQuote;
+ }
+
+ public String getCsvQuoteMode() {
+ return csvQuoteMode;
+ }
+
public List<String> getColumns() {
if (columns == null) {
return Collections.emptyList();
@@ -77,9 +128,27 @@ public class FluoScan {
return columns;
}
+ /**
+ * Check if the parameters informed can be used together.
+ * @since 1.2
+ */
+ private void checkScanOptions() {
+ if (this.exportAsCsv && this.exportAsJson) {
+ throw new IllegalArgumentException(
+ "Both \"--csv\" and \"--json\" can not be set together.");
+ }
+
+ if (!this.exportAsCsv && (StringUtils.isNotEmpty(this.csvDelimiter)
+ | StringUtils.isNotEmpty(this.csvEscape) | StringUtils.isNotEmpty(this.csvHeader)
+ | StringUtils.isNotEmpty(this.csvQuote) | StringUtils.isNotEmpty(this.csvQuoteMode))) {
+ throw new IllegalArgumentException("No \"--csv\" detected");
+ }
+ }
+
public ScanUtil.ScanOpts getScanOpts() {
return new ScanUtil.ScanOpts(startRow, endRow, columns, exactRow, rowPrefix, help,
- hexEncNonAscii, scanAccumuloTable);
+ hexEncNonAscii, scanAccumuloTable, exportAsCsv, csvDelimiter, csvEscape, csvHeader,
+ csvQuote, csvQuoteMode, exportAsJson);
}
public static ScanOptions parse(String[] args) {
@@ -95,16 +164,24 @@ public class FluoScan {
Logger.getLogger("org.apache.fluo").setLevel(Level.ERROR);
ScanOptions options = ScanOptions.parse(args);
+ options.checkScanOptions();
FluoConfiguration config = CommandUtil.resolveFluoConfig();
config.setApplicationName(options.getApplicationName());
options.overrideFluoConfig(config);
CommandUtil.verifyAppRunning(config);
- if (options.scanAccumuloTable) {
- config = FluoAdminImpl.mergeZookeeperConfig(config);
- ScanUtil.scanAccumulo(options.getScanOpts(), config);
- } else {
- ScanUtil.scanFluo(options.getScanOpts(), config);
+ try {
+ options.overrideFluoConfig(config);
+ if (options.scanAccumuloTable) {
+ config = FluoAdminImpl.mergeZookeeperConfig(config);
+ ScanUtil.scanAccumulo(options.getScanOpts(), config, System.out);
+ } else {
+ ScanUtil.scanFluo(options.getScanOpts(), config, System.out);
+ }
+ } catch (RuntimeException | IOException e) {
+ System.err.println("Scan failed - " + e.getMessage());
+ System.exit(-1);
}
}
+
}
diff --git a/modules/core/pom.xml b/modules/core/pom.xml
index fdd1f69..0cfc6b3 100644
--- a/modules/core/pom.xml
+++ b/modules/core/pom.xml
@@ -50,6 +50,14 @@
<artifactId>accumulo-core</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-csv</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-client</artifactId>
</dependency>
diff --git a/modules/core/src/main/java/org/apache/fluo/core/util/ScanUtil.java b/modules/core/src/main/java/org/apache/fluo/core/util/ScanUtil.java
index 46aacca..78c3efb 100644
--- a/modules/core/src/main/java/org/apache/fluo/core/util/ScanUtil.java
+++ b/modules/core/src/main/java/org/apache/fluo/core/util/ScanUtil.java
@@ -15,15 +15,31 @@
package org.apache.fluo.core.util;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.text.DateFormat;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
import java.util.List;
+import java.util.Map;
import com.google.common.collect.Iterables;
+import com.google.gson.FieldNamingPolicy;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonIOException;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.security.Authorizations;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVPrinter;
+import org.apache.commons.csv.QuoteMode;
+import org.apache.commons.lang3.BooleanUtils;
+import org.apache.commons.lang3.ObjectUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.fluo.accumulo.format.FluoFormatter;
import org.apache.fluo.api.client.FluoClient;
import org.apache.fluo.api.client.FluoFactory;
@@ -37,6 +53,16 @@ import org.apache.fluo.api.data.Span;
import org.apache.fluo.api.exceptions.FluoException;
public class ScanUtil {
+ public static final String CSV_HEADER = "csv.header";
+ public static final String CSV_QUOTE_MODE = "csv.quoteMode";
+ public static final String CSV_QUOTE = "csv.quote";
+ public static final String CSV_ESCAPE = "csv.escape";
+ public static final String CSV_DELIMITER = "csv.delimiter";
+ public static final String FLUO_VALUE = "value";
+ public static final String FLUO_COLUMN_VISIBILITY = "visibility";
+ public static final String FLUO_COLUMN_QUALIFIER = "qualifier";
+ public static final String FLUO_COLUMN_FAMILY = "family";
+ public static final String FLUO_ROW = "row";
public static Span getSpan(ScanOpts options) {
Span span = new Span();
@@ -89,68 +115,170 @@ public class ScanUtil {
return columns;
}
- public static void scanFluo(ScanOpts options, FluoConfiguration sConfig) {
+ public static void scanFluo(ScanOpts options, FluoConfiguration sConfig, PrintStream out)
+ throws IOException {
try (FluoClient client = FluoFactory.newClient(sConfig)) {
try (Snapshot s = client.newSnapshot()) {
- Span span = null;
- Collection<Column> columns = null;
- try {
- span = getSpan(options);
- columns = getColumns(options);
- } catch (IllegalArgumentException e) {
- System.err.println(e.getMessage());
- System.exit(-1);
+ Span span = getSpan(options);
+ Collection<Column> columns = getColumns(options);
+
+ if (options.exportAsJson) {
+ generateJson(options, span, columns, s, out);
+ } else { // TSV or CSV format
+ generateTsvCsv(options, span, columns, s, out);
}
- CellScanner cellScanner = s.scanner().over(span).fetch(columns).build();
-
- StringBuilder sb = new StringBuilder();
- for (RowColumnValue rcv : cellScanner) {
- if (options.hexEncNonAscii) {
- sb.setLength(0);
- Hex.encNonAscii(sb, rcv.getRow());
- sb.append(" ");
- Hex.encNonAscii(sb, rcv.getColumn(), " ");
- sb.append("\t");
- Hex.encNonAscii(sb, rcv.getValue());
- System.out.println(sb.toString());
- } else {
- sb.setLength(0);
- sb.append(rcv.getsRow());
- sb.append(" ");
- sb.append(rcv.getColumn());
- sb.append("\t");
- sb.append(rcv.getsValue());
- System.out.println(sb.toString());
- }
+ } catch (FluoException e) {
+ throw e;
+ }
+ }
+ }
- if (System.out.checkError()) {
- break;
- }
+ /**
+ * Generate TSV or CSV format as result of the scan.
+ *
+ * @since 1.2
+ */
+ private static void generateTsvCsv(ScanOpts options, Span span, Collection<Column> columns,
+ final Snapshot snapshot, PrintStream out) throws IOException {
+ // CSV Formater
+ CSVFormat csvFormat = CSVFormat.DEFAULT;
+ csvFormat = csvFormat.withQuoteMode(QuoteMode.ALL);
+ csvFormat = csvFormat.withRecordSeparator("\n");
+
+ // when "--csv" parameter is passed the "fluo.scan.csv" is analised
+ if (options.exportAsCsv) {
+ if (StringUtils.isNotEmpty(options.csvDelimiter)) {
+ if (options.csvDelimiter.length() > 1) {
+ throw new IllegalArgumentException(
+ "Invalid character for the \"--csv-delimiter\" parameter.");
+ }
+ csvFormat = csvFormat.withDelimiter(options.csvDelimiter.charAt(0));
+ }
+
+ if (StringUtils.isNotEmpty(options.csvEscape)) {
+ if (options.csvEscape.length() > 1) {
+ throw new IllegalArgumentException(
+ "Invalid character for the \"--csv-escape\" parameter.");
}
+ csvFormat = csvFormat.withEscape(options.csvEscape.charAt(0));
+ }
- } catch (FluoException e) {
- System.err.println("Scan failed - " + e.getMessage());
- System.exit(-1);
+ if (StringUtils.isNotEmpty(options.csvQuote)) {
+ if (options.csvQuote.length() > 1) {
+ throw new IllegalArgumentException(
+ "Invalid character for the \"--csv-quote\" parameter.");
+ }
+ csvFormat = csvFormat.withQuote(options.csvQuote.charAt(0));
+ }
+
+ // It can throw "java.lang.IllegalArgumentException" if the value not exists
+ // in "org.apache.commons.csv.QuoteMode"
+ if (StringUtils.isNotEmpty(options.csvQuoteMode)) {
+ csvFormat = csvFormat.withQuoteMode(QuoteMode.valueOf(options.csvQuoteMode));
+ }
+
+ if (BooleanUtils.toBooleanObject(
+ ObjectUtils.defaultIfNull(options.csvHeader, Boolean.FALSE.toString()))) {
+ csvFormat = csvFormat.withHeader(FLUO_ROW, FLUO_COLUMN_FAMILY, FLUO_COLUMN_QUALIFIER,
+ FLUO_COLUMN_VISIBILITY, FLUO_VALUE);
+ }
+ } else {
+ // Default TAB separator and NO quotes if possible.
+ csvFormat = csvFormat.withDelimiter(CSVFormat.TDF.getDelimiter());
+ csvFormat = csvFormat.withQuoteMode(QuoteMode.MINIMAL);
+ }
+
+ try (CSVPrinter printer = new CSVPrinter(out, csvFormat)) {
+ CellScanner cellScanner = snapshot.scanner().over(span).fetch(columns).build();
+
+ List<Object> record = new LinkedList<>();
+ StringBuilder sb = new StringBuilder();
+ int lines2check = 0;
+ for (RowColumnValue rcv : cellScanner) {
+ record.clear();
+ if (options.hexEncNonAscii) {
+ sb.setLength(0);
+ Hex.encNonAscii(sb, rcv.getRow());
+ record.add(sb.toString());
+ sb.setLength(0);
+ Hex.encNonAscii(sb, rcv.getColumn().getFamily());
+ record.add(sb.toString());
+ sb.setLength(0);
+ Hex.encNonAscii(sb, rcv.getColumn().getQualifier());
+ record.add(sb.toString());
+ sb.setLength(0);
+ Hex.encNonAscii(sb, rcv.getColumn().getVisibility());
+ record.add(sb.toString());
+ sb.setLength(0);
+ Hex.encNonAscii(sb, rcv.getValue());
+ record.add(sb.toString());
+ } else {
+ record.add(rcv.getsRow());
+ record.add(rcv.getColumn().getFamily());
+ record.add(rcv.getColumn().getQualifier());
+ record.add(rcv.getColumn().getVisibility());
+ record.add(rcv.getsValue());
+ }
+
+ printer.printRecord(record);
+ lines2check++;
+ if (lines2check == 100) {
+ lines2check = 0;
+ if (out.checkError()) {
+ throw new IOException("Fail to write data to stream.");
+ }
+ }
}
}
+ out.flush();
}
- public static void scanAccumulo(ScanOpts options, FluoConfiguration sConfig) {
+ /**
+ * Generate JSON format as result of the scan.
+ *
+ * @since 1.2
+ */
+ private static void generateJson(ScanOpts options, Span span, Collection<Column> columns,
+ final Snapshot snapshot, PrintStream out) throws JsonIOException {
+ Gson gson = new GsonBuilder().serializeNulls().setDateFormat(DateFormat.LONG)
+ .setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES).setVersion(1.0)
+ .create();
- Connector conn = AccumuloUtil.getConnector(sConfig);
+ CellScanner cellScanner = snapshot.scanner().over(span).fetch(columns).build();
- Span span = null;
- Collection<Column> columns = null;
- try {
- span = getSpan(options);
- columns = getColumns(options);
- } catch (IllegalArgumentException e) {
- System.err.println(e.getMessage());
- System.exit(-1);
+ StringBuilder sb = new StringBuilder();
+ Map<String, String> json = new LinkedHashMap<>();
+ for (RowColumnValue rcv : cellScanner) {
+ sb.setLength(0);
+ Hex.encNonAscii(sb, rcv.getRow());
+ json.put(FLUO_ROW, sb.toString());
+ sb.setLength(0);
+ Hex.encNonAscii(sb, rcv.getColumn().getFamily());
+ json.put(FLUO_COLUMN_FAMILY, sb.toString());
+ Hex.encNonAscii(sb, rcv.getColumn().getQualifier());
+ json.put(FLUO_COLUMN_QUALIFIER, sb.toString());
+ sb.setLength(0);
+ Hex.encNonAscii(sb, rcv.getColumn().getVisibility());
+ json.put(FLUO_COLUMN_VISIBILITY, sb.toString());
+ sb.setLength(0);
+ Hex.encNonAscii(sb, rcv.getValue());
+ json.put(FLUO_VALUE, sb.toString());
+
+ gson.toJson(json, out);
+ out.append("\n");
}
+ out.flush();
+ }
+
+ public static void scanAccumulo(ScanOpts options, FluoConfiguration sConfig, PrintStream out) {
+
+ Connector conn = AccumuloUtil.getConnector(sConfig);
+
+ Span span = getSpan(options);
+ Collection<Column> columns = getColumns(options);
try {
Scanner scanner = conn.createScanner(sConfig.getAccumuloTable(), Authorizations.EMPTY);
@@ -165,11 +293,11 @@ public class ScanUtil {
}
for (String entry : Iterables.transform(scanner, FluoFormatter::toString)) {
- System.out.println(entry);
+ out.println(entry);
}
+ out.flush();
} catch (Exception e) {
- System.err.println("Scan failed - " + e.getMessage());
- System.exit(-1);
+ throw new RuntimeException(e);
}
}
@@ -183,9 +311,18 @@ public class ScanUtil {
public boolean help;
public boolean hexEncNonAscii = true;
public boolean scanAccumuloTable = false;
+ public boolean exportAsCsv = false;
+ public boolean exportAsJson = false;
+ public final String csvDelimiter;
+ public final String csvEscape;
+ public final String csvHeader;
+ public final String csvQuote;
+ public final String csvQuoteMode;
public ScanOpts(String startRow, String endRow, List<String> columns, String exactRow,
- String rowPrefix, boolean help, boolean hexEncNonAscii, boolean scanAccumuloTable) {
+ String rowPrefix, boolean help, boolean hexEncNonAscii, boolean scanAccumuloTable,
+ boolean exportAsCsv, String csvDelimiter, String csvEscape, String csvHeader,
+ String csvQuote, String csvQuoteMode, boolean exportAsJson) {
this.startRow = startRow;
this.endRow = endRow;
this.columns = columns;
@@ -194,6 +331,13 @@ public class ScanUtil {
this.help = help;
this.hexEncNonAscii = hexEncNonAscii;
this.scanAccumuloTable = scanAccumuloTable;
+ this.exportAsCsv = exportAsCsv;
+ this.csvDelimiter = csvDelimiter;
+ this.csvEscape = csvEscape;
+ this.csvHeader = csvHeader;
+ this.csvQuote = csvQuote;
+ this.csvQuoteMode = csvQuoteMode;
+ this.exportAsJson = exportAsJson;
}
public String getStartRow() {
diff --git a/modules/distribution/src/main/lib/fetch.sh b/modules/distribution/src/main/lib/fetch.sh
index d945d83..51a54f4 100755
--- a/modules/distribution/src/main/lib/fetch.sh
+++ b/modules/distribution/src/main/lib/fetch.sh
@@ -17,7 +17,7 @@ lib_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
maven_prefix=https://repo1.maven.org/maven2
function download {
- IFS=':' read -ra DEP <<< "$1"
+ IFS=':' read -ra DEP <<< "$1"
dir=$lib_dir/
if [ -n "$2" ]; then
dir=$lib_dir/$2
@@ -35,7 +35,7 @@ function download {
if [ -f $dir/$fn ]; then
echo "SUCCESS: Dependency exists - $dir/$fn"
- else
+ else
wget -q $download_url -P $dir
if [ $? == 0 ]; then
echo "SUCCESS: Dependency downloaded from $download_url"
@@ -56,12 +56,14 @@ extra)
echo "Fetching extra Fluo dependencies"
download aopalliance:aopalliance:jar:1.0
download com.beust:jcommander:jar:1.32
- download com.google.code.gson:gson:jar:2.2.4
+ download com.google.code.gson:gson:jar:2.8.0
download com.google.guava:guava:jar:13.0.1
download com.google.inject:guice:jar:4.0
download commons-collections:commons-collections:jar:3.2.1
download commons-configuration:commons-configuration:jar:1.10
download commons-io:commons-io:jar:2.4
+ download org.apache.commons:commons-csv:jar:1.5
+ download org.apache.commons:commons-lang3:jar:3.7
download io.dropwizard.metrics:metrics-core:jar:3.1.1
download io.dropwizard.metrics:metrics-graphite:jar:3.1.1
download javax.inject:javax.inject:jar:1
@@ -117,4 +119,3 @@ extra)
echo -e "However, you can override them using the command below:\n"
echo "./fetch.sh ahz -Daccumulo.version=1.7.2 -Dhadoop.version=2.7.2 -Dzookeeper.version=3.4.8"
esac
-
diff --git a/pom.xml b/pom.xml
index ccd069e..7307daa 100644
--- a/pom.xml
+++ b/pom.xml
@@ -5,9 +5,9 @@
copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance with the License. You may obtain a
copy of the License at
-
+
http://www.apache.org/licenses/LICENSE-2.0
-
+
Unless required by applicable law or agreed to in writing, software distributed under the License
is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
or implied. See the License for the specific language governing permissions and limitations under
@@ -140,6 +140,16 @@
<version>${accumulo.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-csv</artifactId>
+ <version>1.5</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <version>3.7</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-client</artifactId>
<version>${curator.version}</version>
--
To stop receiving notification emails like this one, please contact
kturner@apache.org.