You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by do...@apache.org on 2019/04/22 06:02:55 UTC
[incubator-iotdb] branch master updated: [IOTDB-83] Add process bar
for import script and show how many rows have been exported (#165)
This is an automated email from the ASF dual-hosted git repository.
dope pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 519e398 [IOTDB-83] Add process bar for import script and show how many rows have been exported (#165)
519e398 is described below
commit 519e3980d19ebd6a5f413a1af39d90b9777652ea
Author: XuYi <My...@users.noreply.github.com>
AuthorDate: Mon Apr 22 14:02:50 2019 +0800
[IOTDB-83] Add process bar for import script and show how many rows have been exported (#165)
---
License | 1 +
clean.sh | 36 -------------------
iotdb-cli/pom.xml | 5 +++
.../java/org/apache/iotdb/cli/tool/ExportCsv.java | 24 ++++++++++---
.../java/org/apache/iotdb/cli/tool/ImportCsv.java | 40 ++++++++++++++++------
5 files changed, 55 insertions(+), 51 deletions(-)
diff --git a/License b/License
index 1ea4af3..58f5c09 100644
--- a/License
+++ b/License
@@ -255,6 +255,7 @@ MIT License
------------
org.slf4j:slf4j-api
org.mockito:mockito-all:1.10.19
+me.tongfei:progressbar:0.7.3
EDL 1.0
diff --git a/clean.sh b/clean.sh
deleted file mode 100644
index 0483bdc..0000000
--- a/clean.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/bin/sh
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-git checkout checkstyle.xml
-git checkout java-google-style.xml
-git checkout example/**/pom.xml
-git checkout grafana/pom.xml
-git checkout hadoop/pom.xml
-git checkout example/pom.xml
-git checkout jdbc/pom.xml
-git checkout service-rpc/pom.xml
-git checkout spark/pom.xml
-git checkout tsfile/pom.xml
-git checkout iotdb-cli/pom.xml
-git checkout iotdb/pom.xml
-git checkout pom.xml
-git checkout iotdb-cli/src/test/resources/logback.xml
-git checkout iotdb/iotdb/conf/logback.xml
-git checkout iotdb/src/test/resources/logback.xml
-git checkout tsfile/src/test/resources/logback.xml
diff --git a/iotdb-cli/pom.xml b/iotdb-cli/pom.xml
index 6977892..a6634e5 100644
--- a/iotdb-cli/pom.xml
+++ b/iotdb-cli/pom.xml
@@ -54,6 +54,11 @@
<version>3.1</version>
</dependency>
<dependency>
+ <groupId>me.tongfei</groupId>
+ <artifactId>progressbar</artifactId>
+ <version>0.7.3</version>
+ </dependency>
+ <dependency>
<groupId>jline</groupId>
<artifactId>jline</artifactId>
<version>${jline.version}</version>
diff --git a/iotdb-cli/src/main/java/org/apache/iotdb/cli/tool/ExportCsv.java b/iotdb-cli/src/main/java/org/apache/iotdb/cli/tool/ExportCsv.java
index 4762519..f9d7012 100644
--- a/iotdb-cli/src/main/java/org/apache/iotdb/cli/tool/ExportCsv.java
+++ b/iotdb-cli/src/main/java/org/apache/iotdb/cli/tool/ExportCsv.java
@@ -68,6 +68,8 @@ public class ExportCsv extends AbstractCsvTool {
private static String targetFile = DUMP_FILE_NAME_DEFAULT;
private static String targetDirectory;
+
+ private static final int EXPORT_PER_LINE_COUNT = 10000;
/**
* main function of export csv tool.
@@ -147,6 +149,9 @@ public class ExportCsv extends AbstractCsvTool {
throws ArgsErrorException {
targetDirectory = checkRequiredArg(TARGET_DIR_ARGS, TARGET_DIR_NAME, commandLine);
targetFile = commandLine.getOptionValue(TARGET_FILE_ARGS);
+ if(targetFile == null){
+ targetFile = DUMP_FILE_NAME_DEFAULT;
+ }
timeFormat = commandLine.getOptionValue(TIME_FORMAT_ARGS);
if (timeFormat == null) {
timeFormat = "default";
@@ -252,7 +257,7 @@ public class ExportCsv extends AbstractCsvTool {
System.out.println("Cannot create dump file "+ path + "because: " + e.getMessage());
return;
}
-
+ System.out.println("Start to export data from sql statement: "+ sql);
try (Statement statement = connection.createStatement();
ResultSet rs = statement.executeQuery(sql);
BufferedWriter bw = new BufferedWriter(new FileWriter(tf))) {
@@ -263,9 +268,9 @@ public class ExportCsv extends AbstractCsvTool {
// write data in csv file
writeMetadata(bw, count, metadata);
- writeResultSet(rs, bw, count);
- System.out.println(String.format("Statement [%s] has dumped to file %s successfully! "
- + "It costs %dms.", sql, path, System.currentTimeMillis() - startTime));
+ int line = writeResultSet(rs, bw, count);
+ System.out.println(String.format("Statement [%s] has dumped to file %s successfully! It costs "
+ + "%dms to export %d lines.", sql, path, System.currentTimeMillis() - startTime, line));
} catch (IOException e) {
System.out.println("Cannot dump result because: " + e.getMessage());
}
@@ -282,8 +287,10 @@ public class ExportCsv extends AbstractCsvTool {
}
}
- private static void writeResultSet(ResultSet rs, BufferedWriter bw, int count)
+ private static int writeResultSet(ResultSet rs, BufferedWriter bw, int count)
throws SQLException, IOException {
+ int line = 0;
+ long timestamp = System.currentTimeMillis();
while (rs.next()) {
if (rs.getString(1) == null ||
"null".equalsIgnoreCase(rs.getString(1))) {
@@ -292,7 +299,14 @@ public class ExportCsv extends AbstractCsvTool {
writeTime(rs, bw);
writeValue(rs, count, bw);
}
+ line++;
+ if(line % EXPORT_PER_LINE_COUNT == 0){
+ long tmp = System.currentTimeMillis();
+ System.out.println(String.format("%d lines have been exported, it takes %dms", line, (tmp-timestamp)));
+ timestamp = tmp;
+ }
}
+ return line;
}
private static void writeTime(ResultSet rs, BufferedWriter bw) throws SQLException, IOException {
diff --git a/iotdb-cli/src/main/java/org/apache/iotdb/cli/tool/ImportCsv.java b/iotdb-cli/src/main/java/org/apache/iotdb/cli/tool/ImportCsv.java
index d3318f9..c805c5e 100644
--- a/iotdb-cli/src/main/java/org/apache/iotdb/cli/tool/ImportCsv.java
+++ b/iotdb-cli/src/main/java/org/apache/iotdb/cli/tool/ImportCsv.java
@@ -25,6 +25,7 @@ import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
+import java.io.LineNumberReader;
import java.sql.DatabaseMetaData;
import java.sql.DriverManager;
import java.sql.ResultSet;
@@ -36,6 +37,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import jline.console.ConsoleReader;
+import me.tongfei.progressbar.ProgressBar;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser;
@@ -66,7 +68,7 @@ public class ImportCsv extends AbstractCsvTool {
private static final String ERROR_INFO_STR = "csvInsertError.error";
private static final String STRING_DATA_TYPE = "TEXT";
- private static final int BATCH_EXECUTE_COUNT = 10;
+ private static final int BATCH_EXECUTE_COUNT = 100;
private static String errorInsertInfo = "";
private static boolean errorFlag;
@@ -122,7 +124,13 @@ public class ImportCsv extends AbstractCsvTool {
*/
private static void loadDataFromCSV(File file, int index) {
statement = null;
-
+ int fileLine;
+ try {
+ fileLine = getFileLineCount(file);
+ } catch (IOException e) {
+ System.out.println("Failed to import file: " + file.getName());
+ return;
+ }
File errorFile = new File(errorInsertInfo + index);
if (!errorFile.exists()) {
try {
@@ -132,11 +140,12 @@ public class ImportCsv extends AbstractCsvTool {
return;
}
}
-
+ System.out.println("Start to import data from: " + file.getName());
errorFlag = true;
try(BufferedReader br = new BufferedReader(new FileReader(file));
- BufferedWriter bw = new BufferedWriter(new FileWriter(errorFile))) {
-
+ BufferedWriter bw = new BufferedWriter(new FileWriter(errorFile));
+ ProgressBar pb = new ProgressBar("Import from: " + file.getName(), fileLine)) {
+ pb.setExtraMessage("Importing...");
String header = br.readLine();
bw.write("From " + file.getAbsolutePath());
@@ -174,13 +183,13 @@ public class ImportCsv extends AbstractCsvTool {
List<String> tmp = new ArrayList<>();
success = readAndGenSqls(br, timeseriesDataType, deviceToColumn, colInfo, headInfo,
- bw, tmp);
+ bw, tmp, pb);
if (!success) {
return;
}
executeSqls(bw, tmp, startTime, file);
-
+ pb.stepTo(fileLine);
} catch (FileNotFoundException e) {
System.out.println("Cannot find " + file.getName() + " because: "+e.getMessage());
} catch (IOException e) {
@@ -219,8 +228,6 @@ public class ImportCsv extends AbstractCsvTool {
}
statement.clearBatch();
tmp.clear();
- System.out.println("Load data from "+ file.getName() +" successfully, it takes "
- + ""+(System.currentTimeMillis() - startTime)+"ms");
} catch (SQLException e) {
bw.write(e.getMessage());
bw.newLine();
@@ -231,7 +238,7 @@ public class ImportCsv extends AbstractCsvTool {
private static boolean readAndGenSqls(BufferedReader br, Map<String, String> timeseriesDataType,
Map<String, ArrayList<Integer>> deviceToColumn, List<String> colInfo,
- List<String> headInfo, BufferedWriter bw, List<String> tmp) throws IOException {
+ List<String> headInfo, BufferedWriter bw, List<String> tmp, ProgressBar pb) throws IOException {
String line;
count = 0;
while ((line = br.readLine()) != null) {
@@ -246,6 +253,7 @@ public class ImportCsv extends AbstractCsvTool {
return false;
}
boolean success = addSqlsToBatch(sqls, tmp, bw);
+ pb.step();
if (!success) {
return false;
}
@@ -492,4 +500,16 @@ public class ImportCsv extends AbstractCsvTool {
}
}
}
+
+ private static int getFileLineCount(File file) throws IOException {
+ int line = 0;
+ try (LineNumberReader count = new LineNumberReader(new FileReader(file))) {
+ while (count.skip(Long.MAX_VALUE) > 0) {
+ // Loop just in case the file is > Long.MAX_VALUE or skip() decides to not read the entire file
+ }
+ // +1 because line index starts at 0
+ line = count.getLineNumber() + 1;
+ }
+ return line;
+ }
}