You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by do...@apache.org on 2019/04/21 13:23:59 UTC
[incubator-iotdb] branch fix_jira_issue_83 created (now 2b00caf)
This is an automated email from the ASF dual-hosted git repository.
dope pushed a change to branch fix_jira_issue_83
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
at 2b00caf remove useless file and add process bar
This branch includes the following new commits:
new 2b00caf remove useless file and add process bar
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-iotdb] 01/01: remove useless file and add process bar
Posted by do...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dope pushed a commit to branch fix_jira_issue_83
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 2b00caf14ffa24f397473802647aca04d2e5a6a4
Author: xuyi556677@163.com <xu...@163.com>
AuthorDate: Sun Apr 21 21:23:39 2019 +0800
remove useless file and add process bar
---
clean.sh | 36 --------------------
iotdb-cli/pom.xml | 5 +++
.../java/org/apache/iotdb/cli/tool/ExportCsv.java | 24 +++++++++++---
.../java/org/apache/iotdb/cli/tool/ImportCsv.java | 38 +++++++++++++++++-----
4 files changed, 54 insertions(+), 49 deletions(-)
diff --git a/clean.sh b/clean.sh
deleted file mode 100644
index 0483bdc..0000000
--- a/clean.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/bin/sh
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-git checkout checkstyle.xml
-git checkout java-google-style.xml
-git checkout example/**/pom.xml
-git checkout grafana/pom.xml
-git checkout hadoop/pom.xml
-git checkout example/pom.xml
-git checkout jdbc/pom.xml
-git checkout service-rpc/pom.xml
-git checkout spark/pom.xml
-git checkout tsfile/pom.xml
-git checkout iotdb-cli/pom.xml
-git checkout iotdb/pom.xml
-git checkout pom.xml
-git checkout iotdb-cli/src/test/resources/logback.xml
-git checkout iotdb/iotdb/conf/logback.xml
-git checkout iotdb/src/test/resources/logback.xml
-git checkout tsfile/src/test/resources/logback.xml
diff --git a/iotdb-cli/pom.xml b/iotdb-cli/pom.xml
index 6977892..a6634e5 100644
--- a/iotdb-cli/pom.xml
+++ b/iotdb-cli/pom.xml
@@ -54,6 +54,11 @@
<version>3.1</version>
</dependency>
<dependency>
+ <groupId>me.tongfei</groupId>
+ <artifactId>progressbar</artifactId>
+ <version>0.7.3</version>
+ </dependency>
+ <dependency>
<groupId>jline</groupId>
<artifactId>jline</artifactId>
<version>${jline.version}</version>
diff --git a/iotdb-cli/src/main/java/org/apache/iotdb/cli/tool/ExportCsv.java b/iotdb-cli/src/main/java/org/apache/iotdb/cli/tool/ExportCsv.java
index 4762519..f9d7012 100644
--- a/iotdb-cli/src/main/java/org/apache/iotdb/cli/tool/ExportCsv.java
+++ b/iotdb-cli/src/main/java/org/apache/iotdb/cli/tool/ExportCsv.java
@@ -68,6 +68,8 @@ public class ExportCsv extends AbstractCsvTool {
private static String targetFile = DUMP_FILE_NAME_DEFAULT;
private static String targetDirectory;
+
+ private static final int EXPORT_PER_LINE_COUNT = 10000;
/**
* main function of export csv tool.
@@ -147,6 +149,9 @@ public class ExportCsv extends AbstractCsvTool {
throws ArgsErrorException {
targetDirectory = checkRequiredArg(TARGET_DIR_ARGS, TARGET_DIR_NAME, commandLine);
targetFile = commandLine.getOptionValue(TARGET_FILE_ARGS);
+ if(targetFile == null){
+ targetFile = DUMP_FILE_NAME_DEFAULT;
+ }
timeFormat = commandLine.getOptionValue(TIME_FORMAT_ARGS);
if (timeFormat == null) {
timeFormat = "default";
@@ -252,7 +257,7 @@ public class ExportCsv extends AbstractCsvTool {
System.out.println("Cannot create dump file "+ path + "because: " + e.getMessage());
return;
}
-
+ System.out.println("Start to export data from sql statement: "+ sql);
try (Statement statement = connection.createStatement();
ResultSet rs = statement.executeQuery(sql);
BufferedWriter bw = new BufferedWriter(new FileWriter(tf))) {
@@ -263,9 +268,9 @@ public class ExportCsv extends AbstractCsvTool {
// write data in csv file
writeMetadata(bw, count, metadata);
- writeResultSet(rs, bw, count);
- System.out.println(String.format("Statement [%s] has dumped to file %s successfully! "
- + "It costs %dms.", sql, path, System.currentTimeMillis() - startTime));
+ int line = writeResultSet(rs, bw, count);
+ System.out.println(String.format("Statement [%s] has dumped to file %s successfully! It costs "
+ + "%dms to export %d lines.", sql, path, System.currentTimeMillis() - startTime, line));
} catch (IOException e) {
System.out.println("Cannot dump result because: " + e.getMessage());
}
@@ -282,8 +287,10 @@ public class ExportCsv extends AbstractCsvTool {
}
}
- private static void writeResultSet(ResultSet rs, BufferedWriter bw, int count)
+ private static int writeResultSet(ResultSet rs, BufferedWriter bw, int count)
throws SQLException, IOException {
+ int line = 0;
+ long timestamp = System.currentTimeMillis();
while (rs.next()) {
if (rs.getString(1) == null ||
"null".equalsIgnoreCase(rs.getString(1))) {
@@ -292,7 +299,14 @@ public class ExportCsv extends AbstractCsvTool {
writeTime(rs, bw);
writeValue(rs, count, bw);
}
+ line++;
+ if(line % EXPORT_PER_LINE_COUNT == 0){
+ long tmp = System.currentTimeMillis();
+ System.out.println(String.format("%d lines have been exported, it takes %dms", line, (tmp-timestamp)));
+ timestamp = tmp;
+ }
}
+ return line;
}
private static void writeTime(ResultSet rs, BufferedWriter bw) throws SQLException, IOException {
diff --git a/iotdb-cli/src/main/java/org/apache/iotdb/cli/tool/ImportCsv.java b/iotdb-cli/src/main/java/org/apache/iotdb/cli/tool/ImportCsv.java
index d3318f9..739ff79 100644
--- a/iotdb-cli/src/main/java/org/apache/iotdb/cli/tool/ImportCsv.java
+++ b/iotdb-cli/src/main/java/org/apache/iotdb/cli/tool/ImportCsv.java
@@ -25,6 +25,7 @@ import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
+import java.io.LineNumberReader;
import java.sql.DatabaseMetaData;
import java.sql.DriverManager;
import java.sql.ResultSet;
@@ -36,6 +37,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import jline.console.ConsoleReader;
+import me.tongfei.progressbar.ProgressBar;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser;
@@ -66,7 +68,7 @@ public class ImportCsv extends AbstractCsvTool {
private static final String ERROR_INFO_STR = "csvInsertError.error";
private static final String STRING_DATA_TYPE = "TEXT";
- private static final int BATCH_EXECUTE_COUNT = 10;
+ private static final int BATCH_EXECUTE_COUNT = 100;
private static String errorInsertInfo = "";
private static boolean errorFlag;
@@ -122,7 +124,13 @@ public class ImportCsv extends AbstractCsvTool {
*/
private static void loadDataFromCSV(File file, int index) {
statement = null;
-
+ int fileLine;
+ try {
+ fileLine = getFileLineCount(file);
+ } catch (IOException e) {
+ System.out.println("Failed to import file: " + file.getName() + " at:" + file.getParent());
+ return;
+ }
File errorFile = new File(errorInsertInfo + index);
if (!errorFile.exists()) {
try {
@@ -132,11 +140,12 @@ public class ImportCsv extends AbstractCsvTool {
return;
}
}
-
+ System.out.println("Start to import data from: " + file.getName() + " at:" + file.getParent());
errorFlag = true;
try(BufferedReader br = new BufferedReader(new FileReader(file));
- BufferedWriter bw = new BufferedWriter(new FileWriter(errorFile))) {
-
+ BufferedWriter bw = new BufferedWriter(new FileWriter(errorFile));
+ ProgressBar pb = new ProgressBar("Import from: " + file.getName(), fileLine)) {
+ pb.setExtraMessage("Importing...");
String header = br.readLine();
bw.write("From " + file.getAbsolutePath());
@@ -174,13 +183,13 @@ public class ImportCsv extends AbstractCsvTool {
List<String> tmp = new ArrayList<>();
success = readAndGenSqls(br, timeseriesDataType, deviceToColumn, colInfo, headInfo,
- bw, tmp);
+ bw, tmp, pb);
if (!success) {
return;
}
executeSqls(bw, tmp, startTime, file);
-
+ pb.stepTo(fileLine);
} catch (FileNotFoundException e) {
System.out.println("Cannot find " + file.getName() + " because: "+e.getMessage());
} catch (IOException e) {
@@ -231,7 +240,7 @@ public class ImportCsv extends AbstractCsvTool {
private static boolean readAndGenSqls(BufferedReader br, Map<String, String> timeseriesDataType,
Map<String, ArrayList<Integer>> deviceToColumn, List<String> colInfo,
- List<String> headInfo, BufferedWriter bw, List<String> tmp) throws IOException {
+ List<String> headInfo, BufferedWriter bw, List<String> tmp, ProgressBar pb) throws IOException {
String line;
count = 0;
while ((line = br.readLine()) != null) {
@@ -246,6 +255,7 @@ public class ImportCsv extends AbstractCsvTool {
return false;
}
boolean success = addSqlsToBatch(sqls, tmp, bw);
+ pb.step();
if (!success) {
return false;
}
@@ -492,4 +502,16 @@ public class ImportCsv extends AbstractCsvTool {
}
}
}
+
+ private static int getFileLineCount(File file) throws IOException {
+ int line = 0;
+ try (LineNumberReader count = new LineNumberReader(new FileReader(file))) {
+ while (count.skip(Long.MAX_VALUE) > 0) {
+ // Loop just in case the file is > Long.MAX_VALUE or skip() decides to not read the entire file
+ }
+ // +1 because line index starts at 0
+ line = count.getLineNumber() + 1;
+ }
+ return line;
+ }
}