You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by su...@apache.org on 2019/05/28 17:39:10 UTC
[incubator-pinot] 01/01: Add support for custom record-readers in
the create-segment tool
This is an automated email from the ASF dual-hosted git repository.
sunithabeeram pushed a commit to branch createSegments
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit bdb264861c502b992c764c80ef6701efec0225b4
Author: Sunitha Beeram <sb...@sbeeram-ld2.linkedin.biz>
AuthorDate: Tue May 28 10:38:18 2019 -0700
Add support for custom record-readers in the create-segment tool
---
pinot-integration-tests/pom.xml | 6 ++++++
pinot-tools/pom.xml | 4 ++++
.../tools/admin/command/CreateSegmentCommand.java | 25 +++++++++++++---------
3 files changed, 25 insertions(+), 10 deletions(-)
diff --git a/pinot-integration-tests/pom.xml b/pinot-integration-tests/pom.xml
index 20ad7c8..01c0532 100644
--- a/pinot-integration-tests/pom.xml
+++ b/pinot-integration-tests/pom.xml
@@ -126,6 +126,12 @@
<dependency>
<groupId>org.apache.pinot</groupId>
<artifactId>pinot-tools</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.pinot</groupId>
+ <artifactId>pinot-orc</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>com.101tec</groupId>
diff --git a/pinot-tools/pom.xml b/pinot-tools/pom.xml
index 766852c..9fcd62e 100644
--- a/pinot-tools/pom.xml
+++ b/pinot-tools/pom.xml
@@ -44,6 +44,10 @@
</dependency>
<dependency>
<groupId>org.apache.pinot</groupId>
+ <artifactId>pinot-orc</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.pinot</groupId>
<artifactId>pinot-server</artifactId>
</dependency>
<dependency>
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java
index 95c1371..af22b87 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java
@@ -228,15 +228,17 @@ public class CreateSegmentCommand extends AbstractBaseAdminCommand implements Co
}
}
- FileFormat configFormat = segmentGeneratorConfig.getFormat();
- if (_format == null) {
- if (configFormat == null) {
- throw new RuntimeException("Format cannot be null in config file.");
- }
- _format = configFormat;
- } else {
- if (configFormat != _format && configFormat != FileFormat.AVRO) {
- LOGGER.warn("Find format conflict in command line and config file, use config in command line: {}", _format);
+ if (segmentGeneratorConfig.getRecordReaderPath() == null) {
+ FileFormat configFormat = segmentGeneratorConfig.getFormat();
+ if (_format == null) {
+ if (configFormat == null) {
+ throw new RuntimeException("Either recordReaderPath (via generatorConfigFile option) or format must be specified.");
+ }
+ _format = configFormat;
+ } else {
+ if (configFormat != _format && configFormat != FileFormat.AVRO) {
+ LOGGER.warn("Find format conflict in command line and config file, use config in command line: {}", _format);
+ }
}
}
@@ -291,7 +293,10 @@ public class CreateSegmentCommand extends AbstractBaseAdminCommand implements Co
File[] files = dir.listFiles(new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
- return name.toLowerCase().endsWith(_format.toString().toLowerCase());
+ if (_format != null) {
+ return name.toLowerCase().endsWith(_format.toString().toLowerCase());
+ }
+ return true;
}
});
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org