You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by cg...@apache.org on 2022/05/19 20:43:47 UTC
[drill] branch master updated: DRILL-8229: Add Parameter to Skip Malformed Records to HTTP UDF (#2553)
This is an automated email from the ASF dual-hosted git repository.
cgivre pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new 8cef9dad56 DRILL-8229: Add Parameter to Skip Malformed Records to HTTP UDF (#2553)
8cef9dad56 is described below
commit 8cef9dad56050e867e243baeed7494b8294ef9b7
Author: Charles S. Givre <cg...@apache.org>
AuthorDate: Thu May 19 16:43:39 2022 -0400
DRILL-8229: Add Parameter to Skip Malformed Records to HTTP UDF (#2553)
* Add Parameter to Skip Malformed Records to HTTP UDF
* Addressed review comments
---
.../org/apache/drill/exec/store/http/udfs/HttpHelperFunctions.java | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/udfs/HttpHelperFunctions.java b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/udfs/HttpHelperFunctions.java
index bdfd0d34c6..25f9f071ae 100644
--- a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/udfs/HttpHelperFunctions.java
+++ b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/udfs/HttpHelperFunctions.java
@@ -65,6 +65,9 @@ public class HttpHelperFunctions {
.allTextMode(options.getOption(org.apache.drill.exec.ExecConstants.JSON_ALL_TEXT_MODE).bool_val)
.enableNanInf(options.getOption(org.apache.drill.exec.ExecConstants.JSON_READER_NAN_INF_NUMBERS).bool_val)
.build();
+
+ jsonReader.setIgnoreJSONParseErrors(
+ options.getBoolean(org.apache.drill.exec.ExecConstants.JSON_READER_SKIP_INVALID_RECORDS_FLAG));
}
@Override
@@ -151,6 +154,9 @@ public class HttpHelperFunctions {
.enableNanInf(options.getOption(org.apache.drill.exec.ExecConstants.JSON_READER_NAN_INF_NUMBERS).bool_val)
.build();
+ jsonReader.setIgnoreJSONParseErrors(
+ options.getBoolean(org.apache.drill.exec.ExecConstants.JSON_READER_SKIP_INVALID_RECORDS_FLAG));
+
String schemaPath = org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.toStringFromUTF8(rawInput.start, rawInput.end, rawInput.buffer);
// Get the plugin name and endpoint name
String[] parts = schemaPath.split("\\.");