You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudberry.apache.org by dj...@apache.org on 2025/04/15 03:18:06 UTC
(cloudberry-pxf) 13/14: Relax the requirement that C string length matches Java string length (#870)
This is an automated email from the ASF dual-hosted git repository.
djwang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/cloudberry-pxf.git
commit e56511ca07c17efd5ef0c2090b595b02bd15e251
Author: Bradford D. Boyle <br...@vmware.com>
AuthorDate: Mon Sep 19 13:27:48 2022 -0700
Relax the requirement that C string length matches Java string length (#870)
Commit 94f8ccad added a check that the length of received strings (as
determined by strnlen) matched the length of the string as determined by
the Java-based PXF service; if they did not, this was treated as a fatal
error. Some users have reported that they have ORC/Parquet files with
strings that contain ASCCII NUL-bytes. These strings would not have a
strnlen calculate length that matches with the Java string lenght.
This commit removes the requirements that the lengths be equal and
instead logs a debug message when they do not match.
Authored-by: Bradford D. Boyle <br...@vmware.com>
---
.../pxf/automation/features/orc/OrcReadTest.java | 14 ++++++++++++
automation/src/test/resources/data/orc/README.md | 7 ++++++
.../resources/data/orc/orc_null_in_string.json | 6 +++++
.../test/resources/data/orc/orc_null_in_string.orc | Bin 0 -> 719 bytes
.../features/orc/read/null_in_string/__init__.py | 0
.../orc/read/null_in_string/expected/query01.ans | 13 +++++++++++
.../features/orc/read/null_in_string/runTest.py | 11 +++++++++
.../orc/read/null_in_string/sql/query01.sql | 3 +++
external-table/src/gpdbwritableformatter.c | 25 ++++++++++++---------
9 files changed, 69 insertions(+), 10 deletions(-)
diff --git a/automation/src/test/java/org/greenplum/pxf/automation/features/orc/OrcReadTest.java b/automation/src/test/java/org/greenplum/pxf/automation/features/orc/OrcReadTest.java
index 1e0feed3..de41d36b 100644
--- a/automation/src/test/java/org/greenplum/pxf/automation/features/orc/OrcReadTest.java
+++ b/automation/src/test/java/org/greenplum/pxf/automation/features/orc/OrcReadTest.java
@@ -13,6 +13,7 @@ public class OrcReadTest extends BaseFeature {
private static final String ORC_PRIMITIVE_TYPES_UNORDERED_SUBSET = "orc_types_unordered_subset.orc";
private static final String ORC_LIST_TYPES = "orc_list_types.orc";
private static final String ORC_MULTIDIM_LIST_TYPES = "orc_multidim_list_types.orc";
+ private static final String ORC_NULL_IN_STRING = "orc_null_in_string.orc";
private static final String[] ORC_TABLE_COLUMNS = {
"id integer",
@@ -73,6 +74,12 @@ public class OrcReadTest extends BaseFeature {
"varchar_arr text[]"
};
+ private static final String[] ORC_NULL_IN_STRING_COLUMNS = new String[]{
+ "id int",
+ "context text",
+ "value text"
+ };
+
private String hdfsPath;
private ProtocolEnum protocol;
@@ -87,6 +94,7 @@ public class OrcReadTest extends BaseFeature {
hdfs.copyFromLocal(resourcePath + ORC_PRIMITIVE_TYPES_UNORDERED_SUBSET, hdfsPath + ORC_PRIMITIVE_TYPES_UNORDERED_SUBSET);
hdfs.copyFromLocal(resourcePath + ORC_LIST_TYPES, hdfsPath + ORC_LIST_TYPES);
hdfs.copyFromLocal(resourcePath + ORC_MULTIDIM_LIST_TYPES, hdfsPath + ORC_MULTIDIM_LIST_TYPES);
+ hdfs.copyFromLocal(resourcePath + ORC_NULL_IN_STRING, hdfsPath + ORC_NULL_IN_STRING);
prepareReadableExternalTable(PXF_ORC_TABLE, ORC_TABLE_COLUMNS, hdfsPath + ORC_PRIMITIVE_TYPES);
}
@@ -146,6 +154,12 @@ public class OrcReadTest extends BaseFeature {
runTincTest("pxf.features.orc.read.multidim_list_types.runTest");
}
+ @Test(groups = {"features", "gpdb", "security", "hcfs"})
+ public void orcReadStringsContainingNullByte() throws Exception {
+ prepareReadableExternalTable("pxf_orc_null_in_string", ORC_NULL_IN_STRING_COLUMNS, hdfsPath + ORC_NULL_IN_STRING);
+ runTincTest("pxf.features.orc.read.null_in_string.runTest");
+ }
+
private void prepareReadableExternalTable(String name, String[] fields, String path) throws Exception {
prepareReadableExternalTable(name, fields, path, false);
}
diff --git a/automation/src/test/resources/data/orc/README.md b/automation/src/test/resources/data/orc/README.md
index 62e5ceb6..0b143878 100644
--- a/automation/src/test/resources/data/orc/README.md
+++ b/automation/src/test/resources/data/orc/README.md
@@ -103,3 +103,10 @@ If desired, you can copy down the CSV file by running the following command:
mv "./orc_multidim_list_types/000000_0" "./${CSV_FILENAME}"
```
+## Generate the orc_null_in_string.orc file
+
+To generate this file, you will need to download a copy of the ORC Tools uber jar: <https://search.maven.org/remotecontent?filepath=org/apache/orc/orc-tools/1.8.0/orc-tools-1.8.0-uber.jar>
+
+```shell
+java -jar orc-tools-1.8.0-uber.jar convert --schema 'struct<id:int,desc:string,value:string>' --output orc_null_in_string.orc orc_null_in_string.json
+```
diff --git a/automation/src/test/resources/data/orc/orc_null_in_string.json b/automation/src/test/resources/data/orc/orc_null_in_string.json
new file mode 100644
index 00000000..e918aad6
--- /dev/null
+++ b/automation/src/test/resources/data/orc/orc_null_in_string.json
@@ -0,0 +1,6 @@
+{"id": 1, "context": "simple string", "value": "hello"}
+{"id": 2, "context": "simple string with space", "value": "hello world"}
+{"id": 3, "context": "simple string with double quote", "value": "hello \"world\""}
+{"id": 4, "context": "NUL-byte in middle of string", "value": "hello\u0000world"}
+{"id": 5, "context": "NUL-byte at the beginning of string", "value": "\u0000hello world"}
+{"id": 6, "context": "NUL-byte at the end of string", "value": "hello world\u0000"}
diff --git a/automation/src/test/resources/data/orc/orc_null_in_string.orc b/automation/src/test/resources/data/orc/orc_null_in_string.orc
new file mode 100644
index 00000000..3cc1e6ae
Binary files /dev/null and b/automation/src/test/resources/data/orc/orc_null_in_string.orc differ
diff --git a/automation/tincrepo/main/pxf/features/orc/read/null_in_string/__init__.py b/automation/tincrepo/main/pxf/features/orc/read/null_in_string/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/automation/tincrepo/main/pxf/features/orc/read/null_in_string/expected/query01.ans b/automation/tincrepo/main/pxf/features/orc/read/null_in_string/expected/query01.ans
new file mode 100644
index 00000000..bbfd641a
--- /dev/null
+++ b/automation/tincrepo/main/pxf/features/orc/read/null_in_string/expected/query01.ans
@@ -0,0 +1,13 @@
+-- start_ignore
+-- end_ignore
+-- @description query01 for reading strings contain NUL-byte from ORC files
+SELECT * FROM pxf_orc_null_in_string ORDER BY id;
+ id | context | value
+----+-------------------------------------+---------------
+ 1 | simple string | hello
+ 2 | simple string with space | hello world
+ 3 | simple string with double quote | hello "world"
+ 4 | NUL-byte in middle of string | hello
+ 5 | NUL-byte at the beginning of string |
+ 6 | NUL-byte at the end of string | hello world
+(6 rows)
\ No newline at end of file
diff --git a/automation/tincrepo/main/pxf/features/orc/read/null_in_string/runTest.py b/automation/tincrepo/main/pxf/features/orc/read/null_in_string/runTest.py
new file mode 100644
index 00000000..916deefe
--- /dev/null
+++ b/automation/tincrepo/main/pxf/features/orc/read/null_in_string/runTest.py
@@ -0,0 +1,11 @@
+from mpp.models import SQLConcurrencyTestCase
+
+class OrcNullInString(SQLConcurrencyTestCase):
+ """
+ @db_name pxfautomation
+ @concurrency 1
+ @gpdiff True
+ """
+ sql_dir = 'sql'
+ ans_dir = 'expected'
+ out_dir = 'output'
diff --git a/automation/tincrepo/main/pxf/features/orc/read/null_in_string/sql/query01.sql b/automation/tincrepo/main/pxf/features/orc/read/null_in_string/sql/query01.sql
new file mode 100644
index 00000000..52e65cb3
--- /dev/null
+++ b/automation/tincrepo/main/pxf/features/orc/read/null_in_string/sql/query01.sql
@@ -0,0 +1,3 @@
+-- @description query01 for reading strings contain NUL-byte from ORC files
+
+SELECT * FROM pxf_orc_null_in_string ORDER BY id;
diff --git a/external-table/src/gpdbwritableformatter.c b/external-table/src/gpdbwritableformatter.c
index 835c9a04..7db9f30b 100644
--- a/external-table/src/gpdbwritableformatter.c
+++ b/external-table/src/gpdbwritableformatter.c
@@ -960,19 +960,24 @@ gpdbwritableformatter_import(PG_FUNCTION_ARGS)
* Read string value from data_buf and convert it to internal representation
*
* PXF service should send all strings with a terminating nul-byte, so we can
- * determine the length of the string and compare it with the length that PXF
- * service computed and sent. Since we've checked that outlen[i] is no larger
- * than the number of bytes left for this tuple in data_buf, we use it as max
- * number of bytes to scan.
+ * determine the number of bytes that the input function will read and compare
+ * it with the length that the PXF service computed and sent. Since we've
+ * checked that outlen[i] is no larger than the number of bytes left for this
+ * tuple in data_buf, we use it as max number of bytes to scan in the call to
+ * strnlen
*/
size_t actual_len = strnlen(data_buf + bufidx, myData->outlen[i]);
- /* outlen[i] includes the terminating nul-byte */
- if (actual_len != myData->outlen[i] - 1)
- ereport(FATAL,
- (errcode(ERRCODE_EXTERNAL_ROUTINE_EXCEPTION),
- errmsg("data for column %d of row %d has invalid length", i + 1, myData->lineno),
- errdetail("expected column %d to have length %d, actual length is %ld", i + 1, myData->outlen[i] - 1, actual_len)));
+ /*
+ * Compare the length as returned by strnlen with the length as determined by
+ * the PXF service. If the source data includes ASCII NUL-byte in the string,
+ * then these two values will not match. It's possible that this will result
+ * in the Postgres input function truncating/mis-reading the value. Rather
+ * than treat this as an error here, we log a debug message.
+ */
+ if (actual_len != myData->outlen[i]-1)
+ ereport(DEBUG1,
+ (errmsg("expected column %d of row %d to have length %d, actual length is %ld", i+1, myData->lineno, myData->outlen[i]-1, actual_len)));
myData->values[i] = InputFunctionCall(iofunc,
data_buf + bufidx,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cloudberry.apache.org
For additional commands, e-mail: commits-help@cloudberry.apache.org