You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hop.apache.org by mc...@apache.org on 2025/12/15 15:50:06 UTC
(hop) branch main updated: [Parquet Input/Output] Add comprehensive support for Parquet Logical Types, fixes #5947 (#6186)
This is an automated email from the ASF dual-hosted git repository.
mcasters pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 6583915b6c [Parquet Input/Output] Add comprehensive support for Parquet Logical Types, fixes #5947 (#6186)
6583915b6c is described below
commit 6583915b6cd22aa52d45eb93860b4c4a2fa8b9a5
Author: jbs-atolcd <jb...@atolcd.com>
AuthorDate: Mon Dec 15 16:50:00 2025 +0100
[Parquet Input/Output] Add comprehensive support for Parquet Logical Types, fixes #5947 (#6186)
This commit introduces significant improvements to the Parquet Input and Output transforms by implementing comprehensive support for Parquet's Logical Types.
Previously, the transforms relied primarily on primitive types, leading to conversions issues and errors with data when handling complex types, such as Timestamps..
Key Changes & Features:
1. Parquet Input:
* Logical Type Mapping: Refactors the field discovery to use `LogicalTypeAnnotation` (instead of only primitive type), enabling correct mapping for semantic types.
* Timestamp/Date Precision: Implements a conversion mechanism to map Parquet's timestamps units (MILLIS, MICROS..) to Hop's `TYPE_TIMESTAMP` and `TYPE_DATE`, preserving precision and handling UTC adjustments.
* JSON Support: Adds explicit support for the JSON Logical Type, converting the Parquet binary/string data into Hop's `TYPE_JSON` object.
* Decimal Handling: Uses precision and scale from `DecimalLogicalTypeAnnotation` to correctly convert binary/long Parquet decimals into Hop's `TYPE_BIGNUMBER`.
2. Parquet Output:
* Date/Timestamp Consistency: Ensures that Hop's `TYPE_DATE` and `TYPE_TIMESTAMP` are consistently converted to a `LONG` representation with the Parquet `timestampMillis` logical annotation, which is the most compatible format.
* Schema Mapping: Maps Hop's `TYPE_JSON` and `TYPE_UUID` to Parquet `STRING` types in the schema definition.
Testing and Validation:
* Test Data Enrichment: The test dataset (`golden-parquet-input.json`) was extended to include new fields: `isActive` (Boolean), `registrationTimestamp` (Timestamp), and `metadataJson` (JSON), ensuring the new types are covered end-to-end.
* Unit Test Update: The unit test configuration (`0029-parquet-input UNIT.json`) was updated to map and validate the new fields, confirming the correct functionality of the transform.
This resolves a major limitation regarding data fidelity when dealing with common modern Parquet schemas.
---
.../java/org/apache/hop/core/row/IValueMeta.java | 3 +
.../transforms/0029-parquet-input.hpl | 40 ++--
.../transforms/0029-parquet-output.hpl | 48 ++++-
.../transforms/datasets/golden-parquet-input.csv | 202 ++++++++++-----------
.../transforms/files/customers-100.txt | 202 ++++++++++-----------
.../metadata/dataset/golden-parquet-input.json | 64 +++++--
.../unit-test/0029-parquet-input UNIT.json | 26 ++-
.../parquet/transforms/input/ParquetInputMeta.java | 52 ++++--
.../transforms/input/ParquetRowConverter.java | 5 +-
.../transforms/input/ParquetValueConverter.java | 102 ++++++++++-
.../parquet/transforms/output/ParquetOutput.java | 49 ++++-
11 files changed, 528 insertions(+), 265 deletions(-)
diff --git a/core/src/main/java/org/apache/hop/core/row/IValueMeta.java b/core/src/main/java/org/apache/hop/core/row/IValueMeta.java
index 072cb48d52..cde4687d83 100644
--- a/core/src/main/java/org/apache/hop/core/row/IValueMeta.java
+++ b/core/src/main/java/org/apache/hop/core/row/IValueMeta.java
@@ -199,6 +199,9 @@ public interface IValueMeta extends Cloneable {
/** Value type indicating that the value contains an Avro Record */
int TYPE_AVRO = 20;
+ /** Value type indicating that the value contains an UUID */
+ int TYPE_UUID = 32;
+
/** The Constant typeCodes. */
String[] typeCodes =
new String[] {
diff --git a/integration-tests/transforms/0029-parquet-input.hpl b/integration-tests/transforms/0029-parquet-input.hpl
index 7d288daa11..260b18f970 100644
--- a/integration-tests/transforms/0029-parquet-input.hpl
+++ b/integration-tests/transforms/0029-parquet-input.hpl
@@ -67,26 +67,21 @@ limitations under the License.
<method>none</method>
<schema_name/>
</partitioning>
- <filter>
- <filterfiletype>all_files</filterfiletype>
- </filter>
<doNotFailIfNoFile>N</doNotFailIfNoFile>
- <rownum>N</rownum>
- <isaddresult>Y</isaddresult>
- <filefield>N</filefield>
- <rownum_field/>
- <filename_Field/>
- <wildcard_Field/>
- <exclude_wildcard_Field/>
<dynamic_include_subfolders>N</dynamic_include_subfolders>
- <limit>0</limit>
<file>
- <name>${java.io.tmpdir}/it/parquet/customers.parquet.snappy</name>
- <filemask/>
- <exclude_filemask/>
<file_required>N</file_required>
<include_subfolders>N</include_subfolders>
+ <name>${java.io.tmpdir}/it/parquet/customers.parquet.snappy</name>
</file>
+ <filefield>N</filefield>
+ <filter>
+ <filterfiletype>all_files</filterfiletype>
+ </filter>
+ <isaddresult>Y</isaddresult>
+ <limit>0</limit>
+ <raiseAnExceptionIfNoFile>N</raiseAnExceptionIfNoFile>
+ <rownum>N</rownum>
<attributes/>
<GUI>
<xloc>160</xloc>
@@ -156,6 +151,21 @@ limitations under the License.
<target_field>state</target_field>
<target_type>String</target_type>
</field>
+ <field>
+ <source_field>isActive</source_field>
+ <target_field>isActive</target_field>
+ <target_type>Boolean</target_type>
+ </field>
+ <field>
+ <source_field>registrationTimestamp</source_field>
+ <target_field>registrationTimestamp</target_field>
+ <target_type>Timestamp</target_type>
+ </field>
+ <field>
+ <source_field>metadataJson</source_field>
+ <target_field>metadataJson</target_field>
+ <target_type>JSON</target_type>
+ </field>
</fields>
<filename_field>filename</filename_field>
<attributes/>
@@ -194,7 +204,9 @@ limitations under the License.
</partitioning>
<fields>
<field>
+ <length>-2</length>
<name>filename</name>
+ <precision>-2</precision>
</field>
<select_unspecified>N</select_unspecified>
</fields>
diff --git a/integration-tests/transforms/0029-parquet-output.hpl b/integration-tests/transforms/0029-parquet-output.hpl
index 4fabba275a..a99fe28f8b 100644
--- a/integration-tests/transforms/0029-parquet-output.hpl
+++ b/integration-tests/transforms/0029-parquet-output.hpl
@@ -101,8 +101,19 @@ limitations under the License.
<source_field>state</source_field>
<target_field>state</target_field>
</field>
+ <field>
+ <source_field>isActive</source_field>
+ <target_field>isActive</target_field>
+ </field>
+ <field>
+ <source_field>registrationTimestamp</source_field>
+ <target_field>registrationTimestamp</target_field>
+ </field>
+ <field>
+ <source_field>metadataJson</source_field>
+ <target_field>metadataJson</target_field>
+ </field>
</fields>
- <filename_split_size>1000000</filename_split_size>
<filename_base>${java.io.tmpdir}/it/parquet/customers</filename_base>
<filename_create_parent_folders>Y</filename_create_parent_folders>
<filename_datetime_format>yyyyMMdd-HHmmss</filename_datetime_format>
@@ -112,6 +123,7 @@ limitations under the License.
<filename_include_datetime>N</filename_include_datetime>
<filename_include_split>N</filename_include_split>
<filename_include_time>N</filename_include_time>
+ <filename_split_size>1000000</filename_split_size>
<row_group_size>20000</row_group_size>
<version>1.0</version>
<attributes/>
@@ -139,6 +151,7 @@ limitations under the License.
<enclosure>"</enclosure>
<header>Y</header>
<buffer_size>50000</buffer_size>
+ <schemaDefinition/>
<lazy_conversion>Y</lazy_conversion>
<add_filename_result>N</add_filename_result>
<parallel>N</parallel>
@@ -255,6 +268,39 @@ limitations under the License.
<precision>-1</precision>
<trim_type>none</trim_type>
</field>
+ <field>
+ <name>isActive</name>
+ <type>Boolean</type>
+ <format/>
+ <currency>$</currency>
+ <decimal>,</decimal>
+ <group> </group>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ </field>
+ <field>
+ <name>registrationTimestamp</name>
+ <type>Timestamp</type>
+ <format>yyyy-MM-dd HH:mm:ss.SSS</format>
+ <currency>$</currency>
+ <decimal>,</decimal>
+ <group> </group>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ </field>
+ <field>
+ <name>metadataJson</name>
+ <type>JSON</type>
+ <format/>
+ <currency>$</currency>
+ <decimal>,</decimal>
+ <group> </group>
+ <length>64</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ </field>
</fields>
<attributes/>
<GUI>
diff --git a/integration-tests/transforms/datasets/golden-parquet-input.csv b/integration-tests/transforms/datasets/golden-parquet-input.csv
index cb21054f8a..539861280a 100644
--- a/integration-tests/transforms/datasets/golden-parquet-input.csv
+++ b/integration-tests/transforms/datasets/golden-parquet-input.csv
@@ -1,101 +1,101 @@
-id,name,firstname,zip,city,birthdate,street,housenr,stateCode,state
-1,jwcdf-name,fsj-firstname,13520,oem-city,1954/02/07,amrb-street,145,AK,ALASKA
-2,flhxu-name,tum-firstname,17520,buo-city,1966/04/24,wfyz-street,96,GA,GEORGIA
-3,xthfg-name,gfe-firstname,12560,vtz-city,1990/01/11,doxx-street,46,NJ,NEW JERSEY
-4,ulzrz-name,bnl-firstname,11620,prz-city,1966/08/02,bxqn-street,104,NY,NEW YORK
-5,oxhyr-name,onx-firstname,15180,bpn-city,1970/11/14,pksn-street,133,IN,INDIANA
-6,fiqjz-name,sce-firstname,16020,fnn-city,1954/09/24,wbhg-street,35,MD,MARYLAND
-7,tkiat-name,xti-firstname,12720,stt-city,1966/08/11,tvnf-street,21,PA,PENNSYLVANIA
-8,kljcz-name,uqd-firstname,13340,ntt-city,1987/01/15,jyje-street,10,PW,PALAU
-9,pgunz-name,hcm-firstname,16680,gxh-city,1970/11/08,shbe-street,184,NC,NORTH CAROLINA
-10,oyjha-name,uhj-firstname,18880,uyg-city,1966/04/10,bjgw-street,176,AR,ARKANSAS
-11,igxbd-name,uph-firstname,13480,ndh-city,1962/12/03,jdcd-street,151,NH,NEW HAMPSHIRE
-12,vnaov-name,wha-firstname,13120,egm-city,1954/03/28,hpep-street,20,CA,CALIFORNIA
-13,dauuz-name,hwg-firstname,13740,khn-city,1958/05/15,etqx-street,5,OK,OKLAHOMA
-14,gkuuo-name,kkb-firstname,13560,xdt-city,1962/04/07,sdoj-street,35,MT,MONTANA
-15,wdhze-name,jjk-firstname,16900,due-city,1970/07/17,pmmu-street,174,AS,AMERICAN SAMOA
-16,ncayz-name,ynb-firstname,15720,lxj-city,1974/04/27,mdtb-street,109,MA,MASSACHUSETTS
-17,rdjin-name,hhu-firstname,14480,lpc-city,1958/11/16,wxik-street,145,KY,KENTUCKY
-18,nxzij-name,bdl-firstname,10740,avx-city,1958/02/20,nybz-street,138,WI,WISCONSIN
-19,xgrzc-name,dxw-firstname,18900,vpq-city,1990/11/16,wzjh-street,58,ME,MAINE
-20,ehgrn-name,vbe-firstname,17500,cik-city,1978/05/21,ucnw-street,135,MD,MARYLAND
-21,gctjx-name,upx-firstname,11960,yqr-city,1958/03/03,rlko-street,141,TN,TENNESSEE
-22,ptzmg-name,hva-firstname,15740,gux-city,1978/05/04,pugy-street,122,VI,VIRGIN ISLANDS
-23,eyeti-name,gnw-firstname,17420,eko-city,1962/10/26,ylph-street,61,NC,NORTH CAROLINA
-24,wccwo-name,zpj-firstname,16600,uim-city,1962/09/29,ygih-street,26,WA,WASHINGTON
-25,bwkoe-name,ayl-firstname,18660,rtw-city,1978/07/16,mzww-street,179,CA,CALIFORNIA
-26,rezku-name,zio-firstname,19080,nvt-city,1982/07/14,wwkd-street,91,CA,CALIFORNIA
-27,mjlsk-name,ecx-firstname,10800,yxu-city,1950/12/11,vttb-street,195,MO,MISSOURI
-28,wdjsi-name,aoq-firstname,13660,smo-city,1954/02/01,kako-street,7,NV,NEVADA
-29,mwfnd-name,nyb-firstname,19760,bbu-city,1986/09/23,apdi-street,91,MS,MISSISSIPPI
-30,vtuoz-name,jhh-firstname,17620,vad-city,1982/05/05,kzup-street,79,GA,GEORGIA
-31,rhhxk-name,ndr-firstname,16760,fub-city,1978/11/12,regd-street,55,OK,OKLAHOMA
-32,lpstk-name,mqz-firstname,18940,tnr-city,1982/09/16,cdhf-street,4,SD,SOUTH DAKOTA
-33,ldhyr-name,yts-firstname,12000,auk-city,1986/11/14,abph-street,147,IN,INDIANA
-34,cjdml-name,iti-firstname,16900,wkq-city,1970/06/05,npow-street,96,NH,NEW HAMPSHIRE
-35,cpenz-name,sbi-firstname,16380,ssl-city,1962/08/19,kilz-street,44,MS,MISSISSIPPI
-36,rxtbg-name,anr-firstname,14720,bqc-city,1958/08/10,pudg-street,140,NV,NEVADA
-37,udblf-name,raa-firstname,11500,wli-city,1978/12/13,xomd-street,41,PW,PALAU
-38,vvyce-name,gep-firstname,13740,gtd-city,1982/05/23,kwbv-street,123,undefined,undefined
-39,kwfnz-name,ucu-firstname,10580,sns-city,1978/08/18,nnun-street,20,OK,OKLAHOMA
-40,zxydx-name,tml-firstname,14680,jda-city,1974/05/29,wfjn-street,157,DC,DISTRICT OF COLUMBIA
-41,bfscx-name,jnl-firstname,16920,yyg-city,1970/11/30,cgfh-street,178,CO,COLORADO
-42,qitur-name,yra-firstname,15560,ijp-city,1978/01/30,fonc-street,155,AK,ALASKA
-43,msixi-name,ynb-firstname,12720,ksl-city,1958/07/17,zpjw-street,46,VI,VIRGIN ISLANDS
-44,wzkjq-name,rgh-firstname,19000,hkm-city,1974/08/12,yixf-street,134,CA,CALIFORNIA
-45,dqfmf-name,yxr-firstname,13840,vie-city,1962/10/23,stvx-street,39,TX,TEXAS
-46,biluz-name,uqe-firstname,17760,wkq-city,1962/07/27,embn-street,183,PW,PALAU
-47,wahfx-name,zwd-firstname,13240,vic-city,1974/03/27,axpw-street,131,UT,UTAH
-48,denwt-name,bta-firstname,17300,hhj-city,1986/12/20,orwy-street,11,WV,WEST VIRGINIA
-49,akdmy-name,ybz-firstname,14560,wtx-city,1962/11/08,nwba-street,123,MP,NORTHERN MARIANA ISLANDS
-50,hqafg-name,nht-firstname,16080,gfu-city,1951/01/12,spsq-street,45,LA,LOUISIANA
-51,zhmbl-name,lnw-firstname,17460,hse-city,1986/12/21,scis-street,97,GA,GEORGIA
-52,snwnj-name,jyy-firstname,16400,hsz-city,1966/02/15,imhl-street,42,NC,NORTH CAROLINA
-53,fuyla-name,mmp-firstname,11840,hgu-city,1986/08/16,ixiz-street,145,NC,NORTH CAROLINA
-54,yvfqz-name,prz-firstname,11260,wjl-city,1982/05/06,fbzd-street,97,MO,MISSOURI
-55,usbgq-name,vhd-firstname,14080,dsb-city,1958/04/01,ggoc-street,54,KS,KANSAS
-56,yaeni-name,zpy-firstname,19100,sen-city,1954/12/10,sbsw-street,158,HI,HAWAII
-57,fgxvr-name,vzi-firstname,17520,lcf-city,1958/11/01,nbdv-street,10,GU,GUAM
-58,tqpbq-name,rwr-firstname,19140,zpd-city,1978/08/23,npvb-street,190,DC,DISTRICT OF COLUMBIA
-59,ieigg-name,ayq-firstname,12960,ljc-city,1962/07/05,dnjz-street,163,FL,FLORIDA
-60,rfvzu-name,edm-firstname,13340,kvz-city,1954/12/08,eijd-street,4,RI,RHODE ISLAND
-61,pduwm-name,gqb-firstname,14240,cyr-city,1954/07/03,ndux-street,13,SD,SOUTH DAKOTA
-62,yyixf-name,yzt-firstname,18020,lwx-city,1974/01/29,iede-street,120,NV,NEVADA
-63,dkszq-name,ytd-firstname,14700,zwh-city,1979/01/11,nbjz-street,65,AS,AMERICAN SAMOA
-64,slkzv-name,zbg-firstname,19880,oee-city,1978/11/01,sphg-street,119,OK,OKLAHOMA
-65,nvxim-name,phc-firstname,19220,vgg-city,1991/01/24,juok-street,106,FM,FEDERATED STATES OF MICRONESIA
-66,piyfg-name,xtn-firstname,13760,nde-city,1954/07/22,vfrv-street,11,NY,NEW YORK
-67,jnusz-name,mjw-firstname,12640,nwb-city,1986/08/23,kcsa-street,138,VA,VIRGINIA
-68,jnypj-name,ioq-firstname,17000,zqy-city,1986/01/09,croe-street,119,PW,PALAU
-69,uohts-name,btx-firstname,13480,dal-city,1990/10/22,llyw-street,150,WA,WASHINGTON
-70,aavpj-name,pvw-firstname,13780,lai-city,1954/09/23,nygu-street,171,FL,FLORIDA
-71,nbjcj-name,rsf-firstname,12000,kjl-city,1986/06/30,ijsb-street,123,ID,IDAHO
-72,syjxh-name,gkq-firstname,19960,rmd-city,1978/10/26,qmyp-street,161,MN,MINNESOTA
-73,vkojz-name,ryo-firstname,14300,bmz-city,1954/09/11,gcpj-street,71,ND,NORTH DAKOTA
-74,pqzfw-name,kld-firstname,16400,qvq-city,1962/09/09,dhbv-street,92,ND,NORTH DAKOTA
-75,owvjk-name,fez-firstname,19740,ldb-city,1978/06/14,kabf-street,87,VA,VIRGINIA
-76,qsfih-name,ixe-firstname,16860,qvr-city,1987/01/07,qean-street,159,CO,COLORADO
-77,slixq-name,gmb-firstname,19980,ftt-city,1982/06/22,xinx-street,111,VT,VERMONT
-78,eegsa-name,xlc-firstname,12680,byk-city,1954/04/23,beul-street,56,MD,MARYLAND
-79,phevp-name,ihs-firstname,16120,adc-city,1978/04/25,voig-street,98,NM,NEW MEXICO
-80,njfoe-name,tag-firstname,16580,tnr-city,1966/12/04,dhky-street,108,LA,LOUISIANA
-81,bdncx-name,hcd-firstname,11260,xcl-city,1970/07/02,jvlp-street,49,GA,GEORGIA
-82,ikedo-name,tks-firstname,17460,odl-city,1958/08/25,iaaq-street,8,GU,GUAM
-83,iafxy-name,vur-firstname,11480,hgt-city,1962/08/03,hmec-street,164,TX,TEXAS
-84,lafhf-name,ssz-firstname,19560,wwp-city,1951/01/25,mxmq-street,96,IN,INDIANA
-85,okyny-name,hbu-firstname,16800,yok-city,1978/03/28,ipjz-street,135,NV,NEVADA
-86,hznby-name,fwy-firstname,13680,wbi-city,1970/07/25,mxui-street,170,CT,CONNECTICUT
-87,ztpoa-name,rzk-firstname,18500,qum-city,1970/07/26,blqr-street,152,ME,MAINE
-88,gitxz-name,axt-firstname,11800,fck-city,1974/01/12,tmjw-street,189,SD,SOUTH DAKOTA
-89,ziomm-name,mcv-firstname,12940,iwq-city,1950/10/22,hqgj-street,140,DC,DISTRICT OF COLUMBIA
-90,otncg-name,tuy-firstname,16540,ulk-city,1971/01/24,yuia-street,166,TX,TEXAS
-91,cnabb-name,hoq-firstname,16300,tuw-city,1962/06/17,ujvv-street,61,ME,MAINE
-92,ucogf-name,ggc-firstname,14500,fsj-city,1978/02/08,asfi-street,53,WV,WEST VIRGINIA
-93,lbpmf-name,sdt-firstname,10780,ewj-city,1978/03/08,hxsp-street,102,NV,NEVADA
-94,tieqq-name,uyu-firstname,17740,wea-city,1966/10/31,abpl-street,187,MO,MISSOURI
-95,fsgwf-name,vjd-firstname,12460,ads-city,1970/11/29,yeou-street,10,MA,MASSACHUSETTS
-96,reeba-name,kzs-firstname,13100,zhc-city,1966/07/08,abmv-street,88,FL,FLORIDA
-97,shybc-name,gcp-firstname,10660,ahg-city,1950/12/15,hrqy-street,174,KS,KANSAS
-98,phszr-name,sst-firstname,13080,ydd-city,1954/09/23,quqn-street,2,RI,RHODE ISLAND
-99,jteco-name,fxc-firstname,19760,agr-city,1986/05/06,dzxc-street,108,MD,MARYLAND
-100,qvaar-name,icx-firstname,16120,boc-city,1978/08/04,bfzf-street,12,NM,NEW MEXICO
+id,name,firstname,zip,city,birthdate,street,housenr,stateCode,state,isActive,registrationTimestamp,metadataJson
+1,jwcdf-name,fsj-firstname,13520,oem-city,1954/02/07,amrb-street,145,AK,ALASKA,true,2025-12-16 09:47:47.123,"{""ip"":""192.168.1.10"",""session_id"":""a1b2c3d4"",""duration_sec"":120}"
+2,flhxu-name,tum-firstname,17520,buo-city,1966/04/24,wfyz-street,96,GA,GEORGIA,false,2024-07-02 15:06:40.456,"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+3,xthfg-name,gfe-firstname,12560,vtz-city,1990/01/11,doxx-street,46,NJ,NEW JERSEY,true,2024-01-01 00:00:00.123,"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+4,ulzrz-name,bnl-firstname,11620,prz-city,1966/08/02,bxqn-street,104,NY,NEW YORK,false,2023-03-31 12:00:00.789,"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+5,oxhyr-name,onx-firstname,15180,bpn-city,1970/11/14,pksn-street,133,IN,INDIANA,true,2022-09-01 05:59:59.123,"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+6,fiqjz-name,sce-firstname,16020,fnn-city,1954/09/24,wbhg-street,35,MD,MARYLAND,false,2022-01-01 00:00:00.000,"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
+7,tkiat-name,xti-firstname,12720,stt-city,1966/08/11,tvnf-street,21,PA,PENNSYLVANIA,true,2021-01-01 00:00:00.123,"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+8,kljcz-name,uqd-firstname,13340,ntt-city,1987/01/15,jyje-street,10,PW,PALAU,false,2020-01-01 00:00:00.456,"{""ip"":""10.0.3.4"",""session_id"":""d9e0f1g2"",""duration_sec"":180}"
+9,pgunz-name,hcm-firstname,16680,gxh-city,1970/11/08,shbe-street,184,NC,NORTH CAROLINA,true,2019-01-01 00:00:00.789,"{""ip"":""172.16.8.9"",""session_id"":""h3i4j5k6"",""duration_sec"":75}"
+10,oyjha-name,uhj-firstname,18880,uyg-city,1966/04/10,bjgw-street,176,AR,ARKANSAS,false,2023-03-31 12:00:00.789,"{""ip"":""192.168.2.34"",""session_id"":""l7m8n9o0"",""duration_sec"":330}"
+11,igxbd-name,uph-firstname,13480,ndh-city,1962/12/03,jdcd-street,151,NH,NEW HAMPSHIRE,true,2022-09-01 05:59:59.123,"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+12,vnaov-name,wha-firstname,13120,egm-city,1954/03/28,hpep-street,20,CA,CALIFORNIA,false,2022-01-01 00:00:00.000,"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+13,dauuz-name,hwg-firstname,13740,khn-city,1958/05/15,etqx-street,5,OK,OKLAHOMA,true,2021-01-01 00:00:00.123,"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+14,gkuuo-name,kkb-firstname,13560,xdt-city,1962/04/07,sdoj-street,35,MT,MONTANA,false,2020-01-01 00:00:00.456,"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+15,wdhze-name,jjk-firstname,16900,due-city,1970/07/17,pmmu-street,174,AS,AMERICAN SAMOA,true,2024-01-01 00:00:00.123,"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
+16,ncayz-name,ynb-firstname,15720,lxj-city,1974/04/27,mdtb-street,109,MA,MASSACHUSETTS,false,2023-03-31 12:00:00.789,"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+17,rdjin-name,hhu-firstname,14480,lpc-city,1958/11/16,wxik-street,145,KY,KENTUCKY,true,2022-09-01 05:59:59.123,"{""ip"":""192.168.1.10"",""session_id"":""a1b2c3d4"",""duration_sec"":120}"
+18,nxzij-name,bdl-firstname,10740,avx-city,1958/02/20,nybz-street,138,WI,WISCONSIN,false,2022-01-01 00:00:00.000,"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+19,xgrzc-name,dxw-firstname,18900,vpq-city,1990/11/16,wzjh-street,58,ME,MAINE,true,2021-01-01 00:00:00.123,"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+20,ehgrn-name,vbe-firstname,17500,cik-city,1978/05/21,ucnw-street,135,MD,MARYLAND,false,2020-01-01 00:00:00.456,"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+21,gctjx-name,upx-firstname,11960,yqr-city,1958/03/03,rlko-street,141,TN,TENNESSEE,true,2019-01-01 00:00:00.789,"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+22,ptzmg-name,hva-firstname,15740,gux-city,1978/05/04,pugy-street,122,VI,VIRGIN ISLANDS,false,2023-03-31 12:00:00.789,"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
+23,eyeti-name,gnw-firstname,17420,eko-city,1962/10/26,ylph-street,61,NC,NORTH CAROLINA,true,2022-09-01 05:59:59.123,"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+24,wccwo-name,zpj-firstname,16600,uim-city,1962/09/29,ygih-street,26,WA,WASHINGTON,false,2022-01-01 00:00:00.000,"{""ip"":""10.0.3.4"",""session_id"":""d9e0f1g2"",""duration_sec"":180}"
+25,bwkoe-name,ayl-firstname,18660,rtw-city,1978/07/16,mzww-street,179,CA,CALIFORNIA,true,2021-01-01 00:00:00.123,"{""ip"":""172.16.8.9"",""session_id"":""h3i4j5k6"",""duration_sec"":75}"
+26,rezku-name,zio-firstname,19080,nvt-city,1982/07/14,wwkd-street,91,CA,CALIFORNIA,false,2023-03-31 12:00:00.789,"{""ip"":""192.168.2.34"",""session_id"":""l7m8n9o0"",""duration_sec"":330}"
+27,mjlsk-name,ecx-firstname,10800,yxu-city,1950/12/11,vttb-street,195,MO,MISSOURI,true,2022-09-01 05:59:59.123,"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+28,wdjsi-name,aoq-firstname,13660,smo-city,1954/02/01,kako-street,7,NV,NEVADA,false,2022-01-01 00:00:00.000,"{""ip"":""192.168.1.10"",""session_id"":""a1b2c3d4"",""duration_sec"":120}"
+29,mwfnd-name,nyb-firstname,19760,bbu-city,1986/09/23,apdi-street,91,MS,MISSISSIPPI,true,2021-01-01 00:00:00.123,"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+30,vtuoz-name,jhh-firstname,17620,vad-city,1982/05/05,kzup-street,79,GA,GEORGIA,false,2020-01-01 00:00:00.456,"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+31,rhhxk-name,ndr-firstname,16760,fub-city,1978/11/12,regd-street,55,OK,OKLAHOMA,true,2019-01-01 00:00:00.789,"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+32,lpstk-name,mqz-firstname,18940,tnr-city,1982/09/16,cdhf-street,4,SD,SOUTH DAKOTA,false,2023-03-31 12:00:00.789,"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+33,ldhyr-name,yts-firstname,12000,auk-city,1986/11/14,abph-street,147,IN,INDIANA,true,2022-09-01 05:59:59.123,"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
+34,cjdml-name,iti-firstname,16900,wkq-city,1970/06/05,npow-street,96,NH,NEW HAMPSHIRE,false,2022-01-01 00:00:00.000,"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+35,cpenz-name,sbi-firstname,16380,ssl-city,1962/08/19,kilz-street,44,MS,MISSISSIPPI,true,2021-01-01 00:00:00.123,"{""ip"":""10.0.3.4"",""session_id"":""d9e0f1g2"",""duration_sec"":180}"
+36,rxtbg-name,anr-firstname,14720,bqc-city,1958/08/10,pudg-street,140,NV,NEVADA,false,2020-01-01 00:00:00.456,"{""ip"":""172.16.8.9"",""session_id"":""h3i4j5k6"",""duration_sec"":75}"
+37,udblf-name,raa-firstname,11500,wli-city,1978/12/13,xomd-street,41,PW,PALAU,true,2019-01-01 00:00:00.789,"{""ip"":""192.168.2.34"",""session_id"":""l7m8n9o0"",""duration_sec"":330}"
+38,vvyce-name,gep-firstname,13740,gtd-city,1982/05/23,kwbv-street,123,undefined,undefined,false,2023-03-31 12:00:00.789,"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+39,kwfnz-name,ucu-firstname,10580,sns-city,1978/08/18,nnun-street,20,OK,OKLAHOMA,true,2022-09-01 05:59:59.123,"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+40,zxydx-name,tml-firstname,14680,jda-city,1974/05/29,wfjn-street,157,DC,DISTRICT OF COLUMBIA,false,2022-01-01 00:00:00.000,"{""ip"":""192.168.1.10"",""session_id"":""a1b2c3d4"",""duration_sec"":120}"
+41,bfscx-name,jnl-firstname,16920,yyg-city,1970/11/30,cgfh-street,178,CO,COLORADO,true,2021-01-01 00:00:00.123,"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+42,qitur-name,yra-firstname,15560,ijp-city,1978/01/30,fonc-street,155,AK,ALASKA,false,2020-01-01 00:00:00.456,"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+43,msixi-name,ynb-firstname,12720,ksl-city,1958/07/17,zpjw-street,46,VI,VIRGIN ISLANDS,true,2024-01-01 00:00:00.123,"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+44,wzkjq-name,rgh-firstname,19000,hkm-city,1974/08/12,yixf-street,134,CA,CALIFORNIA,false,2023-03-31 12:00:00.789,"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+45,dqfmf-name,yxr-firstname,13840,vie-city,1962/10/23,stvx-street,39,TX,TEXAS,true,2022-09-01 05:59:59.123,"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
+46,biluz-name,uqe-firstname,17760,wkq-city,1962/07/27,embn-street,183,PW,PALAU,false,2022-01-01 00:00:00.000,"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+47,wahfx-name,zwd-firstname,13240,vic-city,1974/03/27,axpw-street,131,UT,UTAH,true,2021-01-01 00:00:00.123,"{""ip"":""10.0.3.4"",""session_id"":""d9e0f1g2"",""duration_sec"":180}"
+48,denwt-name,bta-firstname,17300,hhj-city,1986/12/20,orwy-street,11,WV,WEST VIRGINIA,false,2020-01-01 00:00:00.456,"{""ip"":""172.16.8.9"",""session_id"":""h3i4j5k6"",""duration_sec"":75}"
+49,akdmy-name,ybz-firstname,14560,wtx-city,1962/11/08,nwba-street,123,MP,NORTHERN MARIANA ISLANDS,true,2019-01-01 00:00:00.789,"{""ip"":""192.168.2.34"",""session_id"":""l7m8n9o0"",""duration_sec"":330}"
+50,hqafg-name,nht-firstname,16080,gfu-city,1951/01/12,spsq-street,45,LA,LOUISIANA,false,2023-03-31 12:00:00.789,"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+51,zhmbl-name,lnw-firstname,17460,hse-city,1986/12/21,scis-street,97,GA,GEORGIA,true,2022-09-01 05:59:59.123,"{""ip"":""192.168.1.10"",""session_id"":""a1b2c3d4"",""duration_sec"":120}"
+52,snwnj-name,jyy-firstname,16400,hsz-city,1966/02/15,imhl-street,42,NC,NORTH CAROLINA,false,2022-01-01 00:00:00.000,"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+53,fuyla-name,mmp-firstname,11840,hgu-city,1986/08/16,ixiz-street,145,NC,NORTH CAROLINA,true,2021-01-01 00:00:00.123,"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+54,yvfqz-name,prz-firstname,11260,wjl-city,1982/05/06,fbzd-street,97,MO,MISSOURI,false,2023-03-31 12:00:00.789,"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+55,usbgq-name,vhd-firstname,14080,dsb-city,1958/04/01,ggoc-street,54,KS,KANSAS,true,2022-09-01 05:59:59.123,"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+56,yaeni-name,zpy-firstname,19100,sen-city,1954/12/10,sbsw-street,158,HI,HAWAII,false,2023-03-31 12:00:00.789,"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
+57,fgxvr-name,vzi-firstname,17520,lcf-city,1958/11/01,nbdv-street,10,GU,GUAM,true,2022-09-01 05:59:59.123,"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+58,tqpbq-name,rwr-firstname,19140,zpd-city,1978/08/23,npvb-street,190,DC,DISTRICT OF COLUMBIA,false,2022-01-01 00:00:00.000,"{""ip"":""10.0.3.4"",""session_id"":""d9e0f1g2"",""duration_sec"":180}"
+59,ieigg-name,ayq-firstname,12960,ljc-city,1962/07/05,dnjz-street,163,FL,FLORIDA,true,2021-01-01 00:00:00.123,"{""ip"":""172.16.8.9"",""session_id"":""h3i4j5k6"",""duration_sec"":75}"
+60,rfvzu-name,edm-firstname,13340,kvz-city,1954/12/08,eijd-street,4,RI,RHODE ISLAND,false,2020-01-01 00:00:00.456,"{""ip"":""192.168.2.34"",""session_id"":""l7m8n9o0"",""duration_sec"":330}"
+61,pduwm-name,gqb-firstname,14240,cyr-city,1954/07/03,ndux-street,13,SD,SOUTH DAKOTA,true,2019-01-01 00:00:00.789,"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+62,yyixf-name,yzt-firstname,18020,lwx-city,1974/01/29,iede-street,120,NV,NEVADA,false,2023-03-31 12:00:00.789,"{""ip"":""192.168.1.10"",""session_id"":""a1b2c3d4"",""duration_sec"":120}"
+63,dkszq-name,ytd-firstname,14700,zwh-city,1979/01/11,nbjz-street,65,AS,AMERICAN SAMOA,true,2022-09-01 05:59:59.123,"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+64,slkzv-name,zbg-firstname,19880,oee-city,1978/11/01,sphg-street,119,OK,OKLAHOMA,false,2022-01-01 00:00:00.000,"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+65,nvxim-name,phc-firstname,19220,vgg-city,1991/01/24,juok-street,106,FM,FEDERATED STATES OF MICRONESIA,true,2021-01-01 00:00:00.123,"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+66,piyfg-name,xtn-firstname,13760,nde-city,1954/07/22,vfrv-street,11,NY,NEW YORK,false,2020-01-01 00:00:00.456,"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+67,jnusz-name,mjw-firstname,12640,nwb-city,1986/08/23,kcsa-street,138,VA,VIRGINIA,true,2024-01-01 00:00:00.123,"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
+68,jnypj-name,ioq-firstname,17000,zqy-city,1986/01/09,croe-street,119,PW,PALAU,false,2023-03-31 12:00:00.789,"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+69,uohts-name,btx-firstname,13480,dal-city,1990/10/22,llyw-street,150,WA,WASHINGTON,true,2022-09-01 05:59:59.123,"{""ip"":""10.0.3.4"",""session_id"":""d9e0f1g2"",""duration_sec"":180}"
+70,aavpj-name,pvw-firstname,13780,lai-city,1954/09/23,nygu-street,171,FL,FLORIDA,false,2022-01-01 00:00:00.000,"{""ip"":""172.16.8.9"",""session_id"":""h3i4j5k6"",""duration_sec"":75}"
+71,nbjcj-name,rsf-firstname,12000,kjl-city,1986/06/30,ijsb-street,123,ID,IDAHO,true,2021-01-01 00:00:00.123,"{""ip"":""192.168.2.34"",""session_id"":""l7m8n9o0"",""duration_sec"":330}"
+72,syjxh-name,gkq-firstname,19960,rmd-city,1978/10/26,qmyp-street,161,MN,MINNESOTA,false,2020-01-01 00:00:00.456,"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+73,vkojz-name,ryo-firstname,14300,bmz-city,1954/09/11,gcpj-street,71,ND,NORTH DAKOTA,true,2019-01-01 00:00:00.789,"{""ip"":""192.168.1.10"",""session_id"":""a1b2c3d4"",""duration_sec"":120}"
+74,pqzfw-name,kld-firstname,16400,qvq-city,1962/09/09,dhbv-street,92,ND,NORTH DAKOTA,false,2023-03-31 12:00:00.789,"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+75,owvjk-name,fez-firstname,19740,ldb-city,1978/06/14,kabf-street,87,VA,VIRGINIA,true,2022-09-01 05:59:59.123,"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+76,qsfih-name,ixe-firstname,16860,qvr-city,1987/01/07,qean-street,159,CO,COLORADO,false,2022-01-01 00:00:00.000,"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+77,slixq-name,gmb-firstname,19980,ftt-city,1982/06/22,xinx-street,111,VT,VERMONT,true,2021-01-01 00:00:00.123,"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+78,eegsa-name,xlc-firstname,12680,byk-city,1954/04/23,beul-street,56,MD,MARYLAND,false,2023-03-31 12:00:00.789,"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
+79,phevp-name,ihs-firstname,16120,adc-city,1978/04/25,voig-street,98,NM,NEW MEXICO,true,2022-09-01 05:59:59.123,"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+80,njfoe-name,tag-firstname,16580,tnr-city,1966/12/04,dhky-street,108,LA,LOUISIANA,false,2023-03-31 12:00:00.789,"{""ip"":""10.0.3.4"",""session_id"":""d9e0f1g2"",""duration_sec"":180}"
+81,bdncx-name,hcd-firstname,11260,xcl-city,1970/07/02,jvlp-street,49,GA,GEORGIA,true,2022-09-01 05:59:59.123,"{""ip"":""172.16.8.9"",""session_id"":""h3i4j5k6"",""duration_sec"":75}"
+82,ikedo-name,tks-firstname,17460,odl-city,1958/08/25,iaaq-street,8,GU,GUAM,false,2022-01-01 00:00:00.000,"{""ip"":""192.168.2.34"",""session_id"":""l7m8n9o0"",""duration_sec"":330}"
+83,iafxy-name,vur-firstname,11480,hgt-city,1962/08/03,hmec-street,164,TX,TEXAS,true,2021-01-01 00:00:00.123,"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+84,lafhf-name,ssz-firstname,19560,wwp-city,1951/01/25,mxmq-street,96,IN,INDIANA,false,2020-01-01 00:00:00.456,"{""ip"":""192.168.1.10"",""session_id"":""a1b2c3d4"",""duration_sec"":120}"
+85,okyny-name,hbu-firstname,16800,yok-city,1978/03/28,ipjz-street,135,NV,NEVADA,true,2019-01-01 00:00:00.789,"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+86,hznby-name,fwy-firstname,13680,wbi-city,1970/07/25,mxui-street,170,CT,CONNECTICUT,false,2023-03-31 12:00:00.789,"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+87,ztpoa-name,rzk-firstname,18500,qum-city,1970/07/26,blqr-street,152,ME,MAINE,true,2022-09-01 05:59:59.123,"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+88,gitxz-name,axt-firstname,11800,fck-city,1974/01/12,tmjw-street,189,SD,SOUTH DAKOTA,false,2022-01-01 00:00:00.000,"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+89,ziomm-name,mcv-firstname,12940,iwq-city,1950/10/22,hqgj-street,140,DC,DISTRICT OF COLUMBIA,true,2021-01-01 00:00:00.123,"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
+90,otncg-name,tuy-firstname,16540,ulk-city,1971/01/24,yuia-street,166,TX,TEXAS,false,2020-01-01 00:00:00.456,"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+91,cnabb-name,hoq-firstname,16300,tuw-city,1962/06/17,ujvv-street,61,ME,MAINE,true,2024-01-01 00:00:00.123,"{""ip"":""10.0.3.4"",""session_id"":""d9e0f1g2"",""duration_sec"":180}"
+92,ucogf-name,ggc-firstname,14500,fsj-city,1978/02/08,asfi-street,53,WV,WEST VIRGINIA,false,2023-03-31 12:00:00.789,"{""ip"":""172.16.8.9"",""session_id"":""h3i4j5k6"",""duration_sec"":75}"
+93,lbpmf-name,sdt-firstname,10780,ewj-city,1978/03/08,hxsp-street,102,NV,NEVADA,true,2022-09-01 05:59:59.123,"{""ip"":""192.168.2.34"",""session_id"":""l7m8n9o0"",""duration_sec"":330}"
+94,tieqq-name,uyu-firstname,17740,wea-city,1966/10/31,abpl-street,187,MO,MISSOURI,false,2022-01-01 00:00:00.000,"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+95,fsgwf-name,vjd-firstname,12460,ads-city,1970/11/29,yeou-street,10,MA,MASSACHUSETTS,true,2021-01-01 00:00:00.123,"{""ip"":""192.168.1.10"",""session_id"":""a1b2c3d4"",""duration_sec"":120}"
+96,reeba-name,kzs-firstname,13100,zhc-city,1966/07/08,abmv-street,88,FL,FLORIDA,false,2020-01-01 00:00:00.456,"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+97,shybc-name,gcp-firstname,10660,ahg-city,1950/12/15,hrqy-street,174,KS,KANSAS,true,2019-01-01 00:00:00.789,"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+98,phszr-name,sst-firstname,13080,ydd-city,1954/09/23,quqn-street,2,RI,RHODE ISLAND,false,2023-03-31 12:00:00.789,"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+99,jteco-name,fxc-firstname,19760,agr-city,1986/05/06,dzxc-street,108,MD,MARYLAND,true,2022-09-01 05:59:59.123,"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+100,qvaar-name,icx-firstname,16120,boc-city,1978/08/04,bfzf-street,12,NM,NEW MEXICO,false,2022-01-01 00:00:00.000,"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
diff --git a/integration-tests/transforms/files/customers-100.txt b/integration-tests/transforms/files/customers-100.txt
index 8b8bd04955..fca1fbb8d6 100644
--- a/integration-tests/transforms/files/customers-100.txt
+++ b/integration-tests/transforms/files/customers-100.txt
@@ -1,101 +1,101 @@
-id;name;firstname;zip;city;birthdate;street;housenr;stateCode;state
- 1;jwcdf-name;fsj-firstname; 13520;oem-city;1954/02/07;amrb-street; 145;AK;ALASKA
- 2;flhxu-name;tum-firstname; 17520;buo-city;1966/04/24;wfyz-street; 96;GA;GEORGIA
- 3;xthfg-name;gfe-firstname; 12560;vtz-city;1990/01/11;doxx-street; 46;NJ;NEW JERSEY
- 4;ulzrz-name;bnl-firstname; 11620;prz-city;1966/08/02;bxqn-street; 104;NY;NEW YORK
- 5;oxhyr-name;onx-firstname; 15180;bpn-city;1970/11/14;pksn-street; 133;IN;INDIANA
- 6;fiqjz-name;sce-firstname; 16020;fnn-city;1954/09/24;wbhg-street; 35;MD;MARYLAND
- 7;tkiat-name;xti-firstname; 12720;stt-city;1966/08/11;tvnf-street; 21;PA;PENNSYLVANIA
- 8;kljcz-name;uqd-firstname; 13340;ntt-city;1987/01/15;jyje-street; 10;PW;PALAU
- 9;pgunz-name;hcm-firstname; 16680;gxh-city;1970/11/08;shbe-street; 184;NC;NORTH CAROLINA
- 10;oyjha-name;uhj-firstname; 18880;uyg-city;1966/04/10;bjgw-street; 176;AR;ARKANSAS
- 11;igxbd-name;uph-firstname; 13480;ndh-city;1962/12/03;jdcd-street; 151;NH;NEW HAMPSHIRE
- 12;vnaov-name;wha-firstname; 13120;egm-city;1954/03/28;hpep-street; 20;CA;CALIFORNIA
- 13;dauuz-name;hwg-firstname; 13740;khn-city;1958/05/15;etqx-street; 5;OK;OKLAHOMA
- 14;gkuuo-name;kkb-firstname; 13560;xdt-city;1962/04/07;sdoj-street; 35;MT;MONTANA
- 15;wdhze-name;jjk-firstname; 16900;due-city;1970/07/17;pmmu-street; 174;AS;AMERICAN SAMOA
- 16;ncayz-name;ynb-firstname; 15720;lxj-city;1974/04/27;mdtb-street; 109;MA;MASSACHUSETTS
- 17;rdjin-name;hhu-firstname; 14480;lpc-city;1958/11/16;wxik-street; 145;KY;KENTUCKY
- 18;nxzij-name;bdl-firstname; 10740;avx-city;1958/02/20;nybz-street; 138;WI;WISCONSIN
- 19;xgrzc-name;dxw-firstname; 18900;vpq-city;1990/11/16;wzjh-street; 58;ME;MAINE
- 20;ehgrn-name;vbe-firstname; 17500;cik-city;1978/05/21;ucnw-street; 135;MD;MARYLAND
- 21;gctjx-name;upx-firstname; 11960;yqr-city;1958/03/03;rlko-street; 141;TN;TENNESSEE
- 22;ptzmg-name;hva-firstname; 15740;gux-city;1978/05/04;pugy-street; 122;VI;VIRGIN ISLANDS
- 23;eyeti-name;gnw-firstname; 17420;eko-city;1962/10/26;ylph-street; 61;NC;NORTH CAROLINA
- 24;wccwo-name;zpj-firstname; 16600;uim-city;1962/09/29;ygih-street; 26;WA;WASHINGTON
- 25;bwkoe-name;ayl-firstname; 18660;rtw-city;1978/07/16;mzww-street; 179;CA;CALIFORNIA
- 26;rezku-name;zio-firstname; 19080;nvt-city;1982/07/14;wwkd-street; 91;CA;CALIFORNIA
- 27;mjlsk-name;ecx-firstname; 10800;yxu-city;1950/12/11;vttb-street; 195;MO;MISSOURI
- 28;wdjsi-name;aoq-firstname; 13660;smo-city;1954/02/01;kako-street; 7;NV;NEVADA
- 29;mwfnd-name;nyb-firstname; 19760;bbu-city;1986/09/23;apdi-street; 91;MS;MISSISSIPPI
- 30;vtuoz-name;jhh-firstname; 17620;vad-city;1982/05/05;kzup-street; 79;GA;GEORGIA
- 31;rhhxk-name;ndr-firstname; 16760;fub-city;1978/11/12;regd-street; 55;OK;OKLAHOMA
- 32;lpstk-name;mqz-firstname; 18940;tnr-city;1982/09/16;cdhf-street; 4;SD;SOUTH DAKOTA
- 33;ldhyr-name;yts-firstname; 12000;auk-city;1986/11/14;abph-street; 147;IN;INDIANA
- 34;cjdml-name;iti-firstname; 16900;wkq-city;1970/06/05;npow-street; 96;NH;NEW HAMPSHIRE
- 35;cpenz-name;sbi-firstname; 16380;ssl-city;1962/08/19;kilz-street; 44;MS;MISSISSIPPI
- 36;rxtbg-name;anr-firstname; 14720;bqc-city;1958/08/10;pudg-street; 140;NV;NEVADA
- 37;udblf-name;raa-firstname; 11500;wli-city;1978/12/13;xomd-street; 41;PW;PALAU
- 38;vvyce-name;gep-firstname; 13740;gtd-city;1982/05/23;kwbv-street; 123;undefined;undefined
- 39;kwfnz-name;ucu-firstname; 10580;sns-city;1978/08/18;nnun-street; 20;OK;OKLAHOMA
- 40;zxydx-name;tml-firstname; 14680;jda-city;1974/05/29;wfjn-street; 157;DC;DISTRICT OF COLUMBIA
- 41;bfscx-name;jnl-firstname; 16920;yyg-city;1970/11/30;cgfh-street; 178;CO;COLORADO
- 42;qitur-name;yra-firstname; 15560;ijp-city;1978/01/30;fonc-street; 155;AK;ALASKA
- 43;msixi-name;ynb-firstname; 12720;ksl-city;1958/07/17;zpjw-street; 46;VI;VIRGIN ISLANDS
- 44;wzkjq-name;rgh-firstname; 19000;hkm-city;1974/08/12;yixf-street; 134;CA;CALIFORNIA
- 45;dqfmf-name;yxr-firstname; 13840;vie-city;1962/10/23;stvx-street; 39;TX;TEXAS
- 46;biluz-name;uqe-firstname; 17760;wkq-city;1962/07/27;embn-street; 183;PW;PALAU
- 47;wahfx-name;zwd-firstname; 13240;vic-city;1974/03/27;axpw-street; 131;UT;UTAH
- 48;denwt-name;bta-firstname; 17300;hhj-city;1986/12/20;orwy-street; 11;WV;WEST VIRGINIA
- 49;akdmy-name;ybz-firstname; 14560;wtx-city;1962/11/08;nwba-street; 123;MP;NORTHERN MARIANA ISLANDS
- 50;hqafg-name;nht-firstname; 16080;gfu-city;1951/01/12;spsq-street; 45;LA;LOUISIANA
- 51;zhmbl-name;lnw-firstname; 17460;hse-city;1986/12/21;scis-street; 97;GA;GEORGIA
- 52;snwnj-name;jyy-firstname; 16400;hsz-city;1966/02/15;imhl-street; 42;NC;NORTH CAROLINA
- 53;fuyla-name;mmp-firstname; 11840;hgu-city;1986/08/16;ixiz-street; 145;NC;NORTH CAROLINA
- 54;yvfqz-name;prz-firstname; 11260;wjl-city;1982/05/06;fbzd-street; 97;MO;MISSOURI
- 55;usbgq-name;vhd-firstname; 14080;dsb-city;1958/04/01;ggoc-street; 54;KS;KANSAS
- 56;yaeni-name;zpy-firstname; 19100;sen-city;1954/12/10;sbsw-street; 158;HI;HAWAII
- 57;fgxvr-name;vzi-firstname; 17520;lcf-city;1958/11/01;nbdv-street; 10;GU;GUAM
- 58;tqpbq-name;rwr-firstname; 19140;zpd-city;1978/08/23;npvb-street; 190;DC;DISTRICT OF COLUMBIA
- 59;ieigg-name;ayq-firstname; 12960;ljc-city;1962/07/05;dnjz-street; 163;FL;FLORIDA
- 60;rfvzu-name;edm-firstname; 13340;kvz-city;1954/12/08;eijd-street; 4;RI;RHODE ISLAND
- 61;pduwm-name;gqb-firstname; 14240;cyr-city;1954/07/03;ndux-street; 13;SD;SOUTH DAKOTA
- 62;yyixf-name;yzt-firstname; 18020;lwx-city;1974/01/29;iede-street; 120;NV;NEVADA
- 63;dkszq-name;ytd-firstname; 14700;zwh-city;1979/01/11;nbjz-street; 65;AS;AMERICAN SAMOA
- 64;slkzv-name;zbg-firstname; 19880;oee-city;1978/11/01;sphg-street; 119;OK;OKLAHOMA
- 65;nvxim-name;phc-firstname; 19220;vgg-city;1991/01/24;juok-street; 106;FM;FEDERATED STATES OF MICRONESIA
- 66;piyfg-name;xtn-firstname; 13760;nde-city;1954/07/22;vfrv-street; 11;NY;NEW YORK
- 67;jnusz-name;mjw-firstname; 12640;nwb-city;1986/08/23;kcsa-street; 138;VA;VIRGINIA
- 68;jnypj-name;ioq-firstname; 17000;zqy-city;1986/01/09;croe-street; 119;PW;PALAU
- 69;uohts-name;btx-firstname; 13480;dal-city;1990/10/22;llyw-street; 150;WA;WASHINGTON
- 70;aavpj-name;pvw-firstname; 13780;lai-city;1954/09/23;nygu-street; 171;FL;FLORIDA
- 71;nbjcj-name;rsf-firstname; 12000;kjl-city;1986/06/30;ijsb-street; 123;ID;IDAHO
- 72;syjxh-name;gkq-firstname; 19960;rmd-city;1978/10/26;qmyp-street; 161;MN;MINNESOTA
- 73;vkojz-name;ryo-firstname; 14300;bmz-city;1954/09/11;gcpj-street; 71;ND;NORTH DAKOTA
- 74;pqzfw-name;kld-firstname; 16400;qvq-city;1962/09/09;dhbv-street; 92;ND;NORTH DAKOTA
- 75;owvjk-name;fez-firstname; 19740;ldb-city;1978/06/14;kabf-street; 87;VA;VIRGINIA
- 76;qsfih-name;ixe-firstname; 16860;qvr-city;1987/01/07;qean-street; 159;CO;COLORADO
- 77;slixq-name;gmb-firstname; 19980;ftt-city;1982/06/22;xinx-street; 111;VT;VERMONT
- 78;eegsa-name;xlc-firstname; 12680;byk-city;1954/04/23;beul-street; 56;MD;MARYLAND
- 79;phevp-name;ihs-firstname; 16120;adc-city;1978/04/25;voig-street; 98;NM;NEW MEXICO
- 80;njfoe-name;tag-firstname; 16580;tnr-city;1966/12/04;dhky-street; 108;LA;LOUISIANA
- 81;bdncx-name;hcd-firstname; 11260;xcl-city;1970/07/02;jvlp-street; 49;GA;GEORGIA
- 82;ikedo-name;tks-firstname; 17460;odl-city;1958/08/25;iaaq-street; 8;GU;GUAM
- 83;iafxy-name;vur-firstname; 11480;hgt-city;1962/08/03;hmec-street; 164;TX;TEXAS
- 84;lafhf-name;ssz-firstname; 19560;wwp-city;1951/01/25;mxmq-street; 96;IN;INDIANA
- 85;okyny-name;hbu-firstname; 16800;yok-city;1978/03/28;ipjz-street; 135;NV;NEVADA
- 86;hznby-name;fwy-firstname; 13680;wbi-city;1970/07/25;mxui-street; 170;CT;CONNECTICUT
- 87;ztpoa-name;rzk-firstname; 18500;qum-city;1970/07/26;blqr-street; 152;ME;MAINE
- 88;gitxz-name;axt-firstname; 11800;fck-city;1974/01/12;tmjw-street; 189;SD;SOUTH DAKOTA
- 89;ziomm-name;mcv-firstname; 12940;iwq-city;1950/10/22;hqgj-street; 140;DC;DISTRICT OF COLUMBIA
- 90;otncg-name;tuy-firstname; 16540;ulk-city;1971/01/24;yuia-street; 166;TX;TEXAS
- 91;cnabb-name;hoq-firstname; 16300;tuw-city;1962/06/17;ujvv-street; 61;ME;MAINE
- 92;ucogf-name;ggc-firstname; 14500;fsj-city;1978/02/08;asfi-street; 53;WV;WEST VIRGINIA
- 93;lbpmf-name;sdt-firstname; 10780;ewj-city;1978/03/08;hxsp-street; 102;NV;NEVADA
- 94;tieqq-name;uyu-firstname; 17740;wea-city;1966/10/31;abpl-street; 187;MO;MISSOURI
- 95;fsgwf-name;vjd-firstname; 12460;ads-city;1970/11/29;yeou-street; 10;MA;MASSACHUSETTS
- 96;reeba-name;kzs-firstname; 13100;zhc-city;1966/07/08;abmv-street; 88;FL;FLORIDA
- 97;shybc-name;gcp-firstname; 10660;ahg-city;1950/12/15;hrqy-street; 174;KS;KANSAS
- 98;phszr-name;sst-firstname; 13080;ydd-city;1954/09/23;quqn-street; 2;RI;RHODE ISLAND
- 99;jteco-name;fxc-firstname; 19760;agr-city;1986/05/06;dzxc-street; 108;MD;MARYLAND
- 100;qvaar-name;icx-firstname; 16120;boc-city;1978/08/04;bfzf-street; 12;NM;NEW MEXICO
+id;name;firstname;zip;city;birthdate;street;housenr;stateCode;state;isActive;registrationTimestamp;metadataJson
+ 1;jwcdf-name;fsj-firstname; 13520;oem-city;1954/02/07;amrb-street; 145;AK;ALASKA;true;2025-12-16 09:47:47.123;"{""ip"":""192.168.1.10"",""session_id"":""a1b2c3d4"",""duration_sec"":120}"
+ 2;flhxu-name;tum-firstname; 17520;buo-city;1966/04/24;wfyz-street; 96;GA;GEORGIA;false;2024-07-02 15:06:40.456;"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+ 3;xthfg-name;gfe-firstname; 12560;vtz-city;1990/01/11;doxx-street; 46;NJ;NEW JERSEY;true;2024-01-01 00:00:00.123;"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+ 4;ulzrz-name;bnl-firstname; 11620;prz-city;1966/08/02;bxqn-street; 104;NY;NEW YORK;false;2023-03-31 12:00:00.789;"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+ 5;oxhyr-name;onx-firstname; 15180;bpn-city;1970/11/14;pksn-street; 133;IN;INDIANA;true;2022-09-01 05:59:59.123;"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+ 6;fiqjz-name;sce-firstname; 16020;fnn-city;1954/09/24;wbhg-street; 35;MD;MARYLAND;false;2022-01-01 00:00:00.000;"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
+ 7;tkiat-name;xti-firstname; 12720;stt-city;1966/08/11;tvnf-street; 21;PA;PENNSYLVANIA;true;2021-01-01 00:00:00.123;"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+ 8;kljcz-name;uqd-firstname; 13340;ntt-city;1987/01/15;jyje-street; 10;PW;PALAU;false;2020-01-01 00:00:00.456;"{""ip"":""10.0.3.4"",""session_id"":""d9e0f1g2"",""duration_sec"":180}"
+ 9;pgunz-name;hcm-firstname; 16680;gxh-city;1970/11/08;shbe-street; 184;NC;NORTH CAROLINA;true;2019-01-01 00:00:00.789;"{""ip"":""172.16.8.9"",""session_id"":""h3i4j5k6"",""duration_sec"":75}"
+ 10;oyjha-name;uhj-firstname; 18880;uyg-city;1966/04/10;bjgw-street; 176;AR;ARKANSAS;false;2023-03-31 12:00:00.789;"{""ip"":""192.168.2.34"",""session_id"":""l7m8n9o0"",""duration_sec"":330}"
+ 11;igxbd-name;uph-firstname; 13480;ndh-city;1962/12/03;jdcd-street; 151;NH;NEW HAMPSHIRE;true;2022-09-01 05:59:59.123;"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+ 12;vnaov-name;wha-firstname; 13120;egm-city;1954/03/28;hpep-street; 20;CA;CALIFORNIA;false;2022-01-01 00:00:00.000;"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+ 13;dauuz-name;hwg-firstname; 13740;khn-city;1958/05/15;etqx-street; 5;OK;OKLAHOMA;true;2021-01-01 00:00:00.123;"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+ 14;gkuuo-name;kkb-firstname; 13560;xdt-city;1962/04/07;sdoj-street; 35;MT;MONTANA;false;2020-01-01 00:00:00.456;"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+ 15;wdhze-name;jjk-firstname; 16900;due-city;1970/07/17;pmmu-street; 174;AS;AMERICAN SAMOA;true;2024-01-01 00:00:00.123;"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
+ 16;ncayz-name;ynb-firstname; 15720;lxj-city;1974/04/27;mdtb-street; 109;MA;MASSACHUSETTS;false;2023-03-31 12:00:00.789;"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+ 17;rdjin-name;hhu-firstname; 14480;lpc-city;1958/11/16;wxik-street; 145;KY;KENTUCKY;true;2022-09-01 05:59:59.123;"{""ip"":""192.168.1.10"",""session_id"":""a1b2c3d4"",""duration_sec"":120}"
+ 18;nxzij-name;bdl-firstname; 10740;avx-city;1958/02/20;nybz-street; 138;WI;WISCONSIN;false;2022-01-01 00:00:00.000;"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+ 19;xgrzc-name;dxw-firstname; 18900;vpq-city;1990/11/16;wzjh-street; 58;ME;MAINE;true;2021-01-01 00:00:00.123;"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+ 20;ehgrn-name;vbe-firstname; 17500;cik-city;1978/05/21;ucnw-street; 135;MD;MARYLAND;false;2020-01-01 00:00:00.456;"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+ 21;gctjx-name;upx-firstname; 11960;yqr-city;1958/03/03;rlko-street; 141;TN;TENNESSEE;true;2019-01-01 00:00:00.789;"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+ 22;ptzmg-name;hva-firstname; 15740;gux-city;1978/05/04;pugy-street; 122;VI;VIRGIN ISLANDS;false;2023-03-31 12:00:00.789;"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
+ 23;eyeti-name;gnw-firstname; 17420;eko-city;1962/10/26;ylph-street; 61;NC;NORTH CAROLINA;true;2022-09-01 05:59:59.123;"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+ 24;wccwo-name;zpj-firstname; 16600;uim-city;1962/09/29;ygih-street; 26;WA;WASHINGTON;false;2022-01-01 00:00:00.000;"{""ip"":""10.0.3.4"",""session_id"":""d9e0f1g2"",""duration_sec"":180}"
+ 25;bwkoe-name;ayl-firstname; 18660;rtw-city;1978/07/16;mzww-street; 179;CA;CALIFORNIA;true;2021-01-01 00:00:00.123;"{""ip"":""172.16.8.9"",""session_id"":""h3i4j5k6"",""duration_sec"":75}"
+ 26;rezku-name;zio-firstname; 19080;nvt-city;1982/07/14;wwkd-street; 91;CA;CALIFORNIA;false;2023-03-31 12:00:00.789;"{""ip"":""192.168.2.34"",""session_id"":""l7m8n9o0"",""duration_sec"":330}"
+ 27;mjlsk-name;ecx-firstname; 10800;yxu-city;1950/12/11;vttb-street; 195;MO;MISSOURI;true;2022-09-01 05:59:59.123;"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+ 28;wdjsi-name;aoq-firstname; 13660;smo-city;1954/02/01;kako-street; 7;NV;NEVADA;false;2022-01-01 00:00:00.000;"{""ip"":""192.168.1.10"",""session_id"":""a1b2c3d4"",""duration_sec"":120}"
+ 29;mwfnd-name;nyb-firstname; 19760;bbu-city;1986/09/23;apdi-street; 91;MS;MISSISSIPPI;true;2021-01-01 00:00:00.123;"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+ 30;vtuoz-name;jhh-firstname; 17620;vad-city;1982/05/05;kzup-street; 79;GA;GEORGIA;false;2020-01-01 00:00:00.456;"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+ 31;rhhxk-name;ndr-firstname; 16760;fub-city;1978/11/12;regd-street; 55;OK;OKLAHOMA;true;2019-01-01 00:00:00.789;"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+ 32;lpstk-name;mqz-firstname; 18940;tnr-city;1982/09/16;cdhf-street; 4;SD;SOUTH DAKOTA;false;2023-03-31 12:00:00.789;"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+ 33;ldhyr-name;yts-firstname; 12000;auk-city;1986/11/14;abph-street; 147;IN;INDIANA;true;2022-09-01 05:59:59.123;"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
+ 34;cjdml-name;iti-firstname; 16900;wkq-city;1970/06/05;npow-street; 96;NH;NEW HAMPSHIRE;false;2022-01-01 00:00:00.000;"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+ 35;cpenz-name;sbi-firstname; 16380;ssl-city;1962/08/19;kilz-street; 44;MS;MISSISSIPPI;true;2021-01-01 00:00:00.123;"{""ip"":""10.0.3.4"",""session_id"":""d9e0f1g2"",""duration_sec"":180}"
+ 36;rxtbg-name;anr-firstname; 14720;bqc-city;1958/08/10;pudg-street; 140;NV;NEVADA;false;2020-01-01 00:00:00.456;"{""ip"":""172.16.8.9"",""session_id"":""h3i4j5k6"",""duration_sec"":75}"
+ 37;udblf-name;raa-firstname; 11500;wli-city;1978/12/13;xomd-street; 41;PW;PALAU;true;2019-01-01 00:00:00.789;"{""ip"":""192.168.2.34"",""session_id"":""l7m8n9o0"",""duration_sec"":330}"
+ 38;vvyce-name;gep-firstname; 13740;gtd-city;1982/05/23;kwbv-street; 123;undefined;undefined;false;2023-03-31 12:00:00.789;"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+ 39;kwfnz-name;ucu-firstname; 10580;sns-city;1978/08/18;nnun-street; 20;OK;OKLAHOMA;true;2022-09-01 05:59:59.123;"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+ 40;zxydx-name;tml-firstname; 14680;jda-city;1974/05/29;wfjn-street; 157;DC;DISTRICT OF COLUMBIA;false;2022-01-01 00:00:00.000;"{""ip"":""192.168.1.10"",""session_id"":""a1b2c3d4"",""duration_sec"":120}"
+ 41;bfscx-name;jnl-firstname; 16920;yyg-city;1970/11/30;cgfh-street; 178;CO;COLORADO;true;2021-01-01 00:00:00.123;"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+ 42;qitur-name;yra-firstname; 15560;ijp-city;1978/01/30;fonc-street; 155;AK;ALASKA;false;2020-01-01 00:00:00.456;"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+ 43;msixi-name;ynb-firstname; 12720;ksl-city;1958/07/17;zpjw-street; 46;VI;VIRGIN ISLANDS;true;2024-01-01 00:00:00.123;"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+ 44;wzkjq-name;rgh-firstname; 19000;hkm-city;1974/08/12;yixf-street; 134;CA;CALIFORNIA;false;2023-03-31 12:00:00.789;"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+ 45;dqfmf-name;yxr-firstname; 13840;vie-city;1962/10/23;stvx-street; 39;TX;TEXAS;true;2022-09-01 05:59:59.123;"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
+ 46;biluz-name;uqe-firstname; 17760;wkq-city;1962/07/27;embn-street; 183;PW;PALAU;false;2022-01-01 00:00:00.000;"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+ 47;wahfx-name;zwd-firstname; 13240;vic-city;1974/03/27;axpw-street; 131;UT;UTAH;true;2021-01-01 00:00:00.123;"{""ip"":""10.0.3.4"",""session_id"":""d9e0f1g2"",""duration_sec"":180}"
+ 48;denwt-name;bta-firstname; 17300;hhj-city;1986/12/20;orwy-street; 11;WV;WEST VIRGINIA;false;2020-01-01 00:00:00.456;"{""ip"":""172.16.8.9"",""session_id"":""h3i4j5k6"",""duration_sec"":75}"
+ 49;akdmy-name;ybz-firstname; 14560;wtx-city;1962/11/08;nwba-street; 123;MP;NORTHERN MARIANA ISLANDS;true;2019-01-01 00:00:00.789;"{""ip"":""192.168.2.34"",""session_id"":""l7m8n9o0"",""duration_sec"":330}"
+ 50;hqafg-name;nht-firstname; 16080;gfu-city;1951/01/12;spsq-street; 45;LA;LOUISIANA;false;2023-03-31 12:00:00.789;"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+ 51;zhmbl-name;lnw-firstname; 17460;hse-city;1986/12/21;scis-street; 97;GA;GEORGIA;true;2022-09-01 05:59:59.123;"{""ip"":""192.168.1.10"",""session_id"":""a1b2c3d4"",""duration_sec"":120}"
+ 52;snwnj-name;jyy-firstname; 16400;hsz-city;1966/02/15;imhl-street; 42;NC;NORTH CAROLINA;false;2022-01-01 00:00:00.000;"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+ 53;fuyla-name;mmp-firstname; 11840;hgu-city;1986/08/16;ixiz-street; 145;NC;NORTH CAROLINA;true;2021-01-01 00:00:00.123;"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+ 54;yvfqz-name;prz-firstname; 11260;wjl-city;1982/05/06;fbzd-street; 97;MO;MISSOURI;false;2023-03-31 12:00:00.789;"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+ 55;usbgq-name;vhd-firstname; 14080;dsb-city;1958/04/01;ggoc-street; 54;KS;KANSAS;true;2022-09-01 05:59:59.123;"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+ 56;yaeni-name;zpy-firstname; 19100;sen-city;1954/12/10;sbsw-street; 158;HI;HAWAII;false;2023-03-31 12:00:00.789;"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
+ 57;fgxvr-name;vzi-firstname; 17520;lcf-city;1958/11/01;nbdv-street; 10;GU;GUAM;true;2022-09-01 05:59:59.123;"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+ 58;tqpbq-name;rwr-firstname; 19140;zpd-city;1978/08/23;npvb-street; 190;DC;DISTRICT OF COLUMBIA;false;2022-01-01 00:00:00.000;"{""ip"":""10.0.3.4"",""session_id"":""d9e0f1g2"",""duration_sec"":180}"
+ 59;ieigg-name;ayq-firstname; 12960;ljc-city;1962/07/05;dnjz-street; 163;FL;FLORIDA;true;2021-01-01 00:00:00.123;"{""ip"":""172.16.8.9"",""session_id"":""h3i4j5k6"",""duration_sec"":75}"
+ 60;rfvzu-name;edm-firstname; 13340;kvz-city;1954/12/08;eijd-street; 4;RI;RHODE ISLAND;false;2020-01-01 00:00:00.456;"{""ip"":""192.168.2.34"",""session_id"":""l7m8n9o0"",""duration_sec"":330}"
+ 61;pduwm-name;gqb-firstname; 14240;cyr-city;1954/07/03;ndux-street; 13;SD;SOUTH DAKOTA;true;2019-01-01 00:00:00.789;"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+ 62;yyixf-name;yzt-firstname; 18020;lwx-city;1974/01/29;iede-street; 120;NV;NEVADA;false;2023-03-31 12:00:00.789;"{""ip"":""192.168.1.10"",""session_id"":""a1b2c3d4"",""duration_sec"":120}"
+ 63;dkszq-name;ytd-firstname; 14700;zwh-city;1979/01/11;nbjz-street; 65;AS;AMERICAN SAMOA;true;2022-09-01 05:59:59.123;"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+ 64;slkzv-name;zbg-firstname; 19880;oee-city;1978/11/01;sphg-street; 119;OK;OKLAHOMA;false;2022-01-01 00:00:00.000;"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+ 65;nvxim-name;phc-firstname; 19220;vgg-city;1991/01/24;juok-street; 106;FM;FEDERATED STATES OF MICRONESIA;true;2021-01-01 00:00:00.123;"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+ 66;piyfg-name;xtn-firstname; 13760;nde-city;1954/07/22;vfrv-street; 11;NY;NEW YORK;false;2020-01-01 00:00:00.456;"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+ 67;jnusz-name;mjw-firstname; 12640;nwb-city;1986/08/23;kcsa-street; 138;VA;VIRGINIA;true;2024-01-01 00:00:00.123;"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
+ 68;jnypj-name;ioq-firstname; 17000;zqy-city;1986/01/09;croe-street; 119;PW;PALAU;false;2023-03-31 12:00:00.789;"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+ 69;uohts-name;btx-firstname; 13480;dal-city;1990/10/22;llyw-street; 150;WA;WASHINGTON;true;2022-09-01 05:59:59.123;"{""ip"":""10.0.3.4"",""session_id"":""d9e0f1g2"",""duration_sec"":180}"
+ 70;aavpj-name;pvw-firstname; 13780;lai-city;1954/09/23;nygu-street; 171;FL;FLORIDA;false;2022-01-01 00:00:00.000;"{""ip"":""172.16.8.9"",""session_id"":""h3i4j5k6"",""duration_sec"":75}"
+ 71;nbjcj-name;rsf-firstname; 12000;kjl-city;1986/06/30;ijsb-street; 123;ID;IDAHO;true;2021-01-01 00:00:00.123;"{""ip"":""192.168.2.34"",""session_id"":""l7m8n9o0"",""duration_sec"":330}"
+ 72;syjxh-name;gkq-firstname; 19960;rmd-city;1978/10/26;qmyp-street; 161;MN;MINNESOTA;false;2020-01-01 00:00:00.456;"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+ 73;vkojz-name;ryo-firstname; 14300;bmz-city;1954/09/11;gcpj-street; 71;ND;NORTH DAKOTA;true;2019-01-01 00:00:00.789;"{""ip"":""192.168.1.10"",""session_id"":""a1b2c3d4"",""duration_sec"":120}"
+ 74;pqzfw-name;kld-firstname; 16400;qvq-city;1962/09/09;dhbv-street; 92;ND;NORTH DAKOTA;false;2023-03-31 12:00:00.789;"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+ 75;owvjk-name;fez-firstname; 19740;ldb-city;1978/06/14;kabf-street; 87;VA;VIRGINIA;true;2022-09-01 05:59:59.123;"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+ 76;qsfih-name;ixe-firstname; 16860;qvr-city;1987/01/07;qean-street; 159;CO;COLORADO;false;2022-01-01 00:00:00.000;"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+ 77;slixq-name;gmb-firstname; 19980;ftt-city;1982/06/22;xinx-street; 111;VT;VERMONT;true;2021-01-01 00:00:00.123;"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+ 78;eegsa-name;xlc-firstname; 12680;byk-city;1954/04/23;beul-street; 56;MD;MARYLAND;false;2023-03-31 12:00:00.789;"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
+ 79;phevp-name;ihs-firstname; 16120;adc-city;1978/04/25;voig-street; 98;NM;NEW MEXICO;true;2022-09-01 05:59:59.123;"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+ 80;njfoe-name;tag-firstname; 16580;tnr-city;1966/12/04;dhky-street; 108;LA;LOUISIANA;false;2023-03-31 12:00:00.789;"{""ip"":""10.0.3.4"",""session_id"":""d9e0f1g2"",""duration_sec"":180}"
+ 81;bdncx-name;hcd-firstname; 11260;xcl-city;1970/07/02;jvlp-street; 49;GA;GEORGIA;true;2022-09-01 05:59:59.123;"{""ip"":""172.16.8.9"",""session_id"":""h3i4j5k6"",""duration_sec"":75}"
+ 82;ikedo-name;tks-firstname; 17460;odl-city;1958/08/25;iaaq-street; 8;GU;GUAM;false;2022-01-01 00:00:00.000;"{""ip"":""192.168.2.34"",""session_id"":""l7m8n9o0"",""duration_sec"":330}"
+ 83;iafxy-name;vur-firstname; 11480;hgt-city;1962/08/03;hmec-street; 164;TX;TEXAS;true;2021-01-01 00:00:00.123;"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+ 84;lafhf-name;ssz-firstname; 19560;wwp-city;1951/01/25;mxmq-street; 96;IN;INDIANA;false;2020-01-01 00:00:00.456;"{""ip"":""192.168.1.10"",""session_id"":""a1b2c3d4"",""duration_sec"":120}"
+ 85;okyny-name;hbu-firstname; 16800;yok-city;1978/03/28;ipjz-street; 135;NV;NEVADA;true;2019-01-01 00:00:00.789;"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+ 86;hznby-name;fwy-firstname; 13680;wbi-city;1970/07/25;mxui-street; 170;CT;CONNECTICUT;false;2023-03-31 12:00:00.789;"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+ 87;ztpoa-name;rzk-firstname; 18500;qum-city;1970/07/26;blqr-street; 152;ME;MAINE;true;2022-09-01 05:59:59.123;"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+ 88;gitxz-name;axt-firstname; 11800;fck-city;1974/01/12;tmjw-street; 189;SD;SOUTH DAKOTA;false;2022-01-01 00:00:00.000;"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+ 89;ziomm-name;mcv-firstname; 12940;iwq-city;1950/10/22;hqgj-street; 140;DC;DISTRICT OF COLUMBIA;true;2021-01-01 00:00:00.123;"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
+ 90;otncg-name;tuy-firstname; 16540;ulk-city;1971/01/24;yuia-street; 166;TX;TEXAS;false;2020-01-01 00:00:00.456;"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+ 91;cnabb-name;hoq-firstname; 16300;tuw-city;1962/06/17;ujvv-street; 61;ME;MAINE;true;2024-01-01 00:00:00.123;"{""ip"":""10.0.3.4"",""session_id"":""d9e0f1g2"",""duration_sec"":180}"
+ 92;ucogf-name;ggc-firstname; 14500;fsj-city;1978/02/08;asfi-street; 53;WV;WEST VIRGINIA;false;2023-03-31 12:00:00.789;"{""ip"":""172.16.8.9"",""session_id"":""h3i4j5k6"",""duration_sec"":75}"
+ 93;lbpmf-name;sdt-firstname; 10780;ewj-city;1978/03/08;hxsp-street; 102;NV;NEVADA;true;2022-09-01 05:59:59.123;"{""ip"":""192.168.2.34"",""session_id"":""l7m8n9o0"",""duration_sec"":330}"
+ 94;tieqq-name;uyu-firstname; 17740;wea-city;1966/10/31;abpl-street; 187;MO;MISSOURI;false;2022-01-01 00:00:00.000;"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+ 95;fsgwf-name;vjd-firstname; 12460;ads-city;1970/11/29;yeou-street; 10;MA;MASSACHUSETTS;true;2021-01-01 00:00:00.123;"{""ip"":""192.168.1.10"",""session_id"":""a1b2c3d4"",""duration_sec"":120}"
+ 96;reeba-name;kzs-firstname; 13100;zhc-city;1966/07/08;abmv-street; 88;FL;FLORIDA;false;2020-01-01 00:00:00.456;"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+ 97;shybc-name;gcp-firstname; 10660;ahg-city;1950/12/15;hrqy-street; 174;KS;KANSAS;true;2019-01-01 00:00:00.789;"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+ 98;phszr-name;sst-firstname; 13080;ydd-city;1954/09/23;quqn-street; 2;RI;RHODE ISLAND;false;2023-03-31 12:00:00.789;"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+ 99;jteco-name;fxc-firstname; 19760;agr-city;1986/05/06;dzxc-street; 108;MD;MARYLAND;true;2022-09-01 05:59:59.123;"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+ 100;qvaar-name;icx-firstname; 16120;boc-city;1978/08/04;bfzf-street; 12;NM;NEW MEXICO;false;2022-01-01 00:00:00.000;"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
diff --git a/integration-tests/transforms/metadata/dataset/golden-parquet-input.json b/integration-tests/transforms/metadata/dataset/golden-parquet-input.json
index c7b7ea52dd..753123b113 100644
--- a/integration-tests/transforms/metadata/dataset/golden-parquet-input.json
+++ b/integration-tests/transforms/metadata/dataset/golden-parquet-input.json
@@ -8,80 +8,104 @@
"field_length": -1,
"field_type": 5,
"field_precision": 0,
- "field_format": "####0;-####0",
- "field_name": "id"
+ "field_name": "id",
+ "field_format": "####0;-####0"
},
{
"field_comment": "",
"field_length": -1,
"field_type": 2,
"field_precision": -1,
- "field_format": "",
- "field_name": "name"
+ "field_name": "name",
+ "field_format": ""
},
{
"field_comment": "",
"field_length": -1,
"field_type": 2,
"field_precision": -1,
- "field_format": "",
- "field_name": "firstname"
+ "field_name": "firstname",
+ "field_format": ""
},
{
"field_comment": "",
"field_length": -1,
"field_type": 5,
"field_precision": 0,
- "field_format": "####0;-####0",
- "field_name": "zip"
+ "field_name": "zip",
+ "field_format": "####0;-####0"
},
{
"field_comment": "",
"field_length": -1,
"field_type": 2,
"field_precision": -1,
- "field_format": "",
- "field_name": "city"
+ "field_name": "city",
+ "field_format": ""
},
{
"field_comment": "",
"field_length": -1,
"field_type": 3,
"field_precision": -1,
- "field_format": "yyyy/MM/dd",
- "field_name": "birthdate"
+ "field_name": "birthdate",
+ "field_format": "yyyy/MM/dd"
},
{
"field_comment": "",
"field_length": -1,
"field_type": 2,
"field_precision": -1,
- "field_format": "",
- "field_name": "street"
+ "field_name": "street",
+ "field_format": ""
},
{
"field_comment": "",
"field_length": -1,
"field_type": 5,
"field_precision": 0,
- "field_format": "####0;-####0",
- "field_name": "housenr"
+ "field_name": "housenr",
+ "field_format": "####0;-####0"
},
{
"field_comment": "",
"field_length": -1,
"field_type": 2,
"field_precision": -1,
- "field_format": "",
- "field_name": "stateCode"
+ "field_name": "stateCode",
+ "field_format": ""
},
{
"field_comment": "",
"field_length": -1,
"field_type": 2,
"field_precision": -1,
- "field_format": "",
- "field_name": "state"
+ "field_name": "state",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 4,
+ "field_precision": -1,
+ "field_name": "isActive",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 9,
+ "field_precision": -1,
+ "field_name": "registrationTimestamp",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 11,
+ "field_precision": -1,
+ "field_name": "metadataJson",
+ "field_format": ""
}
],
"folder_name": ""
diff --git a/integration-tests/transforms/metadata/unit-test/0029-parquet-input UNIT.json b/integration-tests/transforms/metadata/unit-test/0029-parquet-input UNIT.json
index f90e98b5bd..570cefbbae 100644
--- a/integration-tests/transforms/metadata/unit-test/0029-parquet-input UNIT.json
+++ b/integration-tests/transforms/metadata/unit-test/0029-parquet-input UNIT.json
@@ -1,7 +1,10 @@
{
- "variableValues": [],
"database_replacements": [],
"autoOpening": true,
+ "description": "",
+ "persist_filename": "",
+ "test_type": "UNIT_TEST",
+ "variableValues": [],
"basePath": "",
"golden_data_sets": [
{
@@ -45,20 +48,29 @@
{
"transform_field": "zip",
"data_set_field": "zip"
+ },
+ {
+ "transform_field": "isActive",
+ "data_set_field": "isActive"
+ },
+ {
+ "transform_field": "registrationTimestamp",
+ "data_set_field": "registrationTimestamp"
+ },
+ {
+ "transform_field": "metadataJson",
+ "data_set_field": "metadataJson"
}
],
"field_order": [
"id"
],
- "transform_name": "Verify",
- "data_set_name": "golden-parquet-input"
+ "data_set_name": "golden-parquet-input",
+ "transform_name": "Verify"
}
],
"input_data_sets": [],
"name": "0029-parquet-input UNIT",
- "description": "",
"trans_test_tweaks": [],
- "persist_filename": "",
- "pipeline_filename": "./0029-parquet-input.hpl",
- "test_type": "UNIT_TEST"
+ "pipeline_filename": "./0029-parquet-input.hpl"
}
\ No newline at end of file
diff --git a/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/input/ParquetInputMeta.java b/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/input/ParquetInputMeta.java
index 820b0ba356..274fbbdbfd 100644
--- a/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/input/ParquetInputMeta.java
+++ b/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/input/ParquetInputMeta.java
@@ -42,6 +42,13 @@ import org.apache.hop.pipeline.transform.BaseTransformMeta;
import org.apache.hop.pipeline.transform.TransformMeta;
import org.apache.parquet.column.ColumnDescriptor;
import org.apache.parquet.hadoop.ParquetReader;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.LogicalTypeAnnotation.DateLogicalTypeAnnotation;
+import org.apache.parquet.schema.LogicalTypeAnnotation.DecimalLogicalTypeAnnotation;
+import org.apache.parquet.schema.LogicalTypeAnnotation.IntLogicalTypeAnnotation;
+import org.apache.parquet.schema.LogicalTypeAnnotation.JsonLogicalTypeAnnotation;
+import org.apache.parquet.schema.LogicalTypeAnnotation.TimeLogicalTypeAnnotation;
+import org.apache.parquet.schema.LogicalTypeAnnotation.TimestampLogicalTypeAnnotation;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.PrimitiveType;
@@ -148,21 +155,38 @@ public class ParquetInputMeta extends BaseTransformMeta<ParquetInput, ParquetInp
}
PrimitiveType primitiveType = column.getPrimitiveType();
int hopType = IValueMeta.TYPE_STRING;
- switch (primitiveType.getPrimitiveTypeName()) {
- case INT32, INT64:
- hopType = IValueMeta.TYPE_INTEGER;
- break;
- case INT96:
+ LogicalTypeAnnotation logicalType = primitiveType.getLogicalTypeAnnotation();
+ if (logicalType != null) {
+ if ((logicalType instanceof TimestampLogicalTypeAnnotation)
+ || (logicalType instanceof TimeLogicalTypeAnnotation)) {
hopType = IValueMeta.TYPE_TIMESTAMP;
- break;
- case FLOAT, DOUBLE:
- hopType = IValueMeta.TYPE_NUMBER;
- break;
- case BOOLEAN:
- hopType = IValueMeta.TYPE_BOOLEAN;
- break;
- default:
- break;
+ } else if (logicalType instanceof DateLogicalTypeAnnotation) {
+ hopType = IValueMeta.TYPE_DATE;
+ } else if (logicalType instanceof JsonLogicalTypeAnnotation) {
+ hopType = IValueMeta.TYPE_JSON;
+ } else if (logicalType instanceof DecimalLogicalTypeAnnotation) {
+ hopType = IValueMeta.TYPE_BIGNUMBER;
+ } else if (logicalType instanceof IntLogicalTypeAnnotation) {
+ hopType = IValueMeta.TYPE_INTEGER;
+ }
+ } else {
+ switch (primitiveType.getPrimitiveTypeName()) {
+ case INT32, INT64:
+ hopType = IValueMeta.TYPE_INTEGER;
+ break;
+ case INT96:
+ hopType = IValueMeta.TYPE_BINARY;
+ break;
+ case FLOAT, DOUBLE:
+ hopType = IValueMeta.TYPE_NUMBER;
+ break;
+ case BOOLEAN:
+ hopType = IValueMeta.TYPE_BOOLEAN;
+ break;
+ case BINARY:
+ hopType = IValueMeta.TYPE_BINARY;
+ break;
+ }
}
IValueMeta valueMeta = ValueMetaFactory.createValueMeta(sourceField, hopType, -1, -1);
rowMeta.addValueMeta(valueMeta);
diff --git a/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/input/ParquetRowConverter.java b/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/input/ParquetRowConverter.java
index 95bda47002..5ff78fb73c 100644
--- a/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/input/ParquetRowConverter.java
+++ b/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/input/ParquetRowConverter.java
@@ -55,7 +55,10 @@ public class ParquetRowConverter extends GroupConverter {
}
}
- return new ParquetValueConverter(group, rowIndex);
+ return new ParquetValueConverter(
+ group,
+ rowIndex,
+ messageType.getColumns().get(schemaIndex).getPrimitiveType().getLogicalTypeAnnotation());
}
@Override
diff --git a/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/input/ParquetValueConverter.java b/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/input/ParquetValueConverter.java
index 6328809ac0..21c4c12380 100644
--- a/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/input/ParquetValueConverter.java
+++ b/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/input/ParquetValueConverter.java
@@ -17,27 +17,38 @@
package org.apache.hop.parquet.transforms.input;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.sql.Timestamp;
+import java.time.LocalDate;
+import java.time.ZoneId;
import java.util.Date;
+import java.util.TimeZone;
import org.apache.hop.core.RowMetaAndData;
import org.apache.hop.core.row.IValueMeta;
import org.apache.parquet.io.api.Binary;
import org.apache.parquet.io.api.PrimitiveConverter;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.LogicalTypeAnnotation.DateLogicalTypeAnnotation;
+import org.apache.parquet.schema.LogicalTypeAnnotation.DecimalLogicalTypeAnnotation;
public class ParquetValueConverter extends PrimitiveConverter {
private final RowMetaAndData group;
private final IValueMeta valueMeta;
private final int rowIndex;
+ private final LogicalTypeAnnotation logicalTypeAnnotation;
- public ParquetValueConverter(RowMetaAndData group, int rowIndex) {
+ public ParquetValueConverter(
+ RowMetaAndData group, int rowIndex, LogicalTypeAnnotation logicalTypeAnnotation) {
this.group = group;
this.valueMeta = group.getValueMeta(rowIndex);
this.rowIndex = rowIndex;
+ this.logicalTypeAnnotation = logicalTypeAnnotation;
}
@Override
@@ -57,9 +68,27 @@ public class ParquetValueConverter extends PrimitiveConverter {
try {
object = new BigDecimal(value.toStringUsingUTF8());
} catch (NumberFormatException e) {
- object = binaryToDecimal(value, valueMeta.getLength(), valueMeta.getPrecision());
+ if ((this.logicalTypeAnnotation instanceof DecimalLogicalTypeAnnotation)) {
+ object =
+ binaryToDecimal(
+ value,
+ ((DecimalLogicalTypeAnnotation) this.logicalTypeAnnotation).getPrecision(),
+ ((DecimalLogicalTypeAnnotation) this.logicalTypeAnnotation).getScale());
+ } else {
+ object = binaryToDecimal(value, valueMeta.getLength(), valueMeta.getPrecision());
+ }
}
break;
+ case IValueMeta.TYPE_JSON:
+ JsonNode node = null;
+ try {
+ ObjectMapper mapper = new ObjectMapper();
+ node = mapper.readTree(value.toStringUsingUTF8());
+ } catch (Exception e) {
+ throw new RuntimeException("Unable to parse an json value : " + e.getMessage());
+ }
+ object = node;
+ break;
case IValueMeta.TYPE_TIMESTAMP:
if (value.length() == 12) {
// This is a binary form of an int96 (12-byte) Timestamp with nanosecond precision.
@@ -107,10 +136,23 @@ public class ParquetValueConverter extends PrimitiveConverter {
object = Long.toString(value);
break;
case IValueMeta.TYPE_DATE:
- object = new Date(value);
+ if (this.logicalTypeAnnotation instanceof DateLogicalTypeAnnotation) {
+ LocalDate date = LocalDate.ofEpochDay(value);
+ Date utilDate = Date.from(date.atStartOfDay(ZoneId.systemDefault()).toInstant());
+ object = utilDate;
+ } else {
+ object = convertToTimestamp(value, this.logicalTypeAnnotation);
+ }
break;
case IValueMeta.TYPE_BIGNUMBER:
object = new BigDecimal(value);
+ if ((this.logicalTypeAnnotation instanceof DecimalLogicalTypeAnnotation)) {
+ int scale = ((DecimalLogicalTypeAnnotation) this.logicalTypeAnnotation).getScale();
+ object = (new BigDecimal(((BigDecimal) object).doubleValue())).movePointLeft(scale);
+ }
+ break;
+ case IValueMeta.TYPE_TIMESTAMP:
+ object = convertToTimestamp(value, this.logicalTypeAnnotation);
break;
default:
throw new RuntimeException(
@@ -175,6 +217,60 @@ public class ParquetValueConverter extends PrimitiveConverter {
addLong(value);
}
+ /**
+ * Converts a numeric epoch timestamp (in millis, micros, or nanos) into a java.sql.Timestamp
+ * according to the logical type's time unit.
+ *
+ * @param value
+ * @param logicalTypeAnnotation
+ */
+ private Timestamp convertToTimestamp(long value, LogicalTypeAnnotation logicalTypeAnnotation) {
+ LogicalTypeAnnotation.TimeUnit unit = null;
+ boolean isUTC = true;
+ if (logicalTypeAnnotation instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) {
+ unit =
+ ((LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) logicalTypeAnnotation).getUnit();
+ isUTC =
+ ((LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) this.logicalTypeAnnotation)
+ .isAdjustedToUTC();
+ } else if (logicalTypeAnnotation instanceof LogicalTypeAnnotation.TimeLogicalTypeAnnotation) {
+ unit = ((LogicalTypeAnnotation.TimeLogicalTypeAnnotation) logicalTypeAnnotation).getUnit();
+ isUTC = false;
+ }
+ if (unit == null) {
+ throw new RuntimeException(
+ "Unknown timestamp unit for the logical type: " + logicalTypeAnnotation);
+ }
+ long epochMillis;
+ // Convert the timestamp to milliseconds since the Epoch based on the original unit
+ switch (unit) {
+ case MILLIS:
+ epochMillis = value;
+ break;
+ case MICROS:
+ epochMillis = value / 1_000L;
+ break;
+ case NANOS:
+ epochMillis = value / 1_000_000L;
+ break;
+ default:
+ throw new RuntimeException("Unknown timestamp unit: " + unit);
+ }
+ Timestamp ts = new Timestamp(epochMillis);
+ // If the timestamp is local time, adjust it to UTC
+ if (!isUTC) {
+ int offset = TimeZone.getDefault().getOffset(epochMillis);
+ ts.setTime(ts.getTime() - offset);
+ }
+ // Adjust nanosecond precision for microsecond or nanosecond timestamps
+ if (unit == LogicalTypeAnnotation.TimeUnit.MICROS) {
+ ts.setNanos((int) ((value % 1_000_000L) * 1_000L));
+ } else if (unit == LogicalTypeAnnotation.TimeUnit.NANOS) {
+ ts.setNanos((int) (value % 1_000_000_000L));
+ }
+ return ts;
+ }
+
/**
* Source code from:
*
diff --git a/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/output/ParquetOutput.java b/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/output/ParquetOutput.java
index eb0232e378..c42178c7bd 100644
--- a/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/output/ParquetOutput.java
+++ b/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/output/ParquetOutput.java
@@ -17,6 +17,7 @@
package org.apache.hop.parquet.transforms.output;
+import java.nio.charset.StandardCharsets;
import java.text.DecimalFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
@@ -29,7 +30,9 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hop.core.Const;
import org.apache.hop.core.RowMetaAndData;
import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.core.row.value.ValueMetaInteger;
import org.apache.hop.core.vfs.HopVfs;
import org.apache.hop.pipeline.Pipeline;
import org.apache.hop.pipeline.PipelineMeta;
@@ -114,7 +117,40 @@ public class ParquetOutput extends BaseTransform<ParquetOutputMeta, ParquetOutpu
// Write the row, handled by class ParquetWriteSupport
//
try {
- data.writer.write(new RowMetaAndData(getInputRowMeta(), row));
+ IRowMeta parquetRowMeta = getInputRowMeta().clone();
+
+ // convert date/timestamp => long
+ for (int i = 0; i < data.sourceFieldIndexes.size(); i++) {
+ int idx = data.sourceFieldIndexes.get(i);
+ IValueMeta valueMeta = parquetRowMeta.getValueMeta(idx);
+ if (valueMeta.getType() == IValueMeta.TYPE_TIMESTAMP) {
+ // Update of type meta
+ IValueMeta longMeta = new ValueMetaInteger(valueMeta.getName());
+ longMeta.setConversionMask(valueMeta.getConversionMask());
+ longMeta.setLength(valueMeta.getLength(), valueMeta.getPrecision());
+ parquetRowMeta.setValueMeta(idx, longMeta);
+ }
+ }
+
+ // Clone Rows and convert Date & Timetims to Long
+ Object[] parquetRow = row.clone();
+ for (int i = 0; i < data.sourceFieldIndexes.size(); i++) {
+ int idx = data.sourceFieldIndexes.get(i);
+ Object value = parquetRow[idx];
+ if (getInputRowMeta().getValueMeta(idx).getType() == IValueMeta.TYPE_TIMESTAMP) {
+ if (value instanceof java.util.Date) {
+ parquetRow[idx] = ((java.util.Date) value).getTime();
+ } else if (value instanceof byte[]) {
+ String dateStr = new String((byte[]) value, StandardCharsets.UTF_8);
+ SimpleDateFormat sdf =
+ new SimpleDateFormat(parquetRowMeta.getValueMeta(idx).getFormatMask());
+ Date date = sdf.parse(dateStr);
+ parquetRow[idx] = date.getTime();
+ }
+ }
+ }
+
+ data.writer.write(new RowMetaAndData(parquetRowMeta, parquetRow));
incrementLinesOutput();
data.splitRowCount++;
} catch (Exception e) {
@@ -162,9 +198,10 @@ public class ParquetOutput extends BaseTransform<ParquetOutputMeta, ParquetOutpu
// Match these data types with class ParquetWriteSupport
//
+ Schema timestampMilliType;
switch (valueMeta.getType()) {
- case IValueMeta.TYPE_DATE:
- Schema timestampMilliType =
+ case IValueMeta.TYPE_TIMESTAMP, IValueMeta.TYPE_DATE:
+ timestampMilliType =
LogicalTypes.timestampMillis().addToSchema(Schema.create(Schema.Type.LONG));
fieldAssembler =
fieldAssembler
@@ -194,6 +231,12 @@ public class ParquetOutput extends BaseTransform<ParquetOutputMeta, ParquetOutpu
case IValueMeta.TYPE_BINARY:
fieldAssembler = fieldBuilder.bytesType().noDefault();
break;
+ case IValueMeta.TYPE_JSON:
+ fieldAssembler = fieldBuilder.stringType().noDefault();
+ break;
+ case IValueMeta.TYPE_UUID:
+ fieldAssembler = fieldBuilder.stringType().noDefault();
+ break;
default:
throw new HopException(
"Writing Hop data type '"