You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hop.apache.org by ha...@apache.org on 2022/11/17 21:40:20 UTC
[hop] branch master updated (08cffe606d -> 856ad34215)
This is an automated email from the ASF dual-hosted git repository.
hansva pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hop.git
from 08cffe606d Merge pull request #1805 from hansva/master
new 58d74800e4 HOP-4601 : Parquet File Input needs support for binary encoded BigDecimal values
new 40cb3b478f HOP-4589 : The Google Storage VFS driver fails to delete a file during IT
new a95de5b52b HOP-4589 : The Google Storage VFS driver fails to delete a file during IT (rollback)
new bc4ef7a1d0 HOP-4589 : The Google Storage VFS driver fails to delete a file during IT (XLS Writer fix)
new 603134287d HOP-4324 : Cleanup XML of transform Memory Group By
new 856ad34215 Merge pull request #1806 from mattcasters/cypher-builder
The 4858 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
assemblies/client/pom.xml | 24 +
assemblies/client/src/assembly/assembly.xml | 32 +-
assemblies/core/lib/pom.xml | 4 +-
assemblies/lib-beam/pom.xml | 705 +++++++++++++++++++++
.../{lib => lib-beam}/src/assembly/assembly.xml | 4 +-
assemblies/lib/pom.xml | 32 +-
assemblies/lib/src/assembly/assembly.xml | 2 +-
assemblies/plugins/actions/getpop/pom.xml | 4 +-
assemblies/plugins/actions/mail/pom.xml | 4 +-
.../plugins/databases/access-assemblies/pom.xml | 1 -
.../databases/googlebigquery-assemblies/pom.xml | 6 +-
.../src/assembly/assembly.xml | 36 --
.../plugins/databases/hive-assemblies/pom.xml | 12 +-
.../hive-assemblies/src/assembly/assembly.xml | 3 +-
assemblies/plugins/engines/beam/pom.xml | 343 ++--------
.../plugins/engines/beam/src/assembly/assembly.xml | 81 +--
assemblies/plugins/tech/avro/pom.xml | 5 +
assemblies/plugins/tech/azure/pom.xml | 36 +-
assemblies/plugins/tech/cassandra/pom.xml | 21 +-
.../tech/cassandra/src/assembly/assembly.xml | 8 +-
assemblies/plugins/tech/google/pom.xml | 67 +-
.../plugins/tech/google/src/assembly/assembly.xml | 6 -
assemblies/plugins/tech/parquet/pom.xml | 24 +-
.../plugins/tech/parquet/src/assembly/assembly.xml | 12 +-
assemblies/plugins/transforms/filemetadata/pom.xml | 15 +-
.../filemetadata/src/assembly/assembly.xml | 11 +-
assemblies/plugins/transforms/javascript/pom.xml | 11 +-
.../javascript/src/assembly/assembly.xml | 3 +-
assemblies/plugins/transforms/kafka/pom.xml | 5 +-
assemblies/plugins/transforms/mail/pom.xml | 4 +-
assemblies/plugins/transforms/mailinput/pom.xml | 4 +-
assemblies/plugins/transforms/ssh/pom.xml | 2 +-
assemblies/plugins/transforms/tika/pom.xml | 10 +
.../transforms/tika/src/assembly/assembly.xml | 2 +-
assemblies/pom.xml | 1 +
assemblies/static/src/main/resources/hop-conf.bat | 2 +-
assemblies/static/src/main/resources/hop-conf.sh | 8 +-
.../static/src/main/resources/hop-encrypt.bat | 2 +-
.../static/src/main/resources/hop-encrypt.sh | 8 +-
assemblies/static/src/main/resources/hop-gui.bat | 2 +-
assemblies/static/src/main/resources/hop-gui.sh | 8 +-
.../static/src/main/resources/hop-import.bat | 2 +-
assemblies/static/src/main/resources/hop-import.sh | 8 +-
assemblies/static/src/main/resources/hop-run.bat | 2 +-
assemblies/static/src/main/resources/hop-run.sh | 8 +-
.../static/src/main/resources/hop-search.bat | 2 +-
assemblies/static/src/main/resources/hop-search.sh | 8 +-
.../static/src/main/resources/hop-server.bat | 2 +-
assemblies/static/src/main/resources/hop-server.sh | 8 +-
.../static/src/main/resources/hop-translator.bat | 2 +-
.../static/src/main/resources/hop-translator.sh | 8 +-
assemblies/web/pom.xml | 24 +
assemblies/web/src/assembly/assembly.xml | 25 +-
core/pom.xml | 52 +-
docker/create_hop_web_container.sh | 2 +-
engine-beam/pom.xml | 267 ++++++++
.../org/apache/hop/beam/core/BeamDefaults.java | 1 +
.../java/org/apache/hop/beam/core/BeamHop.java | 25 +-
.../main/java/org/apache/hop/beam/core/HopRow.java | 1 +
.../org/apache/hop/beam/core/fn/HopKeyValueFn.java | 2 -
.../IBeamPipelineEngineRunConfiguration.java | 1 +
.../org/apache/hop/beam/metadata/RunnerType.java | 1 +
.../pipeline/IBeamPipelineTransformHandler.java | 0
engine/pom.xml | 4 +-
.../transform/TransformSerializationTestUtil.java | 10 +-
.../gcp/0001-input-output-validation.hpl | 33 +-
.../gcp/0007-single-thread-validation.hpl | 411 ++++++------
integration-tests/gcp/0007-single-thread.hpl | 372 ++++++-----
.../gcp/main-0008-create-delete-file.hwf | 227 +++++++
.../0007-single-thread-validation UNIT.json | 67 +-
.../mdi/0019-memory-group-by-child.hpl | 133 ++++
.../mdi/0019-memory-group-by-parent.hpl | 425 +++++++++++++
.../mdi/datasets/golden-memory-group-by.csv | 5 +
.../mdi/main-0019-memory-group-by.hwf | 79 +++
.../metadata/dataset/golden-memory-group-by.json | 32 +
.../0019-memory-group-by-parent UNIT.json | 38 ++
plugins/actions/getpop/pom.xml | 4 +-
plugins/actions/mail/pom.xml | 4 +-
plugins/databases/access/pom.xml | 22 +-
.../transforms/accessoutput/AccessOutputMeta.java | 2 +-
plugins/databases/hive/pom.xml | 7 +-
plugins/databases/pom.xml | 2 +-
plugins/engines/beam/pom.xml | 66 +-
.../java/org/apache/hop/beam/core/Failure.java | 77 ---
.../HopPipelineMetaToBeamPipelineConverter.java | 3 -
.../handler/BeamGroupByTransformHandler.java | 103 ---
.../apache/hop/beam/util/BeamPipelineMetaUtil.java | 26 +-
plugins/tech/avro/pom.xml | 1 -
plugins/tech/aws/pom.xml | 3 +-
plugins/tech/azure/pom.xml | 14 +-
plugins/tech/cassandra/pom.xml | 2 -
plugins/tech/google/pom.xml | 9 +-
.../apache/hop/vfs/gs/GoogleStorageFileObject.java | 11 +-
.../transforms/input/ParquetValueConverter.java | 46 +-
plugins/tech/pom.xml | 1 -
.../excelwriter/ExcelWriterTransform.java | 16 +-
plugins/transforms/filemetadata/pom.xml | 13 +-
.../transforms/filemetadata/FileMetadata.java | 14 +-
plugins/transforms/json/pom.xml | 1 -
plugins/transforms/kafka/pom.xml | 2 -
plugins/transforms/mail/pom.xml | 4 +-
plugins/transforms/mailinput/pom.xml | 4 +-
plugins/transforms/memgroupby/pom.xml | 5 +
.../pipeline/transforms/memgroupby/GAggregate.java | 148 +++++
.../pipeline/transforms/memgroupby/GGroup.java} | 64 +-
.../transforms/memgroupby/GroupTypeConverter.java | 49 ++
.../transforms/memgroupby/MemoryGroupBy.java | 281 ++++----
.../transforms/memgroupby/MemoryGroupByDialog.java | 77 +--
.../transforms/memgroupby/MemoryGroupByMeta.java | 642 +++++++++----------
.../memgroupby/beam/AggregationType.java} | 42 +-
.../transforms/memgroupby/beam/GroupByFn.java | 289 +++++++++
.../memgroupby/beam/GroupByTransform.java | 199 ++++++
.../MemoryGroupByAggregationNullsTest.java | 10 +-
.../memgroupby/MemoryGroupByAggregationTest.java | 39 +-
.../memgroupby/MemoryGroupByMetaGetFieldsTest.java | 40 +-
.../memgroupby/MemoryGroupByMetaInjectionTest.java | 42 --
.../memgroupby/MemoryGroupByMetaTest.java | 284 ++-------
.../memgroupby/MemoryGroupByNewAggregateTest.java | 32 +-
.../test/resources/memory-group-by-transform.xml | 94 +++
plugins/transforms/pom.xml | 1 -
plugins/transforms/ssh/pom.xml | 2 +-
pom.xml | 63 +-
ui/pom.xml | 2 +-
123 files changed, 4442 insertions(+), 2277 deletions(-)
create mode 100644 assemblies/lib-beam/pom.xml
copy assemblies/{lib => lib-beam}/src/assembly/assembly.xml (94%)
create mode 100644 engine-beam/pom.xml
rename {plugins/engines/beam => engine-beam}/src/main/java/org/apache/hop/beam/core/BeamDefaults.java (99%)
rename {plugins/engines/beam => engine-beam}/src/main/java/org/apache/hop/beam/core/BeamHop.java (79%)
rename {plugins/engines/beam => engine-beam}/src/main/java/org/apache/hop/beam/core/HopRow.java (99%)
rename {plugins/engines/beam => engine-beam}/src/main/java/org/apache/hop/beam/core/fn/HopKeyValueFn.java (99%)
copy {plugins/engines/beam => engine-beam}/src/main/java/org/apache/hop/beam/engines/IBeamPipelineEngineRunConfiguration.java (99%)
copy {plugins/engines/beam => engine-beam}/src/main/java/org/apache/hop/beam/metadata/RunnerType.java (99%)
rename {plugins/engines/beam => engine-beam}/src/main/java/org/apache/hop/beam/pipeline/IBeamPipelineTransformHandler.java (100%)
create mode 100644 integration-tests/gcp/main-0008-create-delete-file.hwf
create mode 100644 integration-tests/mdi/0019-memory-group-by-child.hpl
create mode 100644 integration-tests/mdi/0019-memory-group-by-parent.hpl
create mode 100644 integration-tests/mdi/datasets/golden-memory-group-by.csv
create mode 100644 integration-tests/mdi/main-0019-memory-group-by.hwf
create mode 100644 integration-tests/mdi/metadata/dataset/golden-memory-group-by.json
create mode 100644 integration-tests/mdi/metadata/unit-test/0019-memory-group-by-parent UNIT.json
delete mode 100644 plugins/engines/beam/src/main/java/org/apache/hop/beam/core/Failure.java
delete mode 100644 plugins/engines/beam/src/main/java/org/apache/hop/beam/pipeline/handler/BeamGroupByTransformHandler.java
create mode 100644 plugins/transforms/memgroupby/src/main/java/org/apache/hop/pipeline/transforms/memgroupby/GAggregate.java
rename plugins/{engines/beam/src/main/java/org/apache/hop/beam/metadata/RunnerType.java => transforms/memgroupby/src/main/java/org/apache/hop/pipeline/transforms/memgroupby/GGroup.java} (51%)
create mode 100644 plugins/transforms/memgroupby/src/main/java/org/apache/hop/pipeline/transforms/memgroupby/GroupTypeConverter.java
rename plugins/{engines/beam/src/main/java/org/apache/hop/beam/engines/IBeamPipelineEngineRunConfiguration.java => transforms/memgroupby/src/main/java/org/apache/hop/pipeline/transforms/memgroupby/beam/AggregationType.java} (58%)
create mode 100644 plugins/transforms/memgroupby/src/main/java/org/apache/hop/pipeline/transforms/memgroupby/beam/GroupByFn.java
create mode 100644 plugins/transforms/memgroupby/src/main/java/org/apache/hop/pipeline/transforms/memgroupby/beam/GroupByTransform.java
delete mode 100644 plugins/transforms/memgroupby/src/test/java/org/apache/hop/pipeline/transforms/memgroupby/MemoryGroupByMetaInjectionTest.java
create mode 100644 plugins/transforms/memgroupby/src/test/resources/memory-group-by-transform.xml