You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hop.apache.org by ha...@apache.org on 2022/09/21 12:18:43 UTC
[hop] branch master updated (dd8b1620ab -> e9b3a074f3)
This is an automated email from the ASF dual-hosted git repository.
hansva pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hop.git
from dd8b1620ab Merge pull request #1696 from ambientelivre/master
new bd16d8d43e HOP-4175 : Update Apache Beam API to 2.41.0
new 9bc7ec0a54 HOP-4024 : Create a new execution information platform (Fixes)
new 2695608a55 HOP-4218 : Hop GUI: The log tab keeps stealing focus when running
new 4193920fd0 HOP-4028 : NPE in Projects plugin
new dc41750a31 HOP-4219 : HopGUI : new metadata items can't be saved
new 76e664a64a HOP-4024 : Create a new execution information platform (Removed unused field)
new 5540cfe207 HOP-4024 : Create a new execution information platform (doc fix)
new 45ca0a99b2 HOP-4220 : Update SnakeYaml library to 1.31
new f4bd48bc2d HOP-4221 : Update hadoop-common library to version 3.2.4
new a65fe3245f HOP-4175 : Update Apache Beam API to 2.41.0 (Spark libs)
new e1cc12b001 HOP-4175 : Update Apache Beam API to 2.41.0 (Spark libs)
new 9a3bb20dd8 HOP-4175 : Update Apache Beam API to 2.41.0 (flink 1.15 update)
new 4b5f4ea582 HOP-3983 : Problem with data types when using MergeJoin
new b47fdca4f0 HOP-3983 : Problem with data types when using MergeJoin HOP-4226 : Beam rows serialize too much data
new 58f5604a7b HOP-4226 : Beam rows serialize too much data
new e4730d032f HOP-4220 : Update SnakeYaml library to 1.32
new 4a3cfbbee5 HOP-3983 : Problem with data types when using MergeJoin
new 6769d4542f HOP-4172 : Forcing to a single thread in Beam doesn't work (fix, IT)
new e9b3a074f3 Merge pull request #1695 from mattcasters/master
The 4510 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
assemblies/plugins/engines/beam/pom.xml | 1431 +++++++++++---------
.../plugins/engines/beam/src/assembly/assembly.xml | 623 ++++-----
assemblies/plugins/tech/azure/pom.xml | 5 +
assemblies/plugins/tech/cassandra/pom.xml | 4 +-
assemblies/plugins/tech/parquet/pom.xml | 2 +-
assemblies/plugins/transforms/fake/pom.xml | 2 +-
assemblies/plugins/transforms/json/pom.xml | 6 +-
assemblies/plugins/transforms/yamlinput/pom.xml | 4 +-
core/src/main/java/org/apache/hop/core/Const.java | 14 +
.../flink-k8s-operator-running-hop-pipeline.adoc | 10 +-
.../pipeline/beam/getting-started-with-beam.adoc | 6 +
.../hop/execution/ExecutionInfoLocation.java | 23 -
.../execution/local/FileExecutionInfoLocation.java | 2 +-
.../java/org/apache/hop/pipeline/Pipeline.java | 4 +
.../hop/pipeline/engine/PipelineEngineFactory.java | 2 +
.../engines/local/LocalPipelineEngine.java | 6 +-
.../hop/pipeline/transform/BaseTransform.java | 21 +-
.../BufferedOutputStreamWithCloseDetection.java | 56 -
.../java/org/apache/hop/workflow/Workflow.java | 1 +
.../org/apache/hop/workflow/action/ActionBase.java | 3 +
.../beam_directrunner/0004-data-sets.hpl | 764 +++++++++++
.../0004-merge-join-validation.hpl | 1399 +++++++++++++++++++
.../beam_directrunner/0004-merge-join.hpl | 374 +++++
.../0005-generate-one-json-file-validation.hpl | 710 ++++++++++
.../0005-generate-one-json-file.hpl | 482 +++++++
.../datasets/0004-full-outer-golden.csv | 1001 ++++++++++++++
.../datasets/0004-inner-join-golden.csv | 869 ++++++++++++
.../datasets/0004-left-outer-golden.csv | 1001 ++++++++++++++
.../datasets/0004-right-outer-golden.csv | 869 ++++++++++++
.../0005-generate-one-json-file-golden.csv | 40 +
.../beam_directrunner/datasets/customers-input.csv | 1201 +++++++++++++---
...e-data-input.csv => state-population-input.csv} | 0
.../beam_directrunner/input/state-population.txt | 0
.../beam_directrunner/main-0004-merge-join.hwf | 158 +++
.../beam_directrunner/main-0005-single-thread.hwf | 139 ++
.../metadata/dataset/0004-full-outer-golden.json | 112 ++
.../metadata/dataset/0004-inner-join-golden.json | 112 ++
.../metadata/dataset/0004-left-outer-golden.json | 112 ++
.../metadata/dataset/0004-right-outer-golden.json | 112 ++
.../0005-generate-one-json-file-golden.json | 88 ++
.../metadata/dataset/state-population-input.json | 24 +
.../metadata/file-definition/state-population.json | 22 +
.../unit-test/0004-merge-join-validation UNIT.json | 259 ++++
...005-generate-one-json-file-validation UNIT.json | 162 +++
integration-tests/gcp/0006-merge-join.hpl | 370 +++++
.../gcp/0007-single-thread-validation.hpl | 710 ++++++++++
integration-tests/gcp/0007-single-thread.hpl | 482 +++++++
.../gcp/datasets/0007-single-thread-golden.csv | 40 +
integration-tests/gcp/main-0006-merge-join.hwf | 117 ++
integration-tests/gcp/main-0007-single-thread.hwf | 139 ++
.../dataset/0007-single-thread-golden.json | 88 ++
.../metadata/file-definition/state-population.json | 22 +
.../pipeline-run-configuration/Dataflow.json | 4 +-
.../0007-single-thread-validation UNIT.json | 162 +++
.../workflow/actions/zipfile/ActionZipFile.java | 5 +-
plugins/engines/beam/pom.xml | 105 +-
.../main/java/org/apache/hop/beam/core/HopRow.java | 56 +-
.../apache/hop/beam/core/coder/HopRowCoder.java | 10 +-
.../hop/beam/core/fn/BQSchemaAndRecordToHopFn.java | 2 +-
.../org/apache/hop/beam/core/fn/HopKeyValueFn.java | 4 +-
.../hop/beam/core/fn/KVLongStringToHopRowFn.java | 2 +-
...{AssemblerFn.java => MergeJoinAssemblerFn.java} | 115 +-
.../hop/beam/core/fn/PubsubMessageToHopRowFn.java | 2 +-
.../org/apache/hop/beam/core/fn/StringToHopFn.java | 2 +-
.../apache/hop/beam/core/fn/StringToHopRowFn.java | 2 +-
.../core/transform/BeamKafkaInputTransform.java | 4 +-
.../core/transform/TransformBatchTransform.java | 25 +-
.../hop/beam/core/transform/TransformFn.java | 600 ++++++++
.../beam/core/transform/TransformTransform.java | 569 +-------
.../org/apache/hop/beam/core/util/HopBeamUtil.java | 11 +-
.../hop/beam/engines/BeamPipelineEngine.java | 84 +-
.../HopPipelineMetaToBeamPipelineConverter.java | 5 +-
.../handler/BeamGenericTransformHandler.java | 18 +-
.../handler/BeamMergeJoinTransformHandler.java | 44 +-
.../transforms/bigtable/BigtableRowToHopRowFn.java | 2 +-
.../hop/beam/transforms/io/BeamInputMeta.java | 4 +-
.../hop/beam/transforms/io/BeamOutputMeta.java | 4 +-
.../kinesis/BeamKinesisConsumeTransform.java | 4 +-
.../kinesis/KinesisRecordToHopRowFn.java | 2 +-
.../io/messages/messages_en_US.properties | 4 +-
.../engines/spark/BeamSparkPipelineEngineTest.java | 1 -
.../apache/hop/projects/gui/ProjectsGuiPlugin.java | 22 +-
.../excelwriter/ExcelWriterTransform.java | 48 +-
plugins/transforms/fake/pom.xml | 2 +-
plugins/transforms/yamlinput/pom.xml | 2 +-
pom.xml | 5 +-
.../main/java/org/apache/hop/ui/core/ConstUi.java | 2 +-
.../ui/execution/ExecutionInfoLocationEditor.java | 2 -
.../perspective/dataorch/HopGuiAbstractGraph.java | 1 -
.../execution/ExecutionPerspective.java | 104 +-
.../execution/PipelineExecutionViewer.java | 108 +-
.../perspective/metadata/MetadataPerspective.java | 2 +
92 files changed, 14211 insertions(+), 2142 deletions(-)
delete mode 100644 engine/src/main/java/org/apache/hop/workarounds/BufferedOutputStreamWithCloseDetection.java
create mode 100644 integration-tests/beam_directrunner/0004-data-sets.hpl
create mode 100644 integration-tests/beam_directrunner/0004-merge-join-validation.hpl
create mode 100644 integration-tests/beam_directrunner/0004-merge-join.hpl
create mode 100644 integration-tests/beam_directrunner/0005-generate-one-json-file-validation.hpl
create mode 100644 integration-tests/beam_directrunner/0005-generate-one-json-file.hpl
create mode 100644 integration-tests/beam_directrunner/datasets/0004-full-outer-golden.csv
create mode 100644 integration-tests/beam_directrunner/datasets/0004-inner-join-golden.csv
create mode 100644 integration-tests/beam_directrunner/datasets/0004-left-outer-golden.csv
create mode 100644 integration-tests/beam_directrunner/datasets/0004-right-outer-golden.csv
create mode 100644 integration-tests/beam_directrunner/datasets/0005-generate-one-json-file-golden.csv
copy integration-tests/beam_directrunner/datasets/{state-data-input.csv => state-population-input.csv} (100%)
copy plugins/engines/beam/src/test/resources/customers/state-data.txt => integration-tests/beam_directrunner/input/state-population.txt (100%)
create mode 100644 integration-tests/beam_directrunner/main-0004-merge-join.hwf
create mode 100644 integration-tests/beam_directrunner/main-0005-single-thread.hwf
create mode 100644 integration-tests/beam_directrunner/metadata/dataset/0004-full-outer-golden.json
create mode 100644 integration-tests/beam_directrunner/metadata/dataset/0004-inner-join-golden.json
create mode 100644 integration-tests/beam_directrunner/metadata/dataset/0004-left-outer-golden.json
create mode 100644 integration-tests/beam_directrunner/metadata/dataset/0004-right-outer-golden.json
create mode 100644 integration-tests/beam_directrunner/metadata/dataset/0005-generate-one-json-file-golden.json
create mode 100644 integration-tests/beam_directrunner/metadata/dataset/state-population-input.json
create mode 100644 integration-tests/beam_directrunner/metadata/file-definition/state-population.json
create mode 100644 integration-tests/beam_directrunner/metadata/unit-test/0004-merge-join-validation UNIT.json
create mode 100644 integration-tests/beam_directrunner/metadata/unit-test/0005-generate-one-json-file-validation UNIT.json
create mode 100644 integration-tests/gcp/0006-merge-join.hpl
create mode 100644 integration-tests/gcp/0007-single-thread-validation.hpl
create mode 100644 integration-tests/gcp/0007-single-thread.hpl
create mode 100644 integration-tests/gcp/datasets/0007-single-thread-golden.csv
create mode 100644 integration-tests/gcp/main-0006-merge-join.hwf
create mode 100644 integration-tests/gcp/main-0007-single-thread.hwf
create mode 100644 integration-tests/gcp/metadata/dataset/0007-single-thread-golden.json
create mode 100644 integration-tests/gcp/metadata/file-definition/state-population.json
create mode 100644 integration-tests/gcp/metadata/unit-test/0007-single-thread-validation UNIT.json
rename plugins/engines/beam/src/main/java/org/apache/hop/beam/core/fn/{AssemblerFn.java => MergeJoinAssemblerFn.java} (54%)
create mode 100644 plugins/engines/beam/src/main/java/org/apache/hop/beam/core/transform/TransformFn.java