You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2022/11/08 05:05:58 UTC
[beam] branch nightly-refs/heads/master updated (e0e10b9e543 -> 62d8bdc604d)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from e0e10b9e543 PR Bot - Dont throw error on return code 1
add 66fb431deb7 Print diff and scope to state path
add c08273b9355 Correctly print diff and swallow empty commits for the moment
add 1d5fc14031d Remove quiet flag on debug
add 8223375d2f9 Add sparkreceiver:2 module.
add 6ae37b6f82a Fix sparkreceiver dependencies
add 0476d649821 Merge pull request #23962: [CdapIO] Add sparkreceiver:2 module.
add c0b41fa0454 Use git diff instead of git diff-index to avoid file timestamp changes being picked up
add 98d2ffd69ab Make `documentation/io/connectors/` canonical (#23877)
add 23676a9ec17 [Tour of Beam] Learning content for "Introduction" module (#23085)
add a5ee669a439 [Website] change case-study-card width on mobile
add c4218e572aa Merge pull request #23934: [Website] change case studies page mobile layout
add 9533fc3757d feat: implement bigtable io connector with write capabilities (#23411)
add 8b3fd2e0d65 Bump google.golang.org/api from 0.101.0 to 0.102.0 in /sdks (#23957)
add 380d4730d3c Enforce splitting invariants by ensuring split state is reset in the same synchronized block as window index increments. (#23882)
add 45100d708d9 Add files then check cached diff to get untracked files
add d4142f7eab7 Switch && for || to fix bug in #23889 resolution (#24017)
add 930c4d2d87d Better surfacing of Scala support via Scio.
add 62d8bdc604d Merge pull request #23789 Better surfacing of Scala support via Scio.
No new revisions were added by this update.
Summary of changes:
.../job_PerformanceTests_SparkReceiverIO_IT.groovy | 2 +-
CHANGES.md | 2 +-
.../learning-content/go/content-info.yaml | 22 +
.../from-memory/description.md | 56 +++
.../from-memory/example/from_memory.go | 54 +++
.../from-memory/unit-info.yaml | 22 +
.../creating-collections/group-info.yaml | 25 ++
.../reading-from-csv/description.md | 35 ++
.../reading-from-csv/example/csvExample.go | 89 ++++
.../reading-from-csv/unit-info.yaml | 22 +
.../reading-from-text/description.md | 41 ++
.../reading-from-text/example/textIo.go | 97 +++++
.../reading-from-text/unit-info.yaml | 22 +
.../introduction-concepts/group-info.yaml | 25 ++
.../creating-pipeline/description.md | 36 ++
.../creating-pipeline/example/main.go | 53 +++
.../creating-pipeline/unit-info.yaml | 22 +
.../pipeline-concepts/group-info.yaml | 25 ++
.../overview-pipeline/description.md | 43 ++
.../overview-pipeline/unit-info.yaml | 21 +
.../setting-pipeline/description.md | 71 ++++
.../setting-pipeline/example/main.go | 73 ++++
.../setting-pipeline/unit-info.yaml | 22 +
.../runner-concepts/description.md | 60 +++
.../runner-concepts/unit-info.yaml | 21 +
.../introduction/introduction-guide/description.md | 22 +
.../introduction/introduction-guide/unit-info.yaml | 21 +
.../introduction/introduction-terms/description.md | 38 ++
.../introduction/introduction-terms/unit-info.yaml | 21 +
.../go/introduction/module-info.yaml | 27 ++
.../learning-content/java/content-info.yaml | 22 +
.../from-memory/description.md | 60 +++
.../from-memory/example/from_memory.java | 87 ++++
.../from-memory/unit-info.yaml | 22 +
.../creating-collections/group-info.yaml | 27 ++
.../reading-from-csv/description.md | 35 ++
.../reading-from-csv/example/CSVExample.java | 107 +++++
.../reading-from-csv/unit-info.yaml | 22 +
.../reading-from-text/description.md | 42 ++
.../reading-from-text/example/TextIOExample.java | 101 +++++
.../reading-from-text/unit-info.yaml | 22 +
.../introduction-concepts/group-info.yaml | 27 ++
.../creating-pipeline/description.md | 36 ++
.../creating-pipeline/example/Task.java | 74 ++++
.../creating-pipeline/unit-info.yaml | 22 +
.../pipeline-concepts/group-info.yaml | 25 ++
.../overview-pipeline/description.md | 42 ++
.../overview-pipeline/unit-info.yaml | 21 +
.../setting-pipeline/description.md | 102 +++++
.../setting-pipeline/example/Task.java | 91 ++++
.../setting-pipeline/unit-info.yaml | 22 +
.../runner-concepts/description.md | 416 +++++++++++++++++++
.../runner-concepts/unit-info.yaml | 21 +
.../introduction/introduction-guide/description.md | 22 +
.../introduction/introduction-guide/unit-info.yaml | 22 +
.../introduction/introduction-terms/description.md | 38 ++
.../introduction/introduction-terms/unit-info.yaml | 22 +
.../java/introduction/module-info.yaml | 26 ++
.../learning-content/python/content-info.yaml | 22 +
.../from-memory/description.md | 52 +++
.../from-memory/example/from_memory.py | 53 +++
.../from-memory/unit-info.yaml | 22 +
.../creating-collections/group-info.yaml | 27 ++
.../reading-from-csv/description.md | 34 ++
.../reading-from-csv/example/csvExample.py | 68 +++
.../reading-from-csv/unit-info.yaml | 22 +
.../reading-from-text/description.md | 37 ++
.../reading-from-text/example/textIo.py | 63 +++
.../reading-from-text/unit-info.yaml | 22 +
.../introduction-concepts/group-info.yaml | 27 ++
.../creating-pipeline/description.md | 46 +++
.../creating-pipeline/example/task.py | 43 ++
.../creating-pipeline/unit-info.yaml | 22 +
.../pipeline-concepts/group-info.yaml | 25 ++
.../overview-pipeline/description.md | 40 ++
.../overview-pipeline/unit-info.yaml | 21 +
.../setting-pipeline/description.md | 88 ++++
.../setting-pipeline/example/task.py | 88 ++++
.../setting-pipeline/unit-info.yaml | 22 +
.../runner-concepts/description.md | 127 ++++++
.../runner-concepts/unit-info.yaml | 21 +
.../introduction/introduction-guide/description.md | 22 +
.../introduction/introduction-guide/unit-info.yaml | 22 +
.../introduction/introduction-terms/description.md | 38 ++
.../introduction/introduction-terms/unit-info.yaml | 22 +
.../python/introduction/module-info.yaml | 27 ++
scripts/ci/pr-bot/shared/persistentState.ts | 8 +-
sdks/go.mod | 15 +-
sdks/go.sum | 24 +-
.../pkg/beam/core/runtime/graphx/schema/schema.go | 2 +-
sdks/go/pkg/beam/io/bigtableio/bigtable.go | 285 +++++++++++++
sdks/go/pkg/beam/io/bigtableio/bigtable_test.go | 195 +++++++++
.../apache/beam/fn/harness/FnApiDoFnRunner.java | 225 +++++-----
sdks/java/io/cdap/build.gradle | 2 +-
sdks/java/io/sparkreceiver/{ => 2}/OWNERS | 0
sdks/java/io/sparkreceiver/{ => 2}/README.md | 0
sdks/java/io/sparkreceiver/{ => 2}/build.gradle | 0
.../beam/sdk/io/sparkreceiver/HasOffset.java | 0
.../ReadFromSparkReceiverWithOffsetDoFn.java | 0
.../beam/sdk/io/sparkreceiver/ReceiverBuilder.java | 0
.../beam/sdk/io/sparkreceiver/SparkConsumer.java | 0
.../beam/sdk/io/sparkreceiver/SparkReceiverIO.java | 0
.../sdk/io/sparkreceiver/WrappedSupervisor.java | 0
.../beam/sdk/io/sparkreceiver/package-info.java | 0
.../io/sparkreceiver/ArrayBufferDataReceiver.java | 0
.../io/sparkreceiver/ByteBufferDataReceiver.java | 0
.../io/sparkreceiver/CustomReceiverWithOffset.java | 0
.../sdk/io/sparkreceiver/IteratorDataReceiver.java | 0
.../sparkreceiver/RabbitMqReceiverWithOffset.java | 0
.../ReadFromSparkReceiverWithOffsetDoFnTest.java | 0
.../sdk/io/sparkreceiver/ReceiverBuilderTest.java | 0
.../sdk/io/sparkreceiver/SparkReceiverIOIT.java | 0
.../sdk/io/sparkreceiver/SparkReceiverIOTest.java | 0
settings.gradle.kts | 3 +-
website/www/site/assets/scss/_case_study.scss | 3 +
.../site/content/en/documentation/io/built-in.md | 60 ---
.../site/content/en/documentation/io/connectors.md | 11 +-
.../content/en/documentation/patterns/custom-io.md | 4 +-
.../site/content/en/documentation/sdks/scala.md | 28 ++
website/www/site/data/io_matrix.yaml | 460 ---------------------
website/www/site/data/pipelines.yaml | 3 +
.../partials/section-menu/en/documentation.html | 3 +-
.../layouts/partials/section-menu/en/sdks.html | 11 +
website/www/site/layouts/shortcodes/io-matrix.html | 55 ---
.../static/images/logos/sdks/scala_pipeline.png | Bin 0 -> 15333 bytes
125 files changed, 4474 insertions(+), 712 deletions(-)
create mode 100644 learning/tour-of-beam/learning-content/go/content-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/creating-collections/from-memory/description.md
create mode 100644 learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/creating-collections/from-memory/example/from_memory.go
create mode 100644 learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/creating-collections/from-memory/unit-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/creating-collections/group-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/creating-collections/reading-from-csv/description.md
create mode 100644 learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/creating-collections/reading-from-csv/example/csvExample.go
create mode 100644 learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/creating-collections/reading-from-csv/unit-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/creating-collections/reading-from-text/description.md
create mode 100644 learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/creating-collections/reading-from-text/example/textIo.go
create mode 100644 learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/creating-collections/reading-from-text/unit-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/group-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/pipeline-concepts/creating-pipeline/description.md
create mode 100644 learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/pipeline-concepts/creating-pipeline/example/main.go
create mode 100644 learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/pipeline-concepts/creating-pipeline/unit-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/pipeline-concepts/group-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/pipeline-concepts/overview-pipeline/description.md
create mode 100644 learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/pipeline-concepts/overview-pipeline/unit-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/pipeline-concepts/setting-pipeline/description.md
create mode 100644 learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/pipeline-concepts/setting-pipeline/example/main.go
create mode 100644 learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/pipeline-concepts/setting-pipeline/unit-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/runner-concepts/description.md
create mode 100644 learning/tour-of-beam/learning-content/go/introduction/introduction-concepts/runner-concepts/unit-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/go/introduction/introduction-guide/description.md
create mode 100644 learning/tour-of-beam/learning-content/go/introduction/introduction-guide/unit-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/go/introduction/introduction-terms/description.md
create mode 100644 learning/tour-of-beam/learning-content/go/introduction/introduction-terms/unit-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/go/introduction/module-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/java/content-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/java/introduction/introduction-concepts/creating-collections/from-memory/description.md
create mode 100644 learning/tour-of-beam/learning-content/java/introduction/introduction-concepts/creating-collections/from-memory/example/from_memory.java
create mode 100644 learning/tour-of-beam/learning-content/java/introduction/introduction-concepts/creating-collections/from-memory/unit-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/java/introduction/introduction-concepts/creating-collections/group-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/java/introduction/introduction-concepts/creating-collections/reading-from-csv/description.md
create mode 100644 learning/tour-of-beam/learning-content/java/introduction/introduction-concepts/creating-collections/reading-from-csv/example/CSVExample.java
create mode 100644 learning/tour-of-beam/learning-content/java/introduction/introduction-concepts/creating-collections/reading-from-csv/unit-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/java/introduction/introduction-concepts/creating-collections/reading-from-text/description.md
create mode 100644 learning/tour-of-beam/learning-content/java/introduction/introduction-concepts/creating-collections/reading-from-text/example/TextIOExample.java
create mode 100644 learning/tour-of-beam/learning-content/java/introduction/introduction-concepts/creating-collections/reading-from-text/unit-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/java/introduction/introduction-concepts/group-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/java/introduction/introduction-concepts/pipeline-concepts/creating-pipeline/description.md
create mode 100644 learning/tour-of-beam/learning-content/java/introduction/introduction-concepts/pipeline-concepts/creating-pipeline/example/Task.java
create mode 100644 learning/tour-of-beam/learning-content/java/introduction/introduction-concepts/pipeline-concepts/creating-pipeline/unit-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/java/introduction/introduction-concepts/pipeline-concepts/group-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/java/introduction/introduction-concepts/pipeline-concepts/overview-pipeline/description.md
create mode 100644 learning/tour-of-beam/learning-content/java/introduction/introduction-concepts/pipeline-concepts/overview-pipeline/unit-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/java/introduction/introduction-concepts/pipeline-concepts/setting-pipeline/description.md
create mode 100644 learning/tour-of-beam/learning-content/java/introduction/introduction-concepts/pipeline-concepts/setting-pipeline/example/Task.java
create mode 100644 learning/tour-of-beam/learning-content/java/introduction/introduction-concepts/pipeline-concepts/setting-pipeline/unit-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/java/introduction/introduction-concepts/runner-concepts/description.md
create mode 100644 learning/tour-of-beam/learning-content/java/introduction/introduction-concepts/runner-concepts/unit-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/java/introduction/introduction-guide/description.md
create mode 100644 learning/tour-of-beam/learning-content/java/introduction/introduction-guide/unit-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/java/introduction/introduction-terms/description.md
create mode 100644 learning/tour-of-beam/learning-content/java/introduction/introduction-terms/unit-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/java/introduction/module-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/python/content-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/python/introduction/introduction-concepts/creating-collections/from-memory/description.md
create mode 100644 learning/tour-of-beam/learning-content/python/introduction/introduction-concepts/creating-collections/from-memory/example/from_memory.py
create mode 100644 learning/tour-of-beam/learning-content/python/introduction/introduction-concepts/creating-collections/from-memory/unit-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/python/introduction/introduction-concepts/creating-collections/group-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/python/introduction/introduction-concepts/creating-collections/reading-from-csv/description.md
create mode 100644 learning/tour-of-beam/learning-content/python/introduction/introduction-concepts/creating-collections/reading-from-csv/example/csvExample.py
create mode 100644 learning/tour-of-beam/learning-content/python/introduction/introduction-concepts/creating-collections/reading-from-csv/unit-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/python/introduction/introduction-concepts/creating-collections/reading-from-text/description.md
create mode 100644 learning/tour-of-beam/learning-content/python/introduction/introduction-concepts/creating-collections/reading-from-text/example/textIo.py
create mode 100644 learning/tour-of-beam/learning-content/python/introduction/introduction-concepts/creating-collections/reading-from-text/unit-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/python/introduction/introduction-concepts/group-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/python/introduction/introduction-concepts/pipeline-concepts/creating-pipeline/description.md
create mode 100644 learning/tour-of-beam/learning-content/python/introduction/introduction-concepts/pipeline-concepts/creating-pipeline/example/task.py
create mode 100644 learning/tour-of-beam/learning-content/python/introduction/introduction-concepts/pipeline-concepts/creating-pipeline/unit-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/python/introduction/introduction-concepts/pipeline-concepts/group-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/python/introduction/introduction-concepts/pipeline-concepts/overview-pipeline/description.md
create mode 100644 learning/tour-of-beam/learning-content/python/introduction/introduction-concepts/pipeline-concepts/overview-pipeline/unit-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/python/introduction/introduction-concepts/pipeline-concepts/setting-pipeline/description.md
create mode 100644 learning/tour-of-beam/learning-content/python/introduction/introduction-concepts/pipeline-concepts/setting-pipeline/example/task.py
create mode 100644 learning/tour-of-beam/learning-content/python/introduction/introduction-concepts/pipeline-concepts/setting-pipeline/unit-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/python/introduction/introduction-concepts/runner-concepts/description.md
create mode 100644 learning/tour-of-beam/learning-content/python/introduction/introduction-concepts/runner-concepts/unit-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/python/introduction/introduction-guide/description.md
create mode 100644 learning/tour-of-beam/learning-content/python/introduction/introduction-guide/unit-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/python/introduction/introduction-terms/description.md
create mode 100644 learning/tour-of-beam/learning-content/python/introduction/introduction-terms/unit-info.yaml
create mode 100644 learning/tour-of-beam/learning-content/python/introduction/module-info.yaml
create mode 100644 sdks/go/pkg/beam/io/bigtableio/bigtable.go
create mode 100644 sdks/go/pkg/beam/io/bigtableio/bigtable_test.go
rename sdks/java/io/sparkreceiver/{ => 2}/OWNERS (100%)
rename sdks/java/io/sparkreceiver/{ => 2}/README.md (100%)
rename sdks/java/io/sparkreceiver/{ => 2}/build.gradle (100%)
rename sdks/java/io/sparkreceiver/{ => 2}/src/main/java/org/apache/beam/sdk/io/sparkreceiver/HasOffset.java (100%)
rename sdks/java/io/sparkreceiver/{ => 2}/src/main/java/org/apache/beam/sdk/io/sparkreceiver/ReadFromSparkReceiverWithOffsetDoFn.java (100%)
rename sdks/java/io/sparkreceiver/{ => 2}/src/main/java/org/apache/beam/sdk/io/sparkreceiver/ReceiverBuilder.java (100%)
rename sdks/java/io/sparkreceiver/{ => 2}/src/main/java/org/apache/beam/sdk/io/sparkreceiver/SparkConsumer.java (100%)
rename sdks/java/io/sparkreceiver/{ => 2}/src/main/java/org/apache/beam/sdk/io/sparkreceiver/SparkReceiverIO.java (100%)
rename sdks/java/io/sparkreceiver/{ => 2}/src/main/java/org/apache/beam/sdk/io/sparkreceiver/WrappedSupervisor.java (100%)
rename sdks/java/io/sparkreceiver/{ => 2}/src/main/java/org/apache/beam/sdk/io/sparkreceiver/package-info.java (100%)
rename sdks/java/io/sparkreceiver/{ => 2}/src/test/java/org/apache/beam/sdk/io/sparkreceiver/ArrayBufferDataReceiver.java (100%)
rename sdks/java/io/sparkreceiver/{ => 2}/src/test/java/org/apache/beam/sdk/io/sparkreceiver/ByteBufferDataReceiver.java (100%)
rename sdks/java/io/sparkreceiver/{ => 2}/src/test/java/org/apache/beam/sdk/io/sparkreceiver/CustomReceiverWithOffset.java (100%)
rename sdks/java/io/sparkreceiver/{ => 2}/src/test/java/org/apache/beam/sdk/io/sparkreceiver/IteratorDataReceiver.java (100%)
rename sdks/java/io/sparkreceiver/{ => 2}/src/test/java/org/apache/beam/sdk/io/sparkreceiver/RabbitMqReceiverWithOffset.java (100%)
rename sdks/java/io/sparkreceiver/{ => 2}/src/test/java/org/apache/beam/sdk/io/sparkreceiver/ReadFromSparkReceiverWithOffsetDoFnTest.java (100%)
rename sdks/java/io/sparkreceiver/{ => 2}/src/test/java/org/apache/beam/sdk/io/sparkreceiver/ReceiverBuilderTest.java (100%)
rename sdks/java/io/sparkreceiver/{ => 2}/src/test/java/org/apache/beam/sdk/io/sparkreceiver/SparkReceiverIOIT.java (100%)
rename sdks/java/io/sparkreceiver/{ => 2}/src/test/java/org/apache/beam/sdk/io/sparkreceiver/SparkReceiverIOTest.java (100%)
delete mode 100644 website/www/site/content/en/documentation/io/built-in.md
create mode 100644 website/www/site/content/en/documentation/sdks/scala.md
delete mode 100644 website/www/site/data/io_matrix.yaml
delete mode 100644 website/www/site/layouts/shortcodes/io-matrix.html
create mode 100644 website/www/site/static/images/logos/sdks/scala_pipeline.png