You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ke...@apache.org on 2017/12/05 08:28:24 UTC
[beam] branch jstorm-runner updated (cd9c548 -> ae7357a)
This is an automated email from the ASF dual-hosted git repository.
kenn pushed a change to branch jstorm-runner
in repository https://gitbox.apache.org/repos/asf/beam.git.
from cd9c548 This closes #3846
add 9b51674 Restore non-recursive globs in GcsUtil
add 1835800 This closes #3173: Restore non-recursive globs in GcsUtil
add 66e564e [BEAM-1702] Document how to use BigtableIO with local emulator
add 8ef812d This closes #3141
add a8d2125 [BEAM-659] WindowFn#isCompatible should provide a meaningful reason
add 8c572ef This closes #2985: WindowFn#isCompatible should provide a meaningful reason
add 26a615e Remove incorrect `Message` bound on writeAvros
add 6a7eeeb This closes #3176: Remove incorrect `Message` bound on writeAvros
add 790e7fe Add ParDos
add 2e0cf00 This closes #2597
add 7a2e829 Splits SplittableParDo into a core-construction part and a KWI-related part
add c93b3aa This closes #3156
add 54808be [BEAM-2318] Ensure that tables that don't exist are not created
add f458065 This closes #3179
add 1da8da7 [BEAM-2391] Clone Scan in HBaseReader
add 4924508 This closes #3178
add f969ea4 [BEAM-2252] Use pluginManagement instead of redefining maven-shade-plugin to inherit build order from root pom.xml
add ea5b9da [BEAM-2252] Use pluginManagement instead of redefining maven-shade-plugin to inherit build order from root pom.xml
add 3efb34b update dataflow python container
add a58275a This closes #3182
add bac7bde [BEAM-2164] Fix generate-sources.sh if directory contains space
add ce00d24 This closes #3171: [BEAM-2164] Fix generate-sources.sh if directory contains space
add c4d14f8 Comply with byte limit for Datastore Commit.
add de95c7f Make the large entities test neater.
add dcf4056 End-to-end test for large entity writes.
add 9a6baef This closes #2948: [BEAM-991] Comply with byte limit for Datastore Commit.
add 383329d [BEAM-1345] Remove FileSystems.setDefaultConfigInWorkers since Dataflow no longer depends on this
add 086148f [BEAM-1345] Remove FileSystems.setDefaultConfigInWorkers since Dataflow no longer depends on this
add 6c6d787 [BEAM-2293] Rename Top.{Largest,Smallest=>Natural,Reversed}
add 64cea06 [BEAM-2293] Rename Top.{Largest,Smallest=>Natural,Reversed}
add 2ed68f7 [BEAM-2253] Use resource transformer plugin to use versions defined in root pom.xml within maven archetype poms
add 82ddd4e [BEAM-2253] Use resource transformer plugin to use versions defined in root pom.xml within maven archetype poms
add 5c163c7 [BEAM-2334] Fix OutOfMemoryError
add d7a4e49 [BEAM-2334] Fix OutOfMemoryError
add 4a63970 Add TransformPayloadTranslatorRegistrar
add 6a68e65 This closes #3192: Add TransformPayloadTranslatorRegistrar
add a7ad0c2 Fix GroupAlsoByWindowEvaluatorFactory doc
add 46a9209 This closes #3174
add c7bd2b4 Move snappy-java version to dependencyManagement
add b68f46c This closes #3185: Move snappy-java version to dependencyManagement
add de0fdbc update JavaDoc for BoundedWindow
add 143334c Minor touch-ups to new BoundedWindow javadoc
add e83fcec This closes #2784: update JavaDoc for BoundedWindow
add 10357c2 Remove Unused DoFnInfo methods
add 888a5e6 Extract the Main Input PCollection in ParDos
add 996782b This closes #3190
add 43e2910 Cleanup k8s scripts naming & don't create insecure svc by default
add fff82b0 This closes #3196
add 13d2012 [BEAM-1999] Remove resolved todo from value providers
add 509baac This closes #3197
add 39e8d37 Update Dataflow BATCH_ENVIRONMENT_MAJOR_VERSION to 6
add 983a449 This closes #3198
add 5e9fceb Remove Pipeline reference from TransformHierarchy
add b633abe This closes #3187: Remove Pipeline reference from TransformHierarchy
add 9336230 Include Additional PTransform inputs in Transform Nodes
add 465f438 This closes #3188
add bb3db2a Add WindowIntoTranslator
add 3c10c0b This closes #3191: Add WindowIntoTranslator
add 7f35c98 Add ReadTranslator
add a32bef9 This closes #3201: Add ReadTranslator
add 9da46fd Fix compile error from bad merge
add 3c0f599 Comply with byte limit for Datastore Commit.
add 597b07e This closes #3043
add 6ed1527 Re-rename fat jar so that install doesn't install the bundled jar as the default jar.
add 16ad7e9 This closes #3204
add 94e619d Revert "Include Additional PTransform inputs in Transform Nodes"
add d0601b3 This closes #3206
add 3a62b4f Automatically convert examples to use with syntax.
add 474345f Closes #3180
add 98a7555 Move StepContext to top level
add bed1c53 Remove StepContext.noteOutput
add 8b7a1f6 Rename BaseExecutionContext.StepContext to BaseStepContext
add 59322d5 Move BaseStepContext to the top level
add 248c808 Remove extraneous ExecutionContext parameter to BaseStepContext
add 5ac24e0 Implement StepContext directly in the DirectRunner
add 32c6cb1 Remove writePCollectionViewData from the Beam codebase
add 0be3cf3 Inline and delete BaseExecutionContext
add 97c230a Delete unused ExecutionContext
add acce24c Remove unused StepContext name methods
add dc58551 Delete unused remnants in DirectExecutionContext
add 62115b2 Delete unused BaseStepContext
add 0dc0334 Shorten excessive name in DirectExecutionContext
add d425b27 Revise StepContext javadoc
add b32a1c3 Remove unused pieces of DirectStepContext
add c84d3da This closes #3203: Eliminate ExecutionContext and strip down StepContext to what we actually need
add 329e4bd Fix lint error in datastoreio_test
add 774d897 This closes #3209
add 2e86f4f [BEAM-2338] Fix the limit counter in gcsio reads
add 4ec3366 This closes #3210
add efc8f42 upgrading python sdk dependencies
add 924dd63 This closes #3202
add 7fb2938 Do not ever shrink allowed lateness
add 6dd5585 This closes #3163: [BEAM-163] Warn the user when a PCollection (via Window.into) has smaller allowe...
add 6a792f3 Adds large key tests to GroupByKeyTest
add 6418bcf This closes #2979: Adds large key tests to GroupByKeyTest
add 940819e Rename PCollections to PCollectionTranslation
add 9b6728e Rename PTransforms to PTransformTranslation
add 4460938 Rename ParDos to ParDoTranslation
add bc4f44f Rename WindowIntoTranslator to WindowIntoTranslation
add b35e91d Rename ReadTranslator to ReadTranslation
add 7e37b70 Rename Coders to CoderTranslation
add c8b2119 Rename WindowingStrategies to WindowingStrategyTranslation
add 4fa38e2 Rename Triggers to TriggerTranslation
add 0637df1 This closes #3216: [BEAM-2333] Runner API translation renames
add 9cd6c3b Initial implementation of SpannerIO.Write
add e53dc91 Minor style, compilation, javadoc fixups
add 8871ead Fix spanner dependency management
add 216b972 Delete SpannerCSVLoader
add 65e740e Refine Spanner API tests
add e207022 SpannerIO.Write cleanup and style fixes
add 539e229 This closes #3146
add bb2b706 Automatically generate Python proto and grpc files.
add c2e85da Remove auto-generated proto and grpc files.
add 24fb4cd Closes #3214
add e9cb40d A couple of worker fixes.
add 4084f71 Closes #3170
add f32cacb Adding a snippet for metrics
add c0d19f9 Closes #3200
add 8b0e4bc Update maven-dependency-plugin to version 3.0.1
add 3643676 This closes #3219
add 6b21ea6 More robust gen_protos on jenkins.
add a03a638 Closes #3224
add 73fbcba Clarify that PTransform#expand shouldn't be called
add 1be6f67 This closes #3215
add 5b899a8 Add CombineTranslation
add 2040e2b This closes #3199: Add CombineTranslation
add 73305d6 Fixup CombineTranslation
add 281eaab Cleanups in SimpleDoFnRunner and ParDoEvaluator
add 2d28ece This closes #3194: Cleanups in SimpleDoFnRunner and ParDoEvaluator
add b4ad7cb Fixes pb2.py path in gitignore
add f10ee01 This closes #3227: Fixes pb2.py path in gitignore
add a75f41c Additional explicit file cleanup in gen_protos.
add 7bb10a6 Closes #3229
add 14329b8 [BEAM-2354] Add a ReadStringsFromPubSub/WriteStringsToPubSub PTransform
add 50200f2 [BEAM-2354] Add a ReadStringsFromPubSub/WriteStringsToPubSub PTransform
add 26c9368 Make SdkComponents public for TransformPayloadTranslator
add 1dc134c Centralize primitive URNs in PTransformTranslation class
add 877408e Add URN for Splittable ProcessElement pseudo-primitive
add 0bf4ddb Allow getting URN for class of transform via translator
add afaebb1 Add registration for Read and WindowInto translators
add d91c840 Register ReadTranslator
add 69bb1be Register WindowIntoTranslator
add 121631c Add trivial FlattenTranslator to access URN
add 8e09596 Add transform-analysis helpers to ReadTranslation
add 663ad88 Add RawPTransform, which can just vend its URN and payload
add 0e29cc5 URNs for DirectRunner TransformEvaluator and RootInputProvider
add 9cd1290 This closes #3195: [BEAM-2333] Key DirectRunner translators off URN
add 4e8c388 [BEAM-2276] Add windowing into default filename policy
add deee5b3 [BEAM-2276] This closes #3142
add de38410 fix FlinkAccumulatorCombiningStateWithContext read null accum bug
add 7568f02 This closes #3234
add bd1dfdf Visit a Transform Hierarchy in Topological Order
add 8eb09aa Update Apex Overrides
add 2e2ae9c Roll-forward Include Additional PTransform inputs in Transform Nodes
add 6bb204f This closes #3211
add c8478fe Mark CombineFnWithContext StateSpecs internal
add b0dc523 Allow translation to throw IOException
add 9497e5e Flesh out TimerSpec and StateSpec in Runner API
add 8fc2eb0 Add case dispatch to StateSpec
add a250ce5 Make Java serialized CombineFn URN public
add 39220db Implement TimerSpec and StateSpec translation
add c687887 This closes #3233: [BEAM-115] Runner API Translations for StateSpec and TimerSpec
add 247f9bc Revert "Roll-forward Include Additional PTransform inputs in Transform Nodes"
add 6ad6433 Revert "Visit a Transform Hierarchy in Topological Order"
add 99e60f4 This closes #3238
add a251d01 [BEAM-1542] SpannerIO sink updates
add c7a37b4 This closes #3241
add 6ec64b9 [BEAM-2366] Don't try to pickle gen_protos in tests.
add b70e25c Closes #3228
add ba142aa Add Missing Space in DataflowMetrics
add f47dadc This closes #3243
add 3878556 [BEAM-2372] Only run Apache RAT at root pom.xml
add 884ac51 This closes #3240
add 4d86cec [BEAM-2369] HadoopFileSystem: prevent NPE on match of non existing file
add b0fb5b2 [BEAM-2369] HadoopFileSystem: prevent NPE on match of non existing file
add 86912a2 [BEAM-2365] Use the highest pickle protocol available.
add 2994fce Closes #3226
add f6ed520 Fix Python Dataflow execution errors due to #3223
add 27dd7a9 This closes #3242
add 320756d Upgrade Avro dependency to version 1.8.2
add 2cb4b03 This closes #3249
add 9afe395 [BEAM-2380] Forward additional outputs to DoFnRunner in Flink Batch
add 838035a Fix flushing of pushed-back data in Flink Runner on +Inf watermark
add 711faff This closes #3258
add dd79d5a fix javadoc of View
add aff406b This closes #3257
add 84a831e [BEAM-2379] Avoid reading projectId from environment variable in tests. SpannerOptions.Builder requires projectId to be not-null.
add 2d3e9fe [BEAM-2379] This closes #3259
add 60779e2 [BEAM-1347] Remove the usage of a thread local on a potentially hot path
add 49067b1 [BEAM-1347] Remove the usage of a thread local on a potentially hot path
add d29e353 [BEAM-2248] KafkaIO support to use start read time to set start offset
add 94d677d This closes #3044: [BEAM-2248] KafkaIO support to use start read time to set start offset
add 2ddaaea Clean up pyc files before running tests
add f9431a3 This closes #3262
add 2f9a38e [BEAM-1544] Java cross-JDK version tests on Jenkins
add 2fa24d8 [BEAM-1544] Java cross-JDK version tests on Jenkins\n\nThis closes #3184
add 636eaff Refactor HadoopInputFormatIO to use SerializableConfiguration from hadoop-common
add 185deeb Make SerializableConfiguration more robust by using Hadoop based serialization
add 006fde4 This closes #2812
add 11bf825 [TRIVIAL] InstantCoder: stop boxing Longs unnecessarily
add dc70383 This closes #3270
add 6a78bd3 Reduce Prevalence of PValue in the DirectRunner
add 19c33df This closes #3269
add a0444b8 Flink*DoFnFunction: fix check for single-output dofns
add 5780fc5 Add RawUnion code to FlinkDoFnFunction
add 4884d48 This closes #3263
add a158fc1 [BEAM-2246] Use CLIENT_ACK instead of AUTO_ACK in JmsIO
add 2df9dbd [BEAM-2246] This closes #3073
add 5cf5a1b Rename class Logger object names to the standard convention 'LOG'
add b833006 This closes #3251
add be3c39f Rephrases messages on RetryHttpRequestInitializer
add 8f9f5f1 Rephrases messages on RetryHttpRequestInitializer
add fdf7e33 Add template examples to snippets.py
add ffd99cb This closes #3230
add 87e0fdc fix python lint errors
add 63bcd39 This closes #3278
add 2ee7d69 Add WriteFilesPayload to Runner API well-known transforms
add 0093cf5 Make WriteFiles config publicly readable
add f3ed5a4 Add WriteFiles translation
add 82b81e5 This closes #3274: Add WriteFiles translation
add 9b80524 Fetch only relevant PR
add ca657c4 This closes #3264
add 3dd2fb1 Add GroupByKey translation
add 217f085 This closes #3280: Add GroupByKey translation
add fa23fa2 [BEAM-1348] Add Runner API constructs to ProcessBundleDescriptor.
add 7a075cc [BEAM-1348] Add Runner API constructs to ProcessBundleDescriptor.
add de75786 Unify Java and Python WindowingStrategy representations.
add b490e84 Add known window serialization for Java.
add 4c20bfa Ignore all proto generated files.
add 99907b9 Increase dataflow worker version.
add 2f9428c Closes #3222
add c5918b2 [BEAM-2378] support FULL OUTER JOIN
add f1386c1 This closes #3267
add c8fce2f Bumps dataflow container version to 0530
add e201c37 Removes unused PropertyNames
add eee6726 This closes #3265: Property names cleanup
add a57ff0e Dead-letter support for BigQuery. Allow users to specify a retry policy, and get failed inserts back.
add 53c9bf4 This closes #1609: BEAM-1151 Add failure handling to BigQueryIO.Write
add 1bc84fb Allow the Distinct transform to deduplicate elements across panes
add 9cdae6c This closes #3165: Allow the Distinct transform to deduplicate elements across panes
add 5e5fbed [BEAM-2175] [BEAM-1115] Support for new State and Timer API in Spark batch mode
add 43c4423 This closes #2917
add d17c013 Fix race condition when outputting pushed-back elements in Flink Runner
add ef56ea4 This closes #3285
add fbc6cc5 [BEAM-2401] Update Flink Runner to Flink 1.3.0
add 1866a01 This closes #3284
add b8035ae Flink runner: specify CheckpointingMode through PipelineOptions.
add 802f10a Flink runner: specify checkpointTimeout through PipelineOptions.
add 462335c This closes #3245
add 9c3fbc3 Add translation convenience for non-composites
add f5e30c5 Add TestStreamPayload to Runner API proto
add 47cea78 Expose internal constructors for TestStream events
add 0cedc61 Add TestStream translation
add 0cba43e This closes #3283: [BEAM-3271] Add TestStream translation
add 52df03a [BEAM-2406] Fix NullPointerException when writing an empty table
add 09d75a0 This closes #3293: [BEAM-2406] Fix NullPointerException when writing an empty table
add 7e82bc4 Remove the requirement to visit PCollectionViews in Dataflow
add 6543e56 This closes #3272
add 94d8547 Make batch loads repeatable across different invocations of the same template job.
add ad2c1f1 This closes #2657: [BEAM-2058] Generate BigQuery load job at runtime
add 9511257 [BEAM-1347] Migrate to Runner API constructs within the Java SDK harness
add bf2d300 [BEAM-1347] Migrate to Runner API constructs within the Java SDK harness
add 7905def [BEAM-1347] Add additional logging
add 1cc6dc1 [BEAM-1347] Add additional logging
add ae7bc1d Clarify javadoc on PTransformTranslation
add c3b036a Add CreatePCollectionView translation
add 6d64c6e This closes #3281: [BEAM-3271] Add CreatePCollectionView translation
add e764167 [BEAM-2276] Cleanups on the windowed DefaultFilenamePolicy
add 88f78fa [BEAM-2276] This closes #3232
add b0601fd [BEAM-1498] Use Flink-native side outputs
add e8f2608 Remove the FnOutputT parameter from DoFnOperator
add aebd3a4 This closes #3290
add dbab052 Flink runner: support MapState in FlinkStateInternals.
add a054550 This closes #3289
add ae3dc5f Do not fail when gen_protos cannot be imported
add 7c608c3 This closes #3279
add 840492d Fix RawPTransform translation
add 1b00d95 Improve Splittable ParDo translation
add 513c952 This closes #3282: [BEAM-3271] Improve Splittable ParDo translation
add b5c257d Migrate Python tests to not depend on fix sharding for file output
add e3139a3 This closes #3299
add 5113950 soft-enable the use of streaming flag
add fa3922b This closes #3305
add 1498684 [BEAM-2405] Write to BQ using the streaming API
add fdfd775 This closes #3288
add dcfb31f Whitelist find for tox environments
add 6853d8e This closes #3304
add 3a9c00a Generate a random table name. Assume Spanner database exists.
add be26dd3 This closes #3298
add efe8e1f Fix imports in sdk_worker.
add 6fed177 This closes #3308
add 171a993 Fix the staging directory path in copying from GCS
add 1d2000d This closes #3300
add 39674ca Increase visibility of some Metrics methods
add b6347d0 This closes #3307: Increase visibility of some Metrics methods
add e641997 [BEAM-2405] Override to sink interface in the batch dataflow BQ
add 3cc4ff6 This closes #3306
add 0b0bb3d [BEAM-245] Add CassandraIO
add c189d5c [BEAM-245] This closes #592
add 78b6e3c Sort IO by alphanumeric order
add d302a0f [BEAM-1231] Add missed "kind:bytes" to CloudObjectKinds/CloudObjectTranslators
add b2de3db [BEAM-1231] Add missed "kind:bytes" to CloudObjectKinds/CloudObjectTranslators
add 62b942a [BEAM-2407] Fix Flink CoderTyperSerializer ConfigSnapshot
add c1dc8f5 [BEAM-1779] Port UnboundedSourceWrapperTest to use Flink operator test harness
add 8f4fa43 [BEAM-2408] Fix watermark emission in Flink UnboundedSourceWrapper
add 609016d This closes #2286
add 5f7e73b This closes #2286
add 1cdb80c [BEAM-2421] Swap to use an Impulse primitive + DoFn for Create when executing with the Fn API.
add caecac3 [BEAM-2421] Swap to use an Impulse primitive + DoFn for Create when executing with the Fn API.
add 9c83ffe Shutdown Flink Streaming Pipeline when reaching +Inf watermark
add 32f22b7 This closes #3294
add f2e3088 Remove support for NativeSinks from the Python DirectRunner
add 66460cb This closes #3321
add d94ac58 Make BytesCoder to be a known type
add 0a0a1bc [BEAM-1226] Make BytesCoder to be a known type
add b5852d2 Add coder info to pubsub io
add 86e1fab Closes #3317
add ccf7344 Expand all PValues to component PCollections always
add e066a9d This closes #3303
add 4ebebfd Avoid flakiness in data channel for empty streams.
add d81ed21 Avoid flakiness in data channel for empty streams.
add 3e04902 Refine Python DirectRunner watermark advancement behavior
add fdfc70e This closes #3322
add 156f326 Raise entity limit per RPC to 9MB.
add b53e6f0 This closes #3302
add ca7b9c2 Add Runner API oriented PTransformMatchers for DirectRunner overrides
add d8d9087 DirectRunner override matchers using Runner API
add 4aef938 This closes #3239: [BEAM-2371] Port DirectRunner override matchers to language-agnostic construction APIs
add 36aea2d Use inner module for non-public coders.
add 7c5a70e Closes #3325
add 349898c Implements HCatalogIO
add 911bfbd This closes #3220: Implements HCatalogIO
add 82a6cb6 Slight debuggability improvements in BigtableIO
add bc2f97c This closes #3331: Slight debuggability improvements in BigtableIO
add 7689e43 Move Runner API protos to portability/runners/api
add e980ae9 This closes #3315
add 7caea7a [BEAM-2410] Remove TransportClient from ElasticSearchIO to decouple IO and ES server versions
add 1c6861f [BEAM-2410] This closes #3328
add b7ae7ec Fix compile error occurs in some JDKs
add 97957ea This closes #3320
add fb61c54 Use beam.Map rather than beam.ParDo for PubSub encoding.
add d2c4093 Closes #3332
add 261e7df Visit a Transform Hierarchy in Topological Order
add 33883ed This closes #3329
add 696f8b2 Roll-forward Include Additional PTransform inputs in Transform Nodes
add b4c7716 This closes #3337: Include Additional PTransform Inputs in Transform Nodes
add 8c5b57e Port ViewOverrideFactory to SDK-agnostic APIs
add 02dbaef Port DirectGroupByKey to SDK-agnostic APIs
add ed6bd18 Port DirectRunner WriteFiles override to SDK-agnostic APIs
add eaaf45f Port DirectRunner TestStream override to SDK-agnostic APIs
add 1597f3c This closes #3338: [BEAM-2371] Port some DirectRunner overrides to SDK-agnostic APIs
add 8362bdb [BEAM-2423] Abstract StateInternalsTest for the different state internals
add f9d51aa This closes #3313
add 809f178 Reverse removal of NativeWrite evaluator in Python DirectRunner
add 86e0489 This closes #3346
add f0f98c7 Cleanup Combine Tests with Context
add ce3dd45 This closes #3345
add 1ac18b2 Check for Deferral on Non-additional inputs
add 32d5532 This closes #3347: Check for Deferral on Non-additional inputs
add 6859f80 [BEAM-2412] Update HBaseIO to use HBase client 1.2.6
add fe3d554 [BEAM-2412] This closes #3327
add 4c36508 Use CoderTypeSerializer and remove unuse code in FlinkStateInternals
add 10b166b [BEAM-1483] Support SetState in Flink runner and fix MapState to be consistent with InMemoryStateInternals.
add 4d18606 Add set and map readable test to StateInternalsTest
add 7126fdc This closes #3309
add e71eb66 Actually test the fn_api_runner.
add bd83612 Closes #3339
add b547b5a Make unique test names for value-provider arguments
add e906fe9 This closes #3341
add ee728f1 Fix WindowValueCoder for large timestamps
add c33e9b4 This closes #3350
add 33662b9 Adds ability to dynamically replace PTransforms during runtime.
add 7d0f24a This closes #3333
add 3a48e47 Improves message when transitively serializing PipelineOptions
add 996e35c This closes #3344: Improves message when transitively serializing PipelineOptions
add 9115af4 SpannerIO: Introduced a MutationGroup.
add 646caf2 This closes #3319: [BEAM-1542] SpannerIO: Introduced a MutationGroup
add 7e4e51f Move Runner API protos to portability/api
add eebff90 Closes #3354
add 8a850af Do not produce Unprocessed Inputs if all inputs were Processed
add dd9abc3 This closes #3353
add 0b600d2 Choose GroupAlsoByWindows implementation based on streaming flag
add c962083 This closes #3318
add 329bf1e [BEAM-1585] Add beam plugins as pipeline options
add 42c88f4 Closes #3359
add be09a16 Migrate DirectRunner evaluators to use Beam state API
add 6fc70b3 This closes #3362
add 628366c [BEAM-1348] Mark Runner API like types declared within Fn API as deprecated.
add 346a77f [BEAM-1348] Mark Runner API like types declared within Fn API as deprecated.
add 81a7219 Fix getAdditionalInputs, etc, for DirectRunner stateful ParDo override
add c528fb2 This closes #3352: Fix getAdditionalInputs, etc, for DirectRunner stateful ParDo override
add 581ee15 [BEAM-2423] Port state internals tests to the new base class StateInternalsTest
add d5261d7 This closes #3348
add 027d89c Use the appropriate context in CombineTest Coder
add 901e968 This closes #3369
add c597a02 Fixed handling of use_public_ips. Added test.
add f2a32b2 This closes #3379
add 5aee624 Introduce pending bundles and RootBundleProvider in DirectRunner
add 54f3078 This closes #3370
add c9c1a05 [BEAM-1347] Break apart ProcessBundleHandler to use service locator pattern based upon URNs.
add aa555f5 [BEAM-1347] Break apart ProcessBundleHandler to use service locator pattern based upon URNs.
add 9a6a277 Tests for reading windowed side input from resumed SDF call
add e827642 This closes #3335: Tests for reading windowed side input from resumed SDF call
add 4519681 Populate PBegin input when decoding from Runner API
add 0cabdf6 Closes #3373
add d7715b7 [BEAM-1348] Model the Fn State Api as per https://s.apache.org/beam-fn-state-api-and-bundle-processing
add 2ab482d [BEAM-1348] Model the Fn State Api as per https://s.apache.org/beam-fn-state-api-and-bundle-processing
add bcd4396 Python streaming Create override as a composite of Impulse and a DoFn
add c12d6ba This closes #3349
add cf654a0 A few cleanups in CombineTest
add 1fb4304 This closes #3380: A few cleanups in CombineTest
add c2683e8 Add dry run option to DataflowRunner
add 1476f34 Closes #3385
add 87be64e [BEAM-975] Improve default connection options, javadoc and style in MongoDbIO
add b400f4a [BEAM-975] This closes #3118
add 6d27282 Make HBaseIO tests faster by only using the core daemons needed by HBase
add c6c8d9e This closes #3392
add 595ca1e [BEAM-1348] Fix type error introduced into Python SDK because of PR/3268
add 2304972 [BEAM-1348] Fix type error introduced into Python SDK because of PR/3268
add 1ec59a0 [BEAM-2411] Make the write transform of HBaseIO simpler
add d42f633 [BEAM-2411] Add HBaseCoderProviderRegistrar for better coder inference
add eae0d05 This closes #3391
add 6e43572 Retries http code 0 (usually network error)
add c1a2226 Retries http code 0 (usually network error)
add 5e12e9d Remove notifications from JDK versions test.
add 0eb4004 This closes #3403: Remove notifications from JDK versions test.
add b7ff103 Increases backoff in GcsUtil
add 59598d8 Increases backoff in GcsUtil
add a0523b2 Fix dataflow runner test to call pipeline.run instead of runner.run
add f51fdd9 This closes #3393
add 08ec0d4 Port fn_api_runner to be able to use runner protos.
add e4ef23e Closes #3361
add f69e3b5 FlinkRunner: remove the unused ReflectiveOneToOneOverrideFactory.
add 5279409 Flink runner: refactor the translator into two phases: rewriting and translating.
add 608a9c4 This closes #3275
add 42a2de9 Support ValueProviders in SpannerIO.Write
add 10e4764 This closes #3358: [BEAM-1542] Support ValueProviders in SpannerIO
add 69b01a6 Add spilling code to WriteFiles.
add 698b89e This closes #3161: [BEAM-2302] Add spilling code to WriteFiles.
add 4f6032c Bump Dataflow worker to 20170619
add a06c8bf [BEAM-1377] Splittable DoFn in Dataflow streaming runner
add fd40d4b This closes #1898: [BEAM-1377] Splittable DoFn in Dataflow streaming runner
add ef19024 Add a cloud-pubsub dependency to the list of gcp extra packages
add 223dbb4 This closes #3404
add cbb922c BEAM-1438 Auto shard streaming sinks
add 7b4f5ee This closes #1952: BEAM-1438 Auto shard streaming sinks
add 5a95d62 None should be a valid return element
add a40d11c Closes #3408
add 6b6d20d Return a valid Coder for any subtype of Mutation on HBaseCoderProviderRegistrar
add f96a127 This closes #3407
add 6681472 [BEAM-2481] Update commons-lang3 dependency to version 3.6
add 50acc6c [BEAM-2481] This closes #3401
add 56041b7 Use state / timer API for DirectRunner timer firings
add d1d7812 This closes #3405
add 65a6d66 Turn notifications for broken Windows test off.
add 2d25b68 This closes #3413: Turn notifications for broken Windows test off.
add c11f0ff Fix minor issues on HCatalogIO
add 28c6fd4 This closes #3412
add 3520f94 Allow production of unprocessed bundles, introduce TestStream evaluator in DirectRunner
add f0467b7 This closes #3409
add c6d0d79 Remove GroupedShuffleRangeTracker which is unused in the SDK
add 3746d4c This closes #3414
add cd88630 ReduceFnRunner.onTrigger: skip storeCurrentPaneInfo() if trigger isFinished.
add e015168 This closes #3235
add 17c5012 [BEAM-2495] Add Python test dependency six>=1.9
add ae50fdd This closes #3416
add aa65ea1 Add example for Bigquery streaming sink
add b3099bb This closes #3399
add 20820fa [BEAM-2359] Fix watermark broadcasting to executors in Spark runner
add 22dbb50 Move Spark runner streaming tests to post commit.
add e2ee595 This closes #3343
add 4c48815 [BEAM-1585] Fix the beam plugins installation
add 3785b5b This closes #3421
add 8cab153 Java Dataflow runner harness compatibility.
add 2a55200 Closes #3418
add 7471e27 Clean up test_stream_test and remove stray print statement
add 3dc454a This closes #3423
add 0292a24 [BEAM-2439] Dynamic sizing of Datastore write RPCs
add 9ed0af8 [BEAM-2439] Dynamic sizing of Datastore write RPCs
add c057644 DataflowRunner: Reject SetState and MapState
add 5506be8 This closes #3420: DataflowRunner: Reject SetState and MapState
add 497cfab Add window matcher for pane info
add d4e5db5 Tidy LateDataDroppingDoFnRunner
add 1c1f239 ReduceFnTester can advance clocks without firing timers
add 795760d ReduceFnTester assertion for windows that have data buffered
add 412fd7e Drop late data in ReduceFnTester
add 50c43d9 Do not GC windows based on processing time timer!
add fda589c Add test reproducing BEAM-2505, ignored
add d2b384a Add tests for corner cases of processing time timers
add 7645c44 This closes #3425: Fix processing time timer handling in ReduceFnRunner
add 5d6ad19 Remove fn api bundle descriptor translation.
add a882e8f Remove unused (and untested) initial splittling logic.
add 7a9f762 Closes #3424
add 6f12e7d Bump Dataflow worker to 0622
add 799173f Uses KV in SplittableParDo expansion instead of ElementAndRestriction
add 8860cce This closes #3417: [BEAM-1377] Uses KV in SplittableParDo expansion instead of ElementAndRestriction
add cab4d89 DataflowRunner: Reject merging windowing for stateful ParDo
add 5f8cfa7 This closes #3419: DataflowRunner: Reject merging windowing for stateful ParDo
add 649994b Allow output from FinishBundle in DoFnTester
add b3c3625 This closes #3367: Allow output from FinishBundle in DoFnTester
add 8dcda6e [BEAM-2497] Fix the reading of concat gzip files
add 336b7f1 This closes #3428
add f291713 [BEAM-2489] Use dynamic ES port in HIFIOWithElasticTest
add 6cef5c7 [BEAM-2489] This closes #3411
add fd8f15f [BEAM-2745] Add Jenkins Suite for Python Performance Test
add af69e97 This closes #3426
add 3209548 Fix python fn API data plane remote grpc port access
add 9acce71 This closes #3431
add 903da41 Avoid pickling the entire pipeline per-transform.
add a90e40a Closes #3433
add e45f522 Fix a typo in function args
add 16f87f4 This closes #3435
add 926f949 Remove old deprecated PubSub code
add 1ea1de4 This closes #3440
add bec32fe Reintroduces DoFn.ProcessContinuation (Dataflow worker compatibility part)
add 2052cc7 Bump Dataflow worker to 0623
add 95e6bbe This closes #3383: [BEAM-2447] Reintroduces DoFn.ProcessContinuation (Dataflow worker compatibility part)
add eb379e7 Implement streaming GroupByKey in Python DirectRunner
add 8036001 This closes #3444
add cbfcad8 [BEAM-1237] Create AmqpIO
add 9df865a [BEAM-1237] This closes #1725
add 01b3feb [BEAM-2514] Improve error message on missing required value
add 99ef92b [BEAM-2514] Improve error message on missing required value
add d33d646 Removed uses of proto builder clone method
add e93c064 [BEAM-2392] Remove uses of proto builder clone
add 8dd0077 Enable grpc controller in fn_api_runner
add 3907489 This closes #3445
add 0c34db9 [BEAM-2522] upgrading jackson to 2.8.9 (mitigating #1599)
add 9dc40d6 [BEAM-2522] upgrading jackson to 2.8.9 (mitigating #1599)
add d6855ac Use installed distribution name for sdk name
add 2365e71 This closes #3448
add 7fee4b9 Add WindowFn#assignsToOneWindow
add beb21f4 This closes #3452
add 03741ba [BEAM-1187] Improve logging to contain the number of retries done due to IOException and unsuccessful response codes.
add 0b19fb4 [BEAM-1187] Improve logging to contain the number of retries done due to IOException and unsuccessful response codes
add 80c9263 Bump spanner version
add 454f1c4 Pre read api refactoring. Extract `SpannerConfig` and `AbstractSpannerFn`
add a21a6d7 Read api with naive implementation
add e5929bd This closes #3395: [BEAM-1542] Cloud Spanner Source
add 58fba59 Add utility to expand list of PCollectionViews
add a66bcd6 Fix getAdditionalInputs for SplittableParDo transforms
add 4238276 Use PCollectionViews.toAdditionalInputs in ParDoMultiOverrideFactory
add ed476dd Use PCollectionViews.toAdditionalInputs in ParDo
add 27674f0 Use PCollectionViews.toAdditionalInputs in Combine
add 16f8000 This closes #3351: [BEAM-2371] Fix getAdditionInputs for SplittableParDo transforms
add b1ed975 [BEAM-2488] Elasticsearch IO should read also in replica shards
add 6573e2e [BEAM-2488] This closes #3410
add 2cb2161 Add Experimental annotation to AMQP and refine Kind for the Experimental IOs
add 23a34f5 This closes #3458
add fecd64f [BEAM-2389] moved GcpCoreApiSurfaceTest to corresponding module, adapted exposed packagees
add 1e16aa2 [BEAM-2389] moved GcpCoreApiSurfaceTest to corresponding module, adapted exposed packagees
add ed815be Upgrade beam bigtable client dependency to 0.9.7.1
add b47e41d This closes #3464: [BEAM-2533] Upgrade beam bigtable client dependency to 0.9.7.1
add 90cc2bc Visit composite nodes when checking for picklability.
add 0d4c85d Closes #3466
add 4f9820b Removed OnceTriggerStateMachine
add b1ece01 This closes #3363: [BEAM-765] TriggerStateMachine does not need to be separate from OnceTriggerStateMachine
add 38dd12d WindowingStrategy: add OnTimeBehavior to control whether to emit empty ON_TIME pane.
add 2efb0d5 ReduceFnRunner.onTrigger: add short circuit for empty pane, and move inputWM and pane after the short circuit.
add 997bf40 This closes #3446
add dbeba09 Only use ASCII 'a' through 'z' for temporary Spanner tables
add 4d41e25 This closes #3459: [BEAM-2529] Only use ASCII a-z to generate Spanner database names
add fb7ec28 Add PubSub I/O support to Python DirectRunner
add 2dd1907 This closes #3454
add f46a40c Use SDK harness container for FnAPI jobs when worker_harness_container_image is not specified. Add a separate image tag to use with the SDK harness container.
add bf5aa1b This closes #3468
add c6de423 Define the projectId in the SpannerIO Read Test (utest, not itest)
add ae3a752 This closes #3471
add 5744fa8 [BEAM-2373] Upgrade commons-compress dependency version to 1.14
add 3690073 [BEAM-2373] Upgrade commons-compress dependency version to 1.14
add dc1dca8 Select SDK distribution based on the selected SDK name
add 99b5b05 This closes #3474
add 0acfe70 GCP IO ITs now all use --project option
add ab7f6f6 Don't call .testingPipelineOptions() a second time
add 52cea71 This closes #3465: [BEAM-2537] GCP IO ITs now all use --project option
add 6ade842 Add a Combine Test for Sliding Windows without Context
add 7fc73d7 This closes #3462: Add a Combine Test for Sliding Windows without Context
add 39a2ed0 Ditch apache commons
add 893bf42 This closes #3472: Remove Apache Commons dependency which DataflowRunner appears to not stage
add 7b8cd64 Properly convert milliseconds whether there's less than 3/more than 9 digits. TimeUtil did not properly convert (and returned null) when the number of digits for fractions of seconds was less than 3 digits or more than 9 digits. The solution is to pad with zeros when there is less than 3 digits and to truncate when there is more than 3.
add f75dfe7 Properly convert milliseconds whether there's less than 3/more than 9 digits.
add 56cb6c5 Add support for PipelineOptions parameters
add 0e429b3 This closes #3355: Add support for PipelineOptions parameters
add ce4e517 Make modules that depend on Hadoop and Spark use the same version property
add 11010cf This closes #3291
add 75475ef [BEAM-2530] Fix compilation of modules with Java 9 that depend on jdk.tools
add 0bd47c0 [BEAM-2530] This closes #3460
add 68f1fb6 For GCS operations use an http client with a default timeout value.
add b8ac326 This closes #3486
add 51877a3 [maven-release-plugin] prepare branch release-2.1.0
add 7f0723c [maven-release-plugin] prepare for next development iteration
add 14fa7f7 Update Python SDK version
add 06897b1 Website Mergebot Job
add f0549b4 This closes #3485
add 6ca410a [BEAM-2553] Update Maven exec plugin to 1.6.0 to incorporate messaging improvements
add 39c80f6 [BEAM-2553] Update Maven exec plugin to 1.6.0 to incorporate messaging improvements
add f99ab1a Fix DoFn javadoc: StateSpec does not require a key
add 63f66d6 This closes #3489: Fix DoFn javadoc: StateSpec does not require a key
add 23e385f Update SDK dependencies
add ea9f7fb This closes #3497
add 29c2bca Disallow Combiner Lifting for multi-window WindowFns
add e8fdd37 This closes #3496: Disallow Combiner Lifting for multi-window WindowFns
add a75202f Fix PValue input in _PubSubReadEvaluator
add bf6dda3 This closes #3499
add 2259c30 [BEAM-2534] Handle offset gaps in Kafka messages.
add 85a99e2 This closes #3461
add 526037b Add timeout to initialization of partition in KafkaIO
add b8f8d18 This closes #3492: [BEAM-2551] KafkaIO reader blocks indefinitely in case of network issues
add bd631b8 Ignore processing time timers in expired windows
add 935c077 Process timer firings for a window together
add 628dace This closes #3429: [BEAM-2502, BEAM-2505] More fixes to ReduceFnRunner.onTimers
add da92256 Made DataflowRunner TransformTranslator public
add 17d7e59 This closes #3467: Made DataflowRunner TransformTranslator public
add 17bc3b1 Fix bad merge
add c8d9833 Simplified ByteBuddyOnTimerInvokerFactory
add 440c7d4 This closes #3470: [BEAM-940] ByteBuddyOnTimerInvokerFactory: key the cache with a (Class, id) tuple or OnTimerMethod
add 35061e8 Fix javadoc generation for AmqpIO, CassandraIO and HCatalogIO
add 9e565f2 This closes #3508
add 29c35cd Update Dataflow container version to 20170706
add 4a2f674 This closes #3510: Update Dataflow container version to 20170706
add 3621b95 Move DirectRunner knob for suppressing runner-determined sharding out of core SDK
add 90ba753 This closes #3516: Move DirectRunner knob for suppressing runner-determined sharding out of core SDK
add 175ff2f Reject stateful ParDo if coder not KvCoder with deterministic key coder
add f62586a This closes #3502: [BEAM-2486] Reject stateful ParDo if coder not deterministic KvCoder
add 2295b90 [BEAM-1347] Rename DoFnRunnerFactory to FnApiDoFnRunner.
add 7da08c9 [BEAM-1347] Add DoFnRunner specific to Fn Api.
add 56e4251 [BEAM-1347] Create a DoFnRunner specific for the Fn API
add 1fa3bfe Set the type of batch jobs to FNAPI_BATCH when beam_fn_api experiment is specified.
add dba5140 This closes #3522
add 25d3baa [BEAM-2570] Fix breakage after cloud-bigquery updated
add f467d6b This closes #3523
add cb5061e BEAM-2575 ApexRunner doesn't emit watermarks for additional outputs
add 9f904dc This closes #3529: BEAM-2575 ApexRunner doesn't emit watermarks for additional outputs
add 521488f [BEAM-1348] Remove deprecated concepts in Fn API (now replaced with Runner API concepts).
add 9d48bd5 [BEAM-1348] Remove deprecated concepts in Fn API (now replaced with Runner API concepts).
add 77ba7a3 Adds DynamicDestinations support to FileBasedSink
add c14a318 This closes #3356: [BEAM-92] Allow value-dependent files in FileBasedSink
add 20ce075 Rehydrate PCollections
add 165dfa6 Add more utilities to ParDoTranslation
add de39f32 Include PCollection in rehydrated PCollectionView
add fa61ed1 Enable SplittableParDo on rehydrated ParDo transform
add 1ac4b7e Port DirectRunner ParDo overrides to SDK-agnostic APIs
add be9a387 Fix misleading comment in TransformHierarchy
add 1518d73 Fix null checks in TransformHierarchy
add b6b1c8b This closes #3509: [BEAM-2371] Port DirectRunner overrides to SDK-agnostic APIs
add c6f9fde Reflect #assignsToOneWindow in WindowingStrategy
add eeb0432 This closes #3453: Reflect #assignsToOneWindow in WindowingStrategy
add 311547a Use URNs, not Java classes, in immutability enforcements
add 0e89df3 This closes #3536: Use URNs, not Java classes, in immutability enforcements
add 3099e81 Split bundle processor into separate class.
add af08f53 Closes #3503
add 21fd302 Reformatting Kinesis IO to comply with official code style
add 138641f This closes #3389
add 435cbcf Add client-side throttling.
add a22f1a0 [BEAM-2556] Add client-side throttling.
add aaffe15 Remove dead (and wrong) viewFromProto overload
add 011e279 This closes #3542: Remove dead (and wrong) viewFromProto overload
add cd216f7 Adds TextIO.readAll(), implemented rather naively
add 8468210 This closes #3443: [BEAM-2511] Implements TextIO.ReadAll
add 2b86a61 Cleanup and fix ptransform_fn decorator.
add 91c7d3d Closes #3544
add 1bff4a7 [BEAM-2447] Reintroduces DoFn.ProcessContinuation
add 66b4a1b This closes #3360: [BEAM-2447] Reintroduces DoFn.ProcessContinuation
add 016baf3 Implement retries in the read connector.
add 889776f This closes #3558
add 64997ef Unbundle Context and WindowedContext.
add 5f972e8 This closes #3539: Unbundle FileNamePolicy Context and WindowedContext
add a6201ed JdbcIOIT now uses writeThenRead style
add 5fd2c6e This closes #2507: JdbcIOIT now uses writeThenRead style
add eb951c2 [BEAM-2595] Allow table schema objects in BQ DoFn
add e8c5574 This closes #3556
add 5d9fe88 Test GBK immediately followed by stateful ParDo
add 621f20f This closes #3562: Test GBK immediately followed by stateful ParDo
add 78fcbd7 [BEAM-2491] Move package org.apache.beam.runners.core.metrics to org.apache.beam.runners.core.construction.metrics
add f77d035 This closes #3422
add c5c0961 BQIO.Read option for use with repeated template invocations.
add 47273b9 This closes #2123: Provide implementation of BQIO.Read that does not use Source API
add ca40663 Fixes sharding and cleanup for dynamic file writes
add 1f1df27 This closes #3546: [BEAM-2601] Fix broken per-destination finalization.
add ce440a3 [BEAM-2608] Closes the reader in TextIO.readAll()
add 36c55eb This closes #3567: [BEAM-2608] Closes the reader in TextIO.readAll()
add 53ce582 [BEAM-2578] Fix DebuggingWordCountTest on Windows platform
add ae0de1b [BEAM-2578] This closes #3535
add 095e791 datastoreio: retry on socket errors
add 7e4719c This closes #3559
add 7257507 Improving labeling of side inputs for Dataflow
add 532256e Closes #3520
add e7059e5 [BEAM-1502] GroupByKey should not return bare lists in DirectRunner.
add 02905c2 Closes #3578
add f6daad4 [BEAM-933] Fix and enable findbugs in Java examples
add 1996869 This closes #3534: [BEAM-933] Fix and enable findbugs in Java examples
add 4c6fa39 Adjust pull request template for Jenkins and mergebot world
add 04d364d This closes #3575: Adjust pull request template for Jenkins and mergebot world
add c5ebbff Make Dataflow Counter Name Parsing more Robust
add 0f06eb2 This closes #3463
add d495d15 Splits large TextIOTest into TextIOReadTest and TextIOWriteTest
add 7c36318 This closes #3442: Splits large TextIOTest into TextIOReadTest and TextIOWriteTest
add a48eefa [BEAM-2084] Adding querying facility for distribution metrics in Java
add d2201f9 This closes #2896
add b827f65 Change PR template from 1234 to XXX
add 5a0b74c This closes #3587: Change PR template from 1234 to XXX
add f1b4700 Fix split package in SDK harness
add 2c2d8a3 This closes #3577: Fix split package in SDK harness
add d14cef0 [BEAM-1963] Update Quickstart link in README
add 2d5b6d7 This closes #3589
add 111603a Accept Region in Dataflow Monitoring Page URL
add dd9e866 This closes #3455
add 911edba [BEAM-2544] Fix flaky AvroIOTest by eliminating race condition in "write then read" tests.
add be5b934 This closes #3475: [BEAM-2544] Fix flaky AvroIOTest
add 1e94704 Add GroupByKey tests for Multiple & Merging windows
add 0d927ef This closes #3576
add bdf5bd6 [BEAM-2630] TestPipeline: construct job/app names based on Description in junit TestRule.
add d510175 This closes #3582
add e86c004 [BEAM-2532] Memoizes TableSchema in BigQuerySourceBase
add 7fde976 This closes #3584: [BEAM-2532] add a Serializable TableSchema Supplier in BigQuerySourceBase
add d290114 [BEAM-2306] Add checkstyle check to fail the build when @Deprecated is used without @deprecated javadoc (or vice versa).
add a6f460f This closes #3531: [BEAM-2306] Fail build when @Deprecated is used without @deprecated javadoc
add 51427a6 [BEAM-2642] Update Google Auth to 0.7.1
add 4d1db22 [BEAM-2642] Update Google Auth to 0.7.1
add b424aa0 Increase the gRPC message size to max value
add eb0850e Increase the gRPC message size to max value
add d128c3b [BEAM-2636] Make sure we only override the correct class
add 2e51bde This closes #3595
add 0a5157e Dynamic sizing of Datastore write RPCs.
add c8e3744 This closes #3585
add 95e9c28 Introduces SpannerIO.readAll()
add afeba37 This closes #3591: [BEAM-1542] Introduced SpannerIO.readAll
add 4192ac6 Add maven support for invoking perfkit benchmarker to run IO ITs
add 2f9cbec io-it-suite-local independent of io-it-suite, k8s properties -> root pom
add 7e63d2c This closes #3588
add c52a908 Minor changes to AvroSource in preparation for refactoring
add d4026da Gets rid of opening Avro files in createForSubrangeOfFile codepath
add 1d9160f This closes #3590: [BEAM-2628] Makes AvroSource not open files while splitting
add c2110c9 Register a PTransformTranslator for Combine
add b6f126d This closes #3597
add 01492e6 Use RehydratedComponents for memoized rehydration
add f870bf5 This closes #3601: Use RehydratedComponents for memoized rehydration
add e38dc5f Use dehydration-insensitive APIs in WindowEvaluatorFactory
add 8bd6475 This closes #3609: Use dehydration-insensitive APIs in WindowEvaluatorFactory
add fa3a5ab Use dehydration-insensitive APIs in ParDoEvaluatorFactory
add 4b35584 This closes #3610: Use dehydration-insensitive APIs in ParDoEvaluatorFactory
add cfadecb Uniquify application nodes in TextIOReadTest and only run pipeline once
add 81c2e90 This closes #3608: Uniquify application nodes in TextIOReadTest and only run pipeline once
add 4f7d0b4 Unbreaks AvroSource compatibility with Dataflow worker
add f398748 This closes #3616: Unbreaks AvroSource compatibility with Dataflow worker
add b1adbfb ReshuffleTest: replace Iterable equal tests with matchers.
add 49d4cf4 This closes #3593
add a7c1849 [BEAM-2662] Fix Java quickstart for Spark to include Spark version
add 5f1d136 [BEAM-2662] This closes #3620
add 8449931 [BEAM-2571] Change DoFnOperator to use Long.MAX_VALUE as max watermark
add b03c4f0 [BEAM-2571] Clarify pushedback variable name in DoFnOperator
add 1825087 [BEAM-2571] Respect watermark contract in Flink DoFnOperator
add f54072a This closes #3600
add dfa983c [BEAM-2314] Add ValidatesRunner test for merging custom windows
add 0064fb3 This closes #3286
add efe2dc1 Add stub DisplayDataTranslation
add 12c277f Fix tests that passed invalid input to DynamicDestinations
add 4348159 Add Pipeline rehydration from proto
add 8ca4591 Dehydrate then rehydrate Pipeline before DirectRunner.run()
add 01408c8 This closes #3334: [BEAM-2333] Go to proto and back before running a pipeline in Java DirectRunner
add 22b8296 ReduceFnRunner: test when watermark leapfrogs EOW and GC
add 73da9cc This closes #3598: ReduceFnRunner: test when watermark leapfrogs EOW and GC
add 2e8ed5a Let IsBounded take True value.
add 71196ec Closes #3580
add e80c83b Adds ValueProvider support to AvroIO.Read
add eaf0b36 Extracts common logic from TextIO.ReadAll into a utility transform
add ee1bcba Introduces AvroIO.readAll() and readAllGenericRecords()
add d919394 This closes #3615: [BEAM-2656] Introduces AvroIO.readAll()
add 483abc0 We shouldn't write to re-created tables for 2 mins
add a9fdc3b This closes #3630
add 56a1bce Sets desired bundle size on AvroIO.readAll
add bc72c94 This closes #3642: Sets desired bundle size on AvroIO.readAll
add 809a7f8 [BEAM-2658] register DefaultCoder first and SerializableCoder last
add 69a51c9 [BEAM-2658] register SerializableCoder last
add 22e49e6 Use runner API for read operation.
add 24b39e4 Disable abc metaclass due to issues with pickling.
add 2d307f2 Create common superclass for all Sources.
add b67a30b Closes #3635
add 82cc1e1 Translate combining operations through the Runner API.
add 673937b Closes #3640
add 51867ad Migrate checkCombineFn in TestUtils to CombineFnTester
add e206b1a Closes #3628
add 86e3355 [BEAM-1984] Fix scope for dependencies needed only for test purposes
add 892a2cf [BEAM-1984] This closes #3652
add 0249da5 Fixes BigQueryIO.Write javadoc for to(ValueInSingleWindow)
add 5727ad2 This closes #3645: Fixes BigQueryIO.Write javadoc for to(ValueInSingleWindow)
add f515c22 [BEAM-2640] Introduces Create.ofProvider(ValueProvider)
add 476efa6 [BEAM-2641] Introduces TextIO.read().withHintMatchesManyFiles()
add e333577 Adds AvroIO.read().withHintMatchesManyFiles()
add 68bb7c0 This closes #3639: [BEAM-2640, BEAM-2641] Introduces TextIO/AvroIO.read().withHintMatchesManyFiles()
add f61e797 [BEAM-2541] Check Elasticsearch backend version when the pipeline is run not when it is constructed
add 6ca5305 [BEAM-2541] This closes #3493
add d9fc36a Add missing displayData to ElasticsearchIO.Read
add da3b8bf This closes #3655
add 0884be5 Fix View.asMultimap javadoc.
add 7ab8954 This closes #3656: Fix View.asMultimap javadoc.
add 62c922b Introduces TypeDescriptors.extractFromTypeParameters
add ebd0041 [BEAM-2677] AvroIO.parseGenericRecords - schemaless AvroIO.read
add c16947e Removes TransformingSource that is now unused
add a94d680 This closes #3632: [BEAM-2677] AvroIO.parseGenericRecords - schemaless AvroIO.read
add 6ea2eda Perform a Multi-step combine in the DirectRunner
add 1f2634d This closes #3579
add 9f2622f [BEAM-92] Supports DynamicDestinations in AvroIO.
add 540fa9b This closes #3541: [BEAM-92] Supports DynamicDestinations in AvroIO.
add 04f2700 [BEAM-2623] Introduces Watch transform
add 95a2ead This closes #3565: [BEAM-2623] Creates Watch transform for watching new elements in a family of growing sets
add 8b60fcb Programmatically Create Beam Jenkins View.
add 870551c This closes #3374
add 42638e2 [BEAM-2530] Upgrade maven plugins to the latest versions
add 2816b15 [BEAM-2530] This closes #3665
add 14563c6 [BEAM-2690] HCatalogIO needs provided scope dependencies on Hadoop, Hive
add 3c81766 This closes #3660
add 64cf18f Remove References to CloudObject from the Java Harness
add 53c2c8f This closes #3647
add 7d64b76 Translate ParDo operations through the Runner API.
add 3c0c337 Streaming fixes.
add b179eca Translate GroupByKey[Only] through the Runner API.
add 2ee7422 More informative references in the proto representation.
add ba5e314 Closes #3650
add e239644 Add the Beam Job API service definition
add 0a358c7 This closes #3676
add ff4b36c [BEAM-2670] Fixes SparkRuntimeContext.getPipelineOptions()
add 7db051a Introduces SerializablePipelineOptions in core-construction
add 339976c This closes #3654: [BEAM-2670] Deduplicates serializable wrappers over PipelineOptions from Spark, Flink and Apex, and removes buggy SparkRuntimeContext
add a919f8e [BEAM-2708] Configure BZIP2 to read all "streams"
add 9582840 [BEAM-2708] This closes #3669
add fc0f9c7 [BEAM-1542] Specifies a User Agent in Spanner Client
add 8ac796c This closes #3574: [BEAM-1542] Specifies a User Agent in Spanner Client
add 9cbc0d3 Fix typo in dataflow_runner.
add 520a51b Closes #3671
add 01f1d94 Remove unneeded surefire configuration for hadoop-file-system
add de0148c This closes #3658
add b5017bf Add local filesystem as default and strip away prefix from local files in pipeline options.
add d2133d3 [BEAM-2298] Making LocalFileSystem default and stripping file spec prefixes.
add 1e582e6 Improving naming of steps that consume side inputs
add 55bb423 Closes #3659
add 5d46243 Adds support for reading concatenated bzip2 files.
add d4f9e92 This closes #3678
add ac7f973 [BEAM-1347] Add utility to be able to model inbound reading as a single input stream
add 38f1890 [BEAM-1347] Add utility to be able to model inbound reading as a single input stream
add e017a0e Makes all Source classes override getOutputCoder instead of getDefaultOutputCoder
add bb1bf3c Requires specifying a Coder on PCollection.createPrimitiveOutputInternal
add 95e2a00 Removes all overrides of PTransform.getDefaultOutputCoder and deprecates it
add 48690bc Remembers the output coders of SplittableParDo
add 9e6530a This closes #3649: [BEAM-2686] Towards deprecating PCollection.setCoder()
add 5e71d53 Refactor FnApiRunner to operate directly on the runner API protos.
add 72bd73a Closes #3682
add fc00ce9 [BEAM-1584] Add clean up in bigquery integration test
add f731de0 This closes #3673
add b10c1c3 [BEAM-2729] Allow GBK of union-typed PCollections.
add 8d337ff Closes #3690
add 5e43b23 Adds coders for boolean, ResourceId and Metadata
add db9aede Introduces EmptyMatchTreatment parameter to FileSystems.match()
add fe002c2 [BEAM-2512] Introduces TextIO.watchForNewFiles() and Match
add df36bd9 This closes #3607: [BEAM-2512] Introduces TextIO.watchForNewFiles() and the Match transform
add 97b12d5 [BEAM-2734] Unbreaks some Dataflow ValidatesRunner tests
add aadbe36 This closes #3693: [BEAM-2734] Unbreaks some Dataflow ValidatesRunner tests
add 545946a Suppress Attempted Metrics tests in Dataflow Batch
add 9781983 This closes #3695
add 7dbe906 Add GRPC to Compilation of the Runner API
add 086ec77 This closes #3687
add 6c81a3a Delete list job creation -- creates in wrong place.
add c9abd15 This closes #3697
add 9478f41 [BEAM-79] add Gearpump runner
add 02b2248 This closes #323
add 2a0ba61 Merge branch master into gearpump-runner
add 1672b54 move integration tests to profile
add 276a2e1 add package-info.java
add 40be715 Update Gearpump runner version to 0.3.0-incubating
add bc1b354 Rename DoFn to OldDoFn in Gearpump runner
add 091a15a This closess #750
add fb74c93 gearpump: switch to stable version
add bf0a2ed Closes #895
add 0dfb8ff Made byteArrayCoder final static
add b9f8263 CompressedSource: CompressedReader is never splittable
add 011bea9 Do not add DataDisks to windmill service jobs.
add 1d86335 Remove timeout in DirectRunnerTest
add 36a9aa2 Improve Write Error Message
add d564155 Remove Streaming Write Overrides in DataflowRunner
add 89921c4 Remove Counter and associated code
add 7fc2c68 [BEAM-495] Create General Verifier for File Checksum
add b47549e Add output checksum to WordCountITOptions
add 37ce2a3 More unit test and code style fix
add 046e36e Using IOChannelUtils to resolve file path
add 58cd781 Added unit tests and error handling in removeTemporaryTables
add d99a652 [flink] add missing maven config to example pom
add 39f763e Remove DataflowPipelineJob from examples
add 424c4c4 [BEAM-432] Corrected BigQueryIO javadoc
add b80d967 Add TransformEvaluatorFactory#cleanup
add 77c90d0 Replace CloningThreadLocal with DoFnLifecycleManager
add d056f46 Add DoFn @Setup and @Teardown
add 6603307 Move ParDo Lifecycle tests to their own file
add d6cf4f2 Exclude ParDoTest from Dataflow @RunnableOnService
add 0f1f114 Exclude guava-testlib from shading relocation
add 09cd1b7 ByteKeyRangeTracker: synchronize toString
add bd53cdc Fix repackaging exclude pattern for guava-testlib
add cc189b4 Rewrites DoFnReflector to go via DoFnSignature
add da638b6 Replace ParDo with simpler transforms where possible
add 235bf3b Set Gcs upload buffer size to 1M in streaming mode in DataflowRunner
add 530b9c0 addressed feedback
add aa541e7 fix unused imports
add bbd0e6b DatastoreIO Sink as ParDo
add 245c3ce Change name of result returned by BigQueryIO.Read
add 16bcf78 Fix NPE in BigQueryIO.TransformingReader
add 178898f Add inEarlyPanesInGlobalWindow as a PAssert Extractor
add 6c82321 Add TestStream to the Testing package
add f37dba8 Implement TestStream in the DirectRunner
add dab9efc Incorporate private IP option for Dataflow runner
add a4053ac Datastore Sink support for writing Mutations
add c996c1e Mark JAXBContext as Volatile in JAXBCoder
add 4ad78b2 Modify example dependencies to only add runners as optional dependencies. Also support excluding runners by disabling the include-runners profile.
add 5c1b9f1 DatastoreIO v1beta3 to v1
add c3c11b1 Remove unused constant in ExecutorServiceParallelExecutor
add 64a2d51 Remove extra timer firings in WatermarkManager
add a60806a FileBasedSink: improve parallelism in GCS copy/remove
add 780ffcb [BEAM-574] Remove log when new records have not been read yet (KafkaIO)
add 730e7b0 Write: Remove invalid import
add 67efb17 JUnit tests: add @RunWith annotation
add 438d8bd Remove ParDoTest Suppression in Google Cloud Dataflow
add 67e095d Fix Emission in startBundle/finishBundle in Flink Wrappers
add 686a286 [BEAM-253] Unify Flink-Streaming Operator Wrappers
add 9eef8a6 Fix Checkstyle Errors in FlinkStreamingTransformTranslators
add 9179e93 [BEAM-102] Add Side Inputs in Flink Streaming Runner
add de744c5 Allow DoFn Reuse in ParDoTest.TestDoFnWithContext
add b4a38c3 Don't Suppress Throwable in PAssert in Streaming Mode
add 7a2cccd Fix Flink Runner Pom for Batch RunnableOnService tests
add f2a992e Enable Flink Streaming Runner RunnableOnService tests
add a07b29f Fix combine tests with Accumulation Mode
add 97e093c Use AllPanes as the PaneExtractor in IterableAssert
add b7ba1d6 Make ParDoLifecycleTest Serializable to Fix Test with TupleTag
add 7012a22 Fix Exception Unwrapping in TestFlinkRunner
add 93f7955 Update checkstyle.xml to put all imports in one group
add 00441f8 Optimize imports
add 8d32196 BigQueryIO.Write: raise size limit to 11 TiB
add 186fe28 Cleanup some javadoc that referring Dataflow
add 433842b Move the samples data to gs://apache-beam-samples/
add 32928c3 [BEAM-545] Promote JobName to PipelineOptions
add f05fbe7 Update DoFn javadocs to remove references to OldDoFn and Dataflow
add 4ec73d8 Make WriteTest more resilient to Randomness
add b3be7b7 checkstyle: prohibit API client repackaged Guava
add 1f8b534 Modified BigtableIO to use DoFn setup/tearDown methods instead of startBundle/finishBundle
add 5b425ac [BEAM-294] Rename dataflow references to beam
add ef312e9 Added support for reporting aggregator values to Spark sinks
add 0fbd9c8 travis.yml: disable updating snapshots
add 79491eb Query latest timestamp
add 4023167 [BEAM-589] Fixing IO.Read transformation
add bce9aef kinesis: a connector for Amazon Kinesis
add aee5fbf Organize imports in Kinesis
add 973081e Fix javadoc in Kinesis
add 1c1115e [BEAM-592] Fix SparkRunner Dependency Problem in WordCount
add 8454d5c DataflowRunner: get PBegin from PInput
add 07dd978 [BEAM-313] Provide a context for SparkRunner
add 435054b Update Dataflow Container Version
add cf9ce2f [BEAM-572] Remove Spark Reference in WordCount
add a58afd3 Returned KafkaIO getWatermark log line in debug mode
add 74d0195 take advantage of setup/teardown for KafkaWriter
add 00b4e95 Add LeaderBoardTest
add 8007bdf [BEAM-569] Define maxNumRecords default value to Long.MAX_VALUE in JmsIO
add 6ae4b6a Address comments of Flink Side-Input PR
add 1524494 Fix condition in FlinkStreamingPipelineTranslator
add 798566c Correct some accidental renames
add 4251761 Test that multiple instances of TestStream are supported
add 28ad44d Remove empty unused method in TestStreamEvaluatorFactory
add 6ee7b62 Add Latest CombineFn and PTransforms
add 0312f15 DatastoreIO SplitQueryFn integration test
add f44fa2c Cloud Datastore naming clean-up
add 9943fd7d Fixed Combine display data
add 60d8cd9 Delegate populateDipslayData to wrapped combineFn's
add 4bf3a3b Put classes in runners-core package into runners.core namespace
add c92e45d Remove the DataflowRunner instructions from examples
add 3f48566 FluentBackoff: a replacement for a variety of custom backoff implementations
add 9ae5cc7 [BEAM-456] Add MongoDbIO
add 5eb44aa [BEAM-242] Enable checkstyle and fix checkstyle errors in Flink runner
add 958f3fe BigQuery: limit max job polling time to 1 minute
add c8052b6 Be more accepting in UnboundedReadDeduplicatorTest
add 8f68085 Remove timeout in JAXBCoderTest
add 50c1c88 [BEAM-242] Enable and fix checkstyle in Flink runner examples
add b235595 Add header/footer support to TextIO.Write
add 1b420db Revised according to comments following a code review.
add 092a187 Reverted header and footer to be of type String.
add 5084580 Added javadoc to TextIO#withHeader and TextIO#withFooter.
add e5db1c7 Added even more javadoc to TextIO#withHeader and TextIO#withFooter.
add 34c731f Added even more javadoc to TextIO#withHeader and TextIO#withFooter (2).
add 6cd48c4 !fixup Minor javadoc clean-up
add 59ae94c fix import order
add ed7c4aa Closes #943
add 272fe9f [BEAM-79] fix integration-test failure
add 8f4334c Closes #956
add 9dc9be9 Merge branch 'master' into gearpump-runner
add 8f013cb post-merge fix
add 94bd47c remove "pipeline" in runner name
add 3f06382 upgrade gearpump-runner to 0.4.0-incubating-SNAPSHOT
add 3933b55 Closes #1193
add 45570b9 [BEAM-79] Port Gearpump runner from OldDoFn to new DoFn
add 323ec11 This closes #1234
add 0c36228 Merge branch 'master' into gearpump-runner
add 2a96a17 [BEAM-79] update GearpumpPipelineResult
add a14927f This closes #1306
add 68363d0 Merge branch 'master' of https://github.com/apache/incubator-beam into gearpump-runner
add 2812405 Merge branch 'master' of https://github.com/apache/incubator-beam into gearpump-runner
add 86414c0 Merge remote-tracking branch 'upstream/master' into gearpump-runner-sync
add 2afc0cd [BEAM-79] fix gearpump runner build failure
add 88de0cb This closes #1507
add 46d3563 Upgrade Gearpump version
add 85d54ab Add Window.Bound translator
add c37de00 Skip window assignment when windows don't change
add cb8c5e5 Remove print to stdout
add 8e0e819 Fix NoOpAggregatorFactory
add b6e7bb6 This closes #1623: [BEAM-1086] Upgrade to latest Gearpump snapshot
add 81d94cf Merge branch 'master' of https://github.com/apache/incubator-beam into gearpump-runner
add c2fb7c0 [BEAM-79] Update to latest Gearpump API
add 647034c [BEAM-79] Upgrade to beam-0.5.0-incubating-SNAPSHOT
add 4c445dd This closes #1663: Merge master (b3de17b) into gearpump-runner
add 2155476 [BEAM-1180] Implement GearpumpPipelineResult
add cfdc971 update ROS configurations
add ea633d2 activate ROS on Gearpump by default
add e63d42d fix group by window
add 3bf8263 update to latest gearpump dsl function interface
add f6aaf0d support OutputTimeFn
add 364a3f0 return encoded key for GroupByKey translation
add b2d326f fix ParDo.BoundMulti translation
add 7613ec4 reduce timeout to wait for result
add 85dcfbd Remove cache for Gearpump on travis
add d814857 note thread is interrupted on InterruptedException
add 1ed16f1 This closes #1661: Implement GearpumpPipelineResult
add 4fd216b [BEAM-79] Fix PostCommit test confs for Gearpump runner
add 4001aeb This closes #1828: Fix PostCommit test confs for Gearpump runner
add 7af6472 [BEAM-79] Support merging windows in GearpumpRunner
add 2d0aed9 This closes #1935: Support merging windows in GearpumpRunner
add 4eb50d1 [BEAM-79] Add SideInput support for GearpumpRunner
add 3dc8fc8 enable ParDoTest
add 15a8ad6 This closes #2150: Add SideInput support for GearpumpRunner
add 3f91798 Merge branch 'master' of https://github.com/apache/incubator-beam into gearpump-runner
add 3eab6a6 [BEAM-79] Fix gearpump-runner merge conflicts and test failure
add 555842a This closes #2241: merge master to gearpump-master and fixup
add eb0d333 [BEAM-972] Add unit tests to Gearpump runner
add f4f2333 This closes #2302: Add unit tests to Gearpump runner
add f3138dd [BEAM-972] Add more unit test to Gearpump runner
add ebbb613 This closes #2521: Add more unit test to Gearpump runner
add 46c41fc Merge branch 'master' of https://github.com/apache/incubator-beam into gearpump-runner
add 44d21ac Update gearpump-runner against master changes.
add 4078c22 This closes #2610: Merge master into gearpump-runner branch
add 9a59ea3 Merge remote-tracking branch 'upstream/master' into gearpump-runner
add 12b9719 Update gearpump-runner against master changes
add 58546ac This closes #2888: Merge master into gearpump-runner branch
add bc8da29 Merge branch 'master' of https://github.com/apache/beam into sync-master
add 6c06967 Update gearpump-runner against master changes
add 99221e7 This closes #3172: Sync gearpump-runner with master
add 9e6c906 Merge branch 'master' of https://github.com/apache/beam into sync-master
add c9aac96 Update against master changes
add 3c7e3e6 Activate Gearpump local-validates-runner-tests in precommit
add 98854d4 Respect WindowFn#getOutputTime in gearpump-runner
add 7653e7e Fix side input handling in DoFnFunction
add 559e3c3 This closes #3292: Merge master into gearpump-runner
add fed98c8 Merge branch 'master' of https://github.com/apache/beam into sync-master
add f61822d upgrade to gearpump 0.8.4-SNAPSHOT
add a7b5d98 Fix PCollectionView translation
add 11caa97 Fix kryo exception
add b21fa04 Remove unused codes
add 99f4f8b This closes #3388: Sync gearpump-runner branch with master
add f158257 Merge branch 'master' of https://github.com/apache/beam into sync-master
add 627ae0b This closes #3479: [BEAM-79] Merge master into gearpump-runner branch
add c2d3fbc Merge branch 'master' of https://github.com/apache/beam into sync-master
add 2206827 Upgrade to gearpump 0.8.4
add 725f547 Fix ParDoTest#testPipelineOptionsParameter
add 1ce60b4 This closes #3515: Sync gearpump-runner with master and upgrade to gearpump 0.8.4
add e655f53 Revert accidental changes to sdks/java/pom.xml
add daa7566 Upgrade BEAM version to 2.2.0-SNAPSHOT in gearpump-runner
add 49d4ed5 Add beam-runners-gearpump dependency to javadoc
add d9bb4cb Deactivate integration-tests for gearpump-runner by default
add a282165 Require java 8 for precommit tests and add gearpump-runner
add 18daed9 Override maven-enforcer-plugin for precommit tests
add bf15483 Remove gearpump-runner in precommit tests
add c35febf This closes #3637: [BEAM-79] Prepare for merging gearpump-runner
add 89236e3 This closes #3636: Merge GearpumpRunner to master branch
add 2fa4fde Fix compile break
add d00ff9e [BEAM-2657] Create Solr IO
add f5714f2 This closes #3618: [BEAM-2657] Create Solr IO
add 2b9b050 Use bytes instead of Any in RunnerApi.FunctionSpec
add 9ed2cf4 This closes #3193
add 76db0aa [BEAM-2431] Add experimental python rpc direct runner
add fb85d84 This closes #3667
add cc699ec Add support for Python's native type hint types in Beam's type hint annotations.
add db4b093 This closes #3672
add f48bb4b [BEAM-1274] Add SSL mutual authentication support
add 02f11d3 [BEAM-1274]Â Add SSL/TLS in the comments, add the self signed policy support for the SSL context
add 04f5bc6 [BEAM-1274] This closes #3626
add 6aadf24 Preparing support for Structured Names in Dataflow counters
add 2e17311 This closes #3684
add 70cd34c Updates BEAM_CONTAINER_VERSION to 2.2.0.
add 7a34042 This closes #3701
add a17978a [BEAM-1584] Add file clean up util for integration tests
add 0a5deea This closes #3688
add cc7689c [BEAM-2760] Disable testMergingCustomWindows* validatesRunner tests in Gearpump runner
add cd4f5a2 This closes #3712
add 24ee742 Update Custom Window Merging Tests
add 4e01fc1 This closes #3698
add 5e99853 Adds two new Read PTransforms that can be used to read a massive number of files.
add 9088a3e This closes #3661
add 02eb091 [BEAM-2684] Fix flaky AmqpIOTest by introducing ActiveMQ AMQP broker instead of peer-to-peer mode
add 07e8cd5 [BEAM-2684] This closes #3714
add 34d9d91 Updating protobuf for dataflow api
add d035a34 This closes #3713
add e1baf55 Makes Python API reference generation more strict
add 84a2379 This closes #3613
add 1f2dddd Add initial bundle retry code
add b0b6421 This closes #3679
add e73ae39 Throw an Exception if no files are found to stage
add 0f8e8dd This closes #3572
add 075d4d4 Allow users to choose the BigQuery insertion method.
add f7e8f88 This closes #3662: [BEAM-2700] Support load jobs in streaming
add 9ff3ac3 Fix Javadoc for waitUntilFinish
add 3a8b0b6 This closes #3718
add ceec7ce [BEAM-1612] Support real Bundle in Flink runner
add 724eda3 This closes #3368
add 06bd00c Add client-side throttling.
add d0deb6c This closes #3644
add cfb7988 [BEAM-1347] Plumb through a yet to be created state client through PTransformRunnerFactory
add 8503adb [BEAM-1347] Plumb through a yet to be created state client through PTransformRunnerFactory
add 8dc6e16 Fixes a pydocs validation failure due to a recent commit.
add 0aae7aa This closes #3731
add 518c158 Ensure that each triggered load generates a different job id (for the case of streaming triggered file loads), and add test coverage to catch this.
add 9ac94e7 Fix failing test.
add d03a128 This closes 3722
add 1b81f1d Updates ByteKeyRangeTracker so that getFractionConsumed() does not fail for completed trackers.
add ae9a2dc This closes #3715
add b949aa1 [BEAM-1347] Convert an InputStream into an Iterable<T> using the Beam Fn data specification
add 95cd37f [BEAM-1347] Convert an InputStream into an Iterable<T> using the Beam Fn data specification
add 17ff7fb [BEAM-79] Add Gearpump runner profile to Java8 Archetype POM
add d338b44 This closes #3726
add a493f32 Allow termination of DirectRunner execution with Ctrl-C
add 32bf7bc This closes #3741
add 90a850e Avoid passing extra argument to object() constructor in GcsIO
add d8d3f30 This closes #3742
add a72b628 [BEAM-2727] Make jar generation reproducible
add c58c42c [BEAM-2727] Disable timestamps from the Javadoc generation
add 10a49a5 Fix leaking directories/files on ApexYarnLauncherTest
add 62294e6 This closes #3685
add 6809129 [BEAM-1534] Create a dockerized developer environment for Beam
add 9b175cc This closes #3651
add 081df6d [BEAM-2788] Use SerializablePipelineOptions to serde PipelineOptions in Gearpump runner
add f0ce31b This closes 3743
add 1f08970 NexMark
add 1bbfb48 Update Nexmark to the current Beam snapshot 0.7.0
add 37ed77c Add comments on queries improvements and fix compilation config
add a095e40 Make NexmarkRunner generic and remove coupling with Google Dataflow
add a1fe33b Activate monitoring on NexmarkSparkRunner and on specific runners
add 1bd5735 Fix Queries tests
add 9ce9bf0 Fix Apex driver and update execution matrix
add a7f9f7d Refactor classes into packages
add 7bfc982 Improve query5, query10 and query11
add bd93c8b Fix compile after ParDo refactor
add 7c28b49 Fix and improve query3 and query12
add 7ef49dc Improve queries tests
add 8098bb1 Change Nexmark pom structure to mirror other modules on Beam
add e10d578 Fix Spark streaming termination via waitUntilFinish and timeout config
add b438fa7 Remove Accumulators and switch to the Metrics API
add 8b96949 Fix compile after Coders and Pubsub refactor
add a6dbdfa Remove NexmarkDrivers and make execution runner-agnostic
add a39cb80 Move WinningBids into the queries package
add 1541fad Fix static analysis issues
add 902050b Disable use of GcsIOChannelFactory on query10
add 77eabba Clean some code that is specific to Dataflow
add 683680b Rename NexmarkDriver to Main and NexmarkRunner to NexmarkLauncher
add dbd1b15 Change benchmark workload settings
add 69953a0 Remove references to dataflow in generic classes
add 3d5c3d0 Migrate to Beam 2.1.0-SNAPSHOT
add ee500b2 Fix query10 log messages
add 6c11670 Improve NexmarkUtils: improve diskBusy() and remove unneeded randomization code
add 2f9b494 Clean, fix findbugs, fix checkstyle
add f4333df Move module beam-integration-java-nexmark to beam-sdks-java-nexmark
add 0acd720 Update to Beam 2.2.0-SNAPSHOT and final touches before merge into master
add 64ff21f This closes #3114
add 12c0fa6 Add Python mobile gaming streaming examples
add 97f3280 This closes #3483
add e183b24 Fix a bug in AvroIO, in which a SerializableFunction is created with a context containing a un-serializable member (Schema)
add d4db66d This closes #3733: Fix a function serialization bug in AvroIO
add b3f7e21 [BEAM-1347] Provide an abstraction which creates an Iterator view over the Beam Fn State API
add c4517d0 [BEAM-1347] Provide an abstraction which creates an Iterator view over the Beam Fn State API
add 15472b2 [BEAM-2671] Implemented an InputDStream that syncs up with the watermark values, this should help with streaming tests in spark-runner.
add 5181e61 This closes #3738
add 185daff Adding IOTargetName and unittests for CounterName
add 260c4fc This closes #3747
add 8857582 Expand SdkComponents Visibility
add 73c2026 This closes #3755
add 0f53e2a Added tests for python gaming examples
add d261d6b This closes #3756
add ef4239a Add URN registration mechanism for coders.
add 9cc004f Runner API encoding of common coders.
add f3de736 Closes #3746
add c9c2e81 Adds SpannerAccessor - a utility for DoFn's that use Spanner
add c33cb03 This closes #3696: [BEAM-1542] Add SpannerAccessor
add 3362d1f Fix min_timestamp used for KafkaIO watermark.
add 20d88db This closes #3758
add e951baa [BEAM-1531] Add dynamic work rebalancing support for HBaseIO
add f4b66f3 This closes #3754
add 428bfde [BEAM-1347] Remove unused FakeStepContext now that FnApiDoFnRunner has its own implementation
add cdf050c [BEAM-1347] Remove unused FakeStepContext now that FnApiDoFnRunner has its own implementation
add e5bdedd Fix code style issues for HBaseIO
add 77a0a2a This closes #3762
add 2dea491 Actually wait for exector service to shutdown
add 9071c55 This closes #3751
add c4d9646 Ensure Dataflow service version matches Google API service version 1.22.0
add 0f77af8 This closes #3766
add 1e21f45 Add Proto Definitions for the Artifact API
add 456dcc1 This closes #3735
add 8f71dc4 Implement cancel and wait_until_finish(duration) for DataflowRunner
add d50c964 This closes #3757
add dba5e5c Small tweak to View.asList javadoc
add f634aec Small tweak to View.asList javadoc
add 8d36a26 [BEAM-1347] Add a BagUserState implementation over the BeamFnStateClient
add ba5c407 [BEAM-1347] Add a BagUserState implementation over the BeamFnStateClient
add 0cf4543 Initialize the Coder in DecodeAndEmitDoFn
add e6d5e08 Initialize the Coder in DecodeAndEmitDoFn
add 08a4487 Wrap unknown coders in LengthPrefixCoder.
add aef89de Closes #3764
add 4764883 Fix beam_job_api to conform to proto naming conventions.
add bbca4f7 Fix beam_job_api to conform to proto naming conventions.
add 16f1132 Use the same termination logic in different places
add 79a594f This closes #3775
add 0eb8abc Updates Dataflow worker to 20170825
add e33cc24 This closes #3771: Updates Dataflow worker to 20170825
add bf1b0a5 Add README.md to KafkaIO
add f816ad8 This closes #3780
add 12362b9 Remove the circular dependency between pipeline & runner
add 1c26b74 This closes #3770
add b0e03a3 Add support for TimePartitioning in BigQueryIO.write().
add 6280d49 This closes #3663: [BEAM-2390] Add support for TimePartitioning in BigQueryIO
add 97810b4 Removes unnecessary calls to ValueProvider.isAccessible
add c1a7574 This closes #3732: Removes unnecessary calls to ValueProvider.isAccessible
add 43fafcd BEAM-2581: KinesisClientProvider interface needs to be public
add d64f2cc This closes #3540
add 184f7a9 Gets rid of raw type in TextIO.Read.watchForNewFiles
add 84eb7f3 Better-organized javadocs for TextIO and AvroIO
add 82b0852 Adds EmptyMatchTreatment to AvroIO
add f1f3987 Adds AvroIO watchForNewFiles
add 6590aed Fixes a findbugs error in Apex runner
add 5c2cab0 This closes #3725: [BEAM-2827] Introduces AvroIO.watchForNewFiles
add f1b19b7 [BEAM-2644] Introduces TestPipeline.newProvider()
add f6c8405 This closes #3753: [BEAM-2644] Introduces TestPipeline.newProvider()
add e0f628cc [BEAM-1347] Create value state, combining state, and bag state views over the BagUserState.
add 585440d [BEAM-1347] Create value state, combining state, and bag state views over the BagUserState.
add fb2d6b5 [BEAM-1347] Implement a BeamFnStateClient which communicates over gRPC.
add 1cd87e3 [BEAM-1347] Implement a BeamFnStateClient which communicates over gRPC.
add 783f26f [BEAM-2753] Fixes translation of WriteFiles side inputs
add 097aec7 This closes #3765: [BEAM-2753] Fixes translation of WriteFiles side inputs
add 521b2d7 Add a log message to ValueError in AsSingleton
add afe8b0e This closes #3777
add 54489f0 Adds a canonical Compression enum for file-based IOs
add 5cb7be7 This closes #3737: [BEAM-2781] Adds a canonical Compression enum for file-based IOs
add 7f8c6e8 [BEAM-1347] Wire up the BeamFnStateGrpcClientCache implementation into the ProcessBundleHandler
add c9653f2 [BEAM-1347] Wire up the BeamFnStateGrpcClientCache implementation into the ProcessBundleHandler
add 1b6cde0 [BEAM-2802] Support multi-byte custom separator in TextIO
add b844126 This closes #3779: [BEAM-2802] Support multi-byte custom separator in TextIO
add 73535da Support new output type on WriteFiles allowing access to all produced filenames.
add 7d53878 This closes #3573: [BEAM-2624] Allow access to created filenames from WriteFiles
add 3151f26 Isort the libraries
add 0f91f08 autopep8 whitespace fixes W391,W293,W291,E306,E305,E304,E303
add 3aa2bef Closes #3785
add 453896a Improve bundle retry display.
add 2389d56 This closes #3796
add 0cefc8a ElasticsearchIO: Fix Null Pointer Exception in keystore management
add 80aebd9 This closes #3801
add ffd08da [BEAM-2669] Fixed Kryo serialization exception when dstream is cached (by using coders and moving to bytes before attempting to serialise an RDD as part of caching it).
add a481d56 This closes #3749
add c8b99ba [BEAM-2825] Refactored SparkGroupAlsoByWindowViaWindowSet to improve readability.
add f7d4583 This closes #3793
add 58e8a01 Moves Match into FileIO.match()/matchAll()
add 80b9cf9 This closes #3759: [BEAM-2828] Moves Match into FileIO.match()/matchAll()
add 910d02f Introduces FileIO.read()
add 6d4a785 Uses FileIO.read() in TextIO and AvroIO
add 513d26c Introduces TextIO.readFiles()
add abda38d Introduces XmlIO.readFiles
add c3bcd4b This finishes #3799: [BEAM-2828, BEAM-2750] Introduces FileIO.read() and uses it in Text, Avro and Xml
add 46ee5a5 Basic Java wrapper for Fn API data plane
add 06f4b0a FnApiControlClient, de-inverts Fn API control plane
add a1dc9d32 Service for pooling incoming Fn API control plane connections
add 44a7a88 Higher-level SdkHarnessClient
add 0388de1 Add SdkHarnessDoFnRunner
add 8b540d2 Merge remote-tracking branch 'github/pr/3750' into HEAD
add a77c9aa Fix TestUtilsTest that fail on Windows
add 7d6f286 This closes #3809
add 17e346e Ignore HTTP 404 when deleting GCS files
add 2e9b728 This closes #3810: Ignore HTTP 404 when deleting GCS files
add f407b4a [BEAM-2803] Improves performance of JdbcIO for a very large number of query results
add c4c6d93 Implements JdbcIO.readAll() and improves validation messages
add 7fe5b75 This closes #3800: [BEAM-2803, BEAM-2706] JdbcIO improvements
add c2fc368 Fix dataflow runner wait until finish tests
add 1882939 This closes #3811
add 095b09b Incorporate partial writeout fix
add 6dfd3a2 This closes #3787
add 247b133 Add option for toProto/fromProto translations in DirectRunner, disabled by default.
add df1476d Closes #3803
add 8f7ffa4 Fix typo in variable name: window_fn --> windowfn
add bd4eab0 Closes #3805
add 3cba56a [BEAM-2764] Change document size range to fix flakiness on SolrIO tests
add 3bd68ec This closes #3727
add 53d8b27 Add fn API progress reporting protos
add 80c86f8 [BEAM-2822] Add fn API progress reporting protos
add fd274ba [BEAM-2787] Fix MongoDbIO exception when trying to write empty bundle
add 27a5961 [BEAM-2787] This closes #3822
add cf21fd9 Upgrade snappy-java to version 1.1.4
add bff97e0 Upgrade slf4j to version 1.7.25
add 6ca9508 Update snappy-java and slf4j to the latest stable versions
add b88a44a Updates bigtable.version to 1.0.0-pre3.
add 7c5768b This closes #3820
add 35dd2dd Add ThrowingBiConsumer to the set of functional interfaces
add c7ff46d Add ThrowingBiConsumer to the set of functional interfaces
add 71e323c Added snippet tags for documentation
add f8e1192 This closes #3776
add 92676a5 Added concrete example for CoGroupByKey snippet
add e9d3a4a This closes #3798
add 775f839 Add license-maven-plugin and some default license merges
add c2dab4d This closes #3823
add 708abff BEAM-934 Fixed build by fixing firebug error.
add 5b942d8 BEAM-934 Enabled firebug after fixing the bug.
add 7ae6dac BEAM-934 Fixed code after review.
add b700bc4 This closes #3526
add 2265b6c [BEAM-2328] Add TikaIO
add d60b29f [BEAM-2328] This closes #3378
add 8b9fb4c Add DSLs module
add d34c1b8 [BEAM-301] Initial skeleton for Beam SQL
add 321ea38 checkstyle and rename package
add 8c4af35 redesign BeamSqlExpression to execute Calcite SQL expression.
add d28d89e Fix inconsistent mapping for SQL FLOAT
add e5b1b71 [BEAM-2158] Implement the arithmetic operators
add 8f93c51 [BEAM-2161] Add support for String operators
add 0185ce8 [BEAM-2079] Support TextIO as SQL source/sink
add 0a9b23f [BEAM-2006] window support Add support for aggregation: global, HOP, TUMBLE, SESSION, only aggregation function COUNT
add a5995e4 Support common-used aggregation functions in SQL, including: COUNT,SUM,AVG,MAX,MIN
add 65e170b [BEAM-2195] Implement conditional operator (CASE)
add 33e1847 [BEAM-2234] Change return type of buildBeamPipeline to `PCollection<BeamSQLRow>`
add c5c5f20 [BEAM-2149] Improved kafka table implemention.
add 2803d15 support UDF
add f47240f update JavaDoc.
add 43235d7 [BEAM-2255] Implement ORDER BY
add 2677031 [BEAM-2288] Refine DSL interface as design doc of BEAM-2010: 1. rename BeamSqlRunner to BeamSQLEnvironment; 2. Move pipeline as external parameter, to integrate with none-DSL components;
add 31c1e6f [BEAM-2292] Add BeamPCollectionTable to create table from PCollection<BeamSQLRow>
add 1dbd62e fix NoSuchFieldException
add 4869087 [BEAM-2309] Implement VALUES and add support for data type CHAR (to be able to test VALUES)
add 80c9843 [BEAM-2310] Support encoding/decoding of TIME and new DECIMAL data type
add c8fcd02 DSL interface for Beam SQL
add 3cc7950 [BEAM-2329] Add ABS and SQRT math functions
add 3172ee2 upgrade to version 2.1.0-SNAPSHOT
add 2ab2c38 rename SQL to Sql in class name
add 36b4858 [BEAM-2247] Implement date functions in SQL DSL
add 678ed1c [BEAM-2325] Support Set operator: intersect & except
add 5efe209 Add ROUND function on DSL_SQL branch.
add 0b10aee register table for both BeamSql.simpleQuery and BeamSql.query
add 56835a8 Add NOT operator on DSL_SQL branch (plus some refactoring)
add 927e601 [BEAM-2444] BeamSql: use java standard exception
add 5972da8 [BEAM-2442] BeamSql surface api test.
add da82116 [BEAM-2440] BeamSql: reduce visibility
add 145c786 Remove unused BeamPipelineCreator class
add 3cb7300 [BEAM-2443] apply AutoValue to BeamSqlRecordType
add e8c0f70 Update filter/project/aggregation tests to use BeamSql
add f1a3e25 Remove UnsupportedOperationVisitor, which is currently just a no-op
add 4873cfe restrict the scope of BeamSqlEnv
add 5fd9045 Add ACOS, ASIN, ATAN, COS, COT, DEGREES, RADIANS, SIN, TAN, SIGN, LN, LOG10, EXP Functions
add 0c2e2d6 [BEAM-2477] BeamAggregationRel should use Combine.perKey instead of GroupByKey
add a0052b8 use static table name PCOLLECTION in BeamSql.simpleQuery.
add fab80f2 Small fixes to make the example run in a runner agnostic way: - Add direct runner default profile - Add findbugs validation and fix existing findbugs issues - Validate division by zero on arithmetic expression + other minor fixes - Update Calcite version to 1.13
add 8e5e458 [BEAM-2193] Implement FULL, INNER, and OUTER JOIN: - FULL and INNER supported on all variations of unbounded/bounded joins. - OUTER JOIN supported when outer side is unbounded. - Unbounded/bounded joins implemented via side inputs.
add 7145d26 UDAF support: - Adds an abstract class BeamSqlUdaf for defining Calcite SQL UDAFs. - Updates built-in COUNT/SUM/AVG/MAX/MIN accumulators to use this new class.
add 22eff43 BeamSql: refactor the MockedBeamSqlTable and related tests
add 2c0e90d MockedBeamSqlTable -> MockedBoundedTable
add b51b9f4 Test unsupported/invalid cases in DSL tests.
add 2c367b8 [BEAM-2550] add UnitTest for JOIN in DSL
add b14b8b7 support TUMBLE/HOP/SESSION _START function
add 5ac5865 Test queries on unbounded PCollections with BeamSql DSL API. Also add getTYPE(fieldName) overrides to BeamSqlRow.
add 87cce4b [BEAM-2564] add integration test for string functions
add 7e715e8 CAST operator supporting numeric, date and timestamp types
add 4f811b6 POWER function
add d8ff78b support UDF/UDAF in BeamSql
add 0955628 upgrade pom to 2.2.0-SNAPSHOT
add 62be709 [BEAM-2560] Add integration test for arithmetic operators.
add ae14a80 cleanup BeamSqlRow
add 0be4f34 proposal for new UDF
add 201a940 [BEAM-2562] Add integration test for logical operators
add b791d1c [BEAM-2384] CEIL, FLOOR, TRUNCATE, PI, ATAN2 math functions
add 5f7796d [BEAM-2561] add integration test for date functions
add 1bf0850 refactor the datetime test to use ExpressionChecker and fix BeamSqlDivideExpression to speicify round mode.
add 56393c5 [BEAM-2565] add integration test for conditional functions
add 9148f46 rebased, add RAND/RAND_INTEGER
add 6a2caf1 [BEAM-2621] BeamSqlRecordType -> BeamSqlRowType
add 514cd62 [BEAM-2563] Add integration test for math operators Misc: 1. no SQRT in Calcite, converted to POWER; 2. add DECIMAL in BeamSqlDslBase; 3. fix error in BeamSqlRoundExpression; 4. fix error in BeamSqlSignExpression;
add ac971d9 [BEAM-2613] add integration test for comparison operators
add def035a remove README.md and update usages in BeamSqlExample
add 92cf0f2 update pom.xml to fix conflict
add e9c7a07 remove maven-jar-plugin
add d5a551b move dsls/sql to sdks/java/extensions/sql
add e4eca2e rename package org.apache.beam.dsls.sql to org.apache.beam.sdk.extensions.sql
add 7eb113b move all implementation classes/packages into impl package
add 88344fc move BeamRecord to sdk/core
add e77f323 use BitSet for nullFields
add 2fc6f24 refactor BeamRecord, BeamRecordType, BeamSqlRecordType, BeamRecordCoder
add 5b1e4a5 Remove redundant windowing information from the BeamRecord itself element window information `BoundedWindow` is added in `BeamSqlExpression`.
add cddf9d9 [rebased] remove nullFields in BeamRecord
add 214c791 [BEAM-2733] update javadoc for BeamSql
add 96b35f1 [BEAM-2742] change Field type from primitives to boxed types
add 9fe774a [BEAM-2730] make BeamRecord an immutable type
add 6453726 [BEAM-2745] add BeamRecordSqlType.getFieldTypeByIndex()
add b05cf8b Update BeamSqlExample: - Fix mvn example - Add aggregation/group by
add 2f99bf6 take SerializableFunction as UDF.
add 3931bbb [BEAM-2744] rename BeamRecordType#size()
add a3c7975 update JavaDoc for BeamRecord, BeamRecordType. Also only create new UDF class instances for SerializableFunction UDFs.
add bbf205d take CombineFn as UDAF.
add b17cf30 [BEAM-2740] Hide BeamSqlEnv.
add b462610 Add @Experimental and ImmutableList.copyOf
add 309265d Switch to unmodifiableList
add 5e0f397 rename simpleQuery to query and query to queryMulti
add 642f312 update JavaDoc of BeamSortRel
add aa12fb7 var_pop and var_samp
add 39e66e9 This closes #3782
add 0e14976 Fix (and test) FnApiRunner in grpc mode.
add 5a995e6 Closes #3836
add 97806e3 [BEAM-2862] Add PTransform overrides specific to execution of StatefulDoFns over the Fn API using the DataflowRunner.
add 774b268 [BEAM-2862] Add PTransform overrides specific to execution of StatefulDoFns over the Fn API using the DataflowRunner.
add eb6f5bc Key StateTable off id, not full StateTag
add 98b0fd2 This closes #3818: [BEAM-2866] Key StateTable off ID
add 1a38de1 Cleans up PTransform validation across Beam
add e8bf045 This closes #3730: Cleans up PTransform validation across Beam
add 5b2f423 Add ability to set job labels to BEAM Java SDK
add 3a83a66 This closes #3797
add a95dc85 [BEAM-2861] Fix tests which fail with google credential
add fcd11fe This closes #3825
add 89ab3d2 Add first few BigtableWriteException to suppressed list when rethrowing BigtableIO failures.
add 001285a This closes #3763
add 2fa9dd0 [BEAM-2804] support TIMESTAMP in sort
add 8e85b6b This closes #3838
add 97b17b6 Fix GqlQueryTranslateFn to pass localhost option to DatastoreFactory
add 8e391d9 This closes #3740
add add2fa6 Improve error message for bad DoFn URN in ParDoTranslation
add 2c018cd Key DoFnInstanceManager cache on AppliedPTransform
add 3b04d13 This closes #3842: [BEAM-2516] Key DirectRunner DoFn cache off step rather than DoFn
add 9b30bca Add fast path to ParDoTranslation for known ParDo class
add 0bc3756 This closes #3843: [BEAM-2516] Add fast path to ParDoTranslation for known ParDo class
add 09b6b8f [BEAM-2632] Use Junit Paramaterized test suits in TextIOReadTest
add 8d71ebf This closes #3790
add c3d4c5d [BEAM-2859] Fixed processing timers not being properly fired when watermark stays put by tweaking the way spark-runner was delivering timers to reduceFnRunner in SparkGroupAlsoByWindowViaWindowSet
add 50532f0 This closes #3840
add 31f51d2 flink-runner: constructs exception string only when neccessary, it reduces per-element expensive calls(String.format and getSimpleName) in FlinkAssignContext.
add fa4ecea This closes #3841
add be9fb29 Key FlinkRunner streaming translation off URN
add e7601aa This closes #3813: [BEAM-2795] Key FlinkRunner streaming translation off URN
add 0d4fd19 [BEAM-1637] Create Elasticsearch IO compatible with ES 5.x
add 6c21cab Piggyback: deflakes MongoDBGridFSIOTest which failed during merge
add 307b036 This closes #3703: [BEAM-1637] Create Elasticsearch IO compatible with ES 5.x
add 81d0c43 [BEAM-2807] Fix NPE error on CoderTypeSerializerConfigSnapshot serialization
add c09886a This closes #3826
add c329752 [BEAM-2790] Use byte[] instead of ByteBuffer to read from HadoopFilesystem
add 9b652ef Fall back to inputStream.read(byteBuffer) in case of off-heap byteBuffer use guava ByteStreams in place of manual byte skip in UTests
add 082f70c [BEAM-2790] Use byte[] instead of ByteBuffer to read from HadoopFilesystem
add c1266ee Improve performance of bundle retry
add 979923a This closes #3815
add 3a2b418 [BEAM-2956] Attempt to correctly report the Dataflow distribution in GCP related modules.
add 2693bd7 [BEAM-2956] Attempt to correctly report the Dataflow distribution in GCP related modules.
add 7944caa [BEAM-2961] Add missing OnTimeBehavior proto translation to WindowingStrategy translation
add 0b69586 [BEAM-2961] WindowingStrategy proto translation does not translate OnTimeBehavior
add 3e514ce [BEAM-2808] Improving error message in ParDo.validateWindowTypeForMethod.
add bd0facc [BEAM-2808] Improve error message when DoFn @ProcessElement has the wrong window type
add 0bb0bb5 [BEAM-2957] Fix flaky ElasticsearchIOTest.testSplit in beam-sdks-java-io-elasticsearch-tests-5
add 5ea2537 [BEAM-2957] This closes #3860
add fecfbc1 Speeds up CompressedSourceTest
add fdd9965 This closes #3854: Speeds up CompressedSourceTest
add 14ea5ab [BEAM-2948] Register default FileSystems in StreamOperator.setup()
add c65aca0 This closes #3855
add f3f3254 Move DoFnInfo to SDK util
add 056b720 Update Dataflow container version to 20170918
add c25bead This closes #3858: [BEAM-2884] Move DoFnInfo to SDK util
add b3a5b67 [BEAM-2964] Exclude incompatible six release.
add 5dab9e1 Closes #3865
add 517192f [BEAM-407] Fixes findbugs warnings in OffsetRangeTracker
add e7eefdd This closes #3827: [BEAM-407] fixes inconsistent synchronization in OffsetRangeTracker.copy
add bd11589 Refactor fn api runner into universal local runner.
add 2999ec9 Implement ULR subprocess mode.
add cb77874 Allow worker to be started as a subprocess.
add 8a62cea Streaming Job API.
add a1abccd Lint and documentation.
add 28d4f09 Closes #3847
add 72960b3 Allow subclasses of tuple, list, and dict as pvaluish inputs/outputs.
add 07c08cc Support multiple materializations of the same pvalue.
add cfbdb61 Closes #3831
add c6bedf1 Updates grpc-google-pubsub-v1 to grpc-google-cloud-pubsub-v1
add e8a5282 This closes #3862: [BEAM-2837] Updated grpc-google-pubsub-v1 dependency
add 7d7baf7 Marks TikaIO as experimental to allow backward-incompatible changes
add 64123e9 This closes #3870
add ab97531 [BEAM-2377] Allow cross compilation (2.10,2.11) for flink runner
add a1835c6 This closes #3255
add f7ebb62 Add PipelineOptionsTranslation
add a267ca8 This closes #3719
add 8a25597 Add Nullable getters to MetricsContainerImpl
add 59542c3 This closes #3868
add fe6e95b Exclude incompatible six release, part 2
add fe39590 Closes #3872
add a241eda Enable progress request handling in python SDK harness
add 0fa7fe1 Minor cleanup.
add ddc0a7d Closes #3784
add e154843 Revert "[BEAM-2377] Allow cross compilation (2.10,2.11) for flink runner"
add a92c45f Closes #3875
add ccc32a2 Execute windowing in Fn API runner.
add aed6773 Closes #3877
add 77840fa Clarify semantics of objects returned by state access
add 766513b This closes #3876
add bdc738e Mark AuctionOrBidWindowCoder as consistentWithEquals
add e93c035 Enforce correctness of consistentWithEquals and fix broken hashCode() method for AuctionOrBidWindowCoder
add e3abd69 Make AuctionOrBidWindowCoder use structuralValue instead of consistentWithEquals
add d6c6339 This closes #3844: Mark AuctionOrBidWindowCoder as consistentWithEquals
add 4f7e0d6 Initial set of pipeline jobs.
add 4e4d102 This closes #3545
add ed4e868 Send portable ParDo protos to Dataflow instead of just DoFnInfo
add b4deb83 Update Dataflow worker to 20170921
add 63e2965 DataflowRunner depends on, and shades, protobuf
add 3c2f363 Shade generated Runner API classes in Dataflow runner
add 0d5d00d This closes #3859: [BEAM-2884] Send portable protos for ParDo in DataflowRunner
add cc24c86 Make sure that we default to alwaysRetry instead of passing in a null retry policy.
add 63b60b1 Add unit-test coverage.
add 0a073af This closes 3864
add 0f50eb7 Strip table decorators before creating tables.
add fb6417f Move stripping code into BigQueryHelpers and add better unit-test coverage.
add 66b864f This closes #3866
add 3bd7ddb Fix type parameter in AvroIO.Write
add 87116cc This closes #3711
add cf665b6 Revert "This closes #3859: [BEAM-2884] Send portable protos for ParDo in DataflowRunner"
add 3971c7d This closes #3887: Revert #3859: "Send portable protos for ParDo in DataflowRunner"
add 5a66ce9 Revert "Initial set of pipeline jobs."
add 74a5c9e This closes #3884
add 465ecfc Artifact API Cleanup
add 2f178fb Add a Local FS implementation of the Artifact Staging API
add 7dacfdc This closes #3852
add 85901d3 Update Dataflow worker to 20170922-01
add 892e6c8 This closes #2889: [BEAM-2981] Update Dataflow worker version to attain ProtoCoder serialVersionUID fix
add 13ed7ff Included immediate results after CoGroupByKey for better readability in docs
add aa2604a This closes #3867
add ef1e719 [BEAM-2876] Add preliminary provision API
add 06809b9 This closes #3885
add ae81003 Updates WriteToBigQuery PTransform to get project id from GoogleCloudOptions when using DirectRunner.
add a2ff162 This closes #3892
add 2e27526 Use the SubscriptionProvider in PubsubUnboundedSource
add 1bd17d1 This closes #3888
add 21a62ed [BEAM-2467] Kinesis source watermark based on approximateArrivalTimestamp
add 3bf65dc This closes #3851: [BEAM-2467] Kinesis source watermark based on approximateArrivalTimestamp
add 95ef860 Update the Dataflow v1b3 API version
add 446f311 This closes #3896
add 166fadb Change checksum back to md5
add 352f106 This closes #3895
add 6d9b8f0 Move file deletion into subsequent ParDo.
add b2138b0 This closes #3863
add ce6a18c Add test to fix partial writouts after a bundle retry
add 2aa3b5c This closes #3833
add 6dcdea7 Catch subprocess error in proto generation.
add 732ae42 Closes #3901
add 97f7d0b Add some timing information to testing scripts.
add ba9440f Closes #3828
add 9052a16 Use de(stributed)tox rather than plain old tox for Python testing.
add b494c44 Closes #3829
add f54477c Sets a TTL on BigQueryIO.read().fromQuery() temp dataset
add 41239d8 This closes #3883: [BEAM-2991] Sets a TTL on BigQueryIO.read().fromQuery() temp dataset
add b6c68a6 [BEAM-2959] Fix proto enums to use "YYY_UNSPECIFIED" as the first declared enum.
add 87ea068 [BEAM-2959] Fix proto enums to use "YYY_UNSPECIFIED" as the first declared enum.
add 18d7c29 Removes codepaths for reading unsplit BigQuery sources
add 7e3f591 This closes #3891: [BEAM-2992] Removes codepaths for reading unsplit BigQuery sources
add 30e7d15 PAssert improvements.
add b74644a Introduces Reshuffle.viaRandomKey()
add a6e2001 Adds ReadableFile.toString() for debugging
add 393e563 This closes #3890: Introduces Reshuffle.viaRandomKey()
add 2144c8d [BEAM-2606] make WindowFnTestUtils use the value in addition to the timestamp of the elements
add da531b7 [BEAM-2606] make WindowFnTestUtils use the value in addition to the timestamp of the elements
add 86de9de Avoid using beta grpc implementation.
add 3436053 Closes #3911
add 9cea17e Option to disable validation of BigtableIO.write target table
add 0724f1c This closes #3897
add d4323e9 Updated Pipeline job for Jenkins PreCommit
add dc59787 Closes #3916
add c98c804 Support for using raw avro records from BigQuery
add 6be9e0b This closes #3894: [BEAM-2986] Support for reading avro GenericRecords from BigQuery
add 9ddaf29 Adapt Flink StateInternals to new state semantics
add 9c14d73 This closes #3909
add e9f60c4 Move Runners Core Fn Api classes into a package
add 4b7813c This closes #3899: Move Runners Core Fn Api classes into a package
add 8345171 Allow users to skip Java or Python branch via a comment. Also use try/catch blocks to stop non-blocking branches from stopping the build.
add 5cdae88 Modify trigger phrase to require specifying which language.
add 9625de5 Introduce a more sustainable way to add new SDKs.
add 714be39 Update variable references.
add bcebdb4 This closes #3918: [BEAM-2596] Pipeline Job Choose Branch
add d5dffe2 Fix "Writing data to multiple destinations" part of AvroIO javadoc
add 294f633 This closes #3910: Fix "Writing data to multiple destinations" part of AvroIO javadoc
add 6fddd4e [BEAM-2997] Encapsulate enums within a message so that C++/Python have meaningful namespaces when importing.
add 25b9c35 This closes #3913: [BEAM-2997] Encapsulate enums within a message so that C++/Python have meaningful namespaces when importing.
add 7cb3dda [BEAM-2455] Backlog size retrieval for Kinesis source
add 7db0f13 This closes #3551: [BEAM-2455] Backlog size retrieval for Kinesis source
add 452ce1f Increase precommit timeout to 150 minutes
add 8479425 This closes #3926: Increase precommit timeout to 150 minutes
add 575e36e BEAM-1037 Support for new State API in ApexRunner
add ea7940d This closes #3893 [BEAM-1037] Support for new State API in ApexRunner
add 81e3d21 Updating BEAM_CONTAINER_VERSION for new worker
add 4a5b3c0 This closes #3912
add bd39e7b Fix code style issues for HBaseIO
add 81d304d This closes #3932
add 6718182 update shade settings to handle Calcite dependencies
add 9c413d2 add relocate in shading; add NOTICE and include Janino license;
add baa6ebc This closes #3848
add beacf9f Change the state sampler to use structured names
add f9bc763 This closes #3786
add bf0182d [BEAM-2989] Fixed error when using Void type in WithKeys.
add 66d7b6f [BEAM-2989] Fixed error when using Void type in WithKeys.
add a83cae5 [BEAM-2958] Adding user agent string to PipelineOptions.
add ba5bee6 [BEAM-2958] Adding a top-level user agent string to PipelineOptions
add b0c7652 [BEAM-3007] Add test which covers PCollection consumed by Flatten and another primitive.
add 8d8f7fa [BEAM-3007] Add test which covers PCollection consumed by Flatten and another primitive.
add bd76630 Disable testReadPdfFile
add 9379ca2 This closes #3923
add 0576538 Use imports from grpc.
add 929a23e Closes #3940
add 25790a4 Split Java and Python precommit jobs
add dee8e93 This closes #3935: [BEAM-2596] Split Java and Python precommit jobs
add 4f00899 [BEAM-2999] Split ValidatesRunner test from Python Postcommit
add afe8da8 This closes #3927
add 8d8a213 [BEAM-3018] Remove duplicated methods in StructuredCoder
add 12e79d0 This closes #3947
add 5fb3aa0 Enable discovery of log_handler tests.
add 1259ee9 Closes #3946
add f0394a6 Add a Bare-bones ReferenceRunner Job Service
add 294518e This closes #3898
add 060bda2 Remove any_param field from FunctionSpec
add 34eb9b0 This closes #3869
add 9bbed6d Add an Endpoints Proto file
add 5c21ba7 This closes #3917
add 9d46353 Fix ReferenceRunnerJobServer checkstyle
add 0dd4a1c This closes #3951
add dbf1dc0 Add an ArtifactServiceStager
add 2090ee3 This closes #3908
add d1a70a3 Adds API for defining Splittable DoFns.
add 31da49c This closes #3882
add a211bd9 Make PackageUtil a proper class encapsulating its ExecutorService
add a328127 Use AutoValue for Dataflow PackageAttributes
add 58b6453 Refactor PackageUtil for more and simpler asynchrony
add ed00299 This closes #3920: [BEAM-2963] Preliminary refactors to PackageUtil
add c18f15c [BEAM-2877][BEAM-2881] Add Java SDK harness container image and support
add 63b54a5 This closes #3928
add 57bde53 OrderedCode library provides encoding a sequence of typed enities into a byte array that can be lexicographically sorted.
add 480e60f This closes #3925
add bccea9d [BEAM-2701] ensure objectinputstream uses the right classloader for serialization
add 7ad45ad [BEAM-2701] ensure objectinputstream uses the right classloader for serialization
add fc41838 Limit Java postcommit to Java SDK and related
add 707e3e7 This closes #3954: Limit Java postcommit to Java SDK and related
add a4cd96c Enabling phrase triggering of Python PreCommit
add 1e356cf This closes #3942: Enabling phrase triggering of Python PreCommit
add d8f71ff [BEAM-2877] Add presubmit for Go code
add c4085f9 [BEAM-2877] Minor Go fixes for typos, proto changes and beamctl structure
add c44697f Fix typo in python script
add 3d78101 [BEAM-2877] Add netty-transport-native-epoll to Java harness uberjar
add 2e6feaf This closes #3955: [BEAM-2877] Add presubmit for Go code
add c2f3811 whitelist time command in tox to prevent warning
add b82195a This closes #3945
add 7aae214 [BEAM-3031] Use blocking grpc dial in Go unit tests
add b44056f This closes #3958: [BEAM-3031] Use blocking grpc dial in Go unit tests
add ca20e69 [BEAM-3027] Correctly set output type on SourceId-stripper
add ec5e724 This closes #3953
add 742a082 Extend nightly build timeout to 4 hours
add eac561f This closes #3963: Extend nightly build timeout to 4 hours
add ec1782c Fix typo in nightly release script
add 35183c7 Fixes TextIO and AvroIO tests of watchForNewFiles
add 5130cab This closes #3957: Fixes TextIO and AvroIO tests of watchForNewFiles
add c1bfe00 Add @FunctionalInterface annotation to several interfaces
add 63de41a This closes #3934: Add @FunctionalInterface annotation to several JdbcIO interfaces
add b6c838e Ensure metric names are not null or empty
add 61777b4 This closes #3914: [BEAM-2996] Ensure metric names are not null or empty
add 6416deb Allow precommits to coexist with phrase triggering
add b47c372 Restore Python precommit
add 834494a Enable phrase triggering of Java PreCommit
add 687828f Enable phrase triggering of Go PreCommit
add cdefb11 Add runners/direct-java to Java precommit
add 296f7fa This closes #3956: Restore Python precommit; enable phrase triggering for precommits in general
add 999e957 Fixes a bug in query splitting.
add 75bab74 This closes #3962
add 1b59e4d Relocate sdks.common.v1 and portability.v1 in Java direct runner
add f446853 This closes #3964: [BEAM-3037] Relocate sdks.common.v1 and portability.v1 in Java direct runner
add db0d47b [maven-release-plugin] prepare branch release-2.2.0
add 1d675b0 [maven-release-plugin] prepare for next development iteration
add 9f69997 Bump Python version for next release.
add 15b0e47 Only fail nightly release at end, for more detailed signal
add 71e6792 Only fail Java postcommit at end
add 1001cf1 This closes #3968: Only fail nightly release at end, for more detailed signal
add fe88940 Pin runner harness container image for Dataflow distributions of BEAM Python SDK
add b0030ae This closes #3966
add 79c55db [BEAM-2600] Add minimal python SDK harness container
add bbc231c This closes #3965
add b3ec734 Revert "Use de(stributed)tox rather than plain old tox for Python testing."
add cf89936 This closes #3974
add 34a2e0c Make linter happy.
add 90f34f2 This closes #3972
add 992d805 Move the Job Server into a Submodule
add f51a9ab This closes #3967
add f45d4d6 Add Artifact Staging Endpoint to PrepareJobResponse
add 4ccb3ba This closes #3973
add fde0b93 Add model/ and submodules to Maven structure and gen_protos.py
add 2f3af31 Move sdk/common/runner-api protos into model/pipeline and model/job-management
add e233af9 Move sdks/common/fn-api protos to model/fn-execution
add 3a9a435 Remove sdks/common
add 7c563ef Re-namespace model/pipeline to model.pipeline
add ed77269 Re-namespace model/job-management to model.job_management
add 80c6f4e Re-namespace model/fn-execution to model.fn_execution
add 7fa37fd Improve error message when standard_coders.yaml not found
add b2daa0c Dataflow runner: shade protobuf and beam model
add 92615ca This closes #3924: [BEAM-2576] Split Beam portability framework into construction, job, and execution modules
add 7165e55 PipelineOptionsFactory should prevent non PipelineOptions interfaces from being constructed
add dc3e2f7 [BEAM-2779] PipelineOptionsFactory should prevent non PipelineOptions interfaces from being constructed
add b1a22a8 [BEAM-3016] Fix blocking issue within run() when channel terminates while blocking within DirectStreamObserver.
add 6dd90d8 [BEAM-3016] Fix blocking issue within run() when channel terminates while blocking within DirectStreamObserver.
add 5fba632 [BEAM-2576] Reflect portability proto changes in Go
add d22a332 Add licenses to generated code
add 8e1caea This closes #3983: [BEAM-2576] Reflect portability proto changes in Go
add 7b6556c Improve the documentation for CombineFn
add 655227a This closes #3984
add 22f9263 Futurize stage 1 run (AUTOMATED futurize)
add dd6a789 Add futurize to pylint script
add 9473937 re-order imports (AUTOMATED isort)
add 7d0040d Cleanup whitespace (AUTOMATED autopep8)
add 5cc6b5f Change the ptransform_test to use assertEqual rather than assertTrue for improved debugging, fix the error message we're looking for since we now throw a tuple exception instead
add f364248 Add special work to handle indexable return types. Introduce a base IndexableTypeConstraint.
add cd702b3 Fix remaining style issues from auto conversion, switch ref in pylint to stage 1 (stages in futurize are apparently 1 indexed)
add c0bc584 Restore license position that got screwed up
add fe0dc2e Add future to deps and explicitly set isort req and update pylint in tox ini
add a795e54 Change the xxx_todo_changeme[x]s to reasonable values
add 252c679 Parallelize pylint
add bf910c1 Clarify docstring and update run_pylint to be more clear about what steps the developer can take
add 14cf6a1 Re-order comment (reviewer feedback)
add cf2be41 Break out the exclusion list for futurize
add 3b0ad58 Change the examples with long-lambdas which were pulled out into defs to use explicit tuple unpacking instead of indexing. Note: this still keeps the lambdas which are short enough and those use indexing (as required).
add 3a04dbe pr/cl-feedback from c-y-koo
add 9de10e2 Fix snippets
add 9c2a6e7 Closes #3804
add b63a3eb Increment dataflow client version
add 6055d28 This closes #3980
add 2d147eb [BEAM-3048] Remove RAND_RANGE in WindowedWordCount
add f398e5a [BEAM-3048] Remove RAND_RANGE in WindowedWordCount
add 3feef91 Add an option for dataflow job labels.
add a3a7807 This closes #3990
add ec192d1 Align names with those produced by the dataflow runner harness.
add d91ebd9 Fix from any -> bytes transition.
add 3dc7559 Closes #3941
add d226c76 Add an element batching transform.
add 7f5753f Closes #3971
add 4b908c2 Introduces Contextful
add e2ad925 Supports side inputs in MapElements and FlatMapElements
add 014614b This closes #3921: [BEAM-3009] Introduces Contextful machinery and uses it to add side input support to Watch
add 3ad8479 Add progress metrics to Python SDK.
add 245d773 Closes #3939
add 6b25948 Sets user agent in BigTableIO.Read.getBigTableService().
add ec052bb This closes #3996
add ec58a80 Add custom tempLocation support to BigQueryIO.
add a5cbd76 This closes #3995: Add custom gcs temp location support to BigQueryIO
add e940456 Return null when timer not found instead of crashing
add 64fb19d Allow checking timers set in TriggerStateMachineTester
add 45c65c5 Set end-of-window timer in AfterWatermarkStateMachine
add 3d36f63 Do not set EOW timer in TriggerStateMachineTester
add 52863ef This closes #4001: [BEAM-3052] Set end-of-window timer in AfterWatermarkStateMachine
add ee2408b [BEAM-2878] Use bintray as default container images
add f433338 Add license to new file
add c8f3a91 This closes #3981: [BEAM-2878] Use bintray as default for container images
add 6cdea08 [BEAM-1487, BEAM-3016] Address termination correctness issues in BufferingStreamObserver & BeamFnLoggingClient
add 3633c40 [BEAM-1487, BEAM-3016] Address termination correctness issues in BufferingStreamObserver & BeamFnLoggingClient
add 380e995 [BEAM-3063] Improve VoidCoder structural value to use a single shared instance.
add 8ff9c00 [BEAM-3063] Improve VoidCoder structural value to use a single shared instance.
add 2f0a53d [BEAM-409] Avoiding integer division in ceil
add 52ad6f1 This closes #3861
add 2af8b07 Remove "beta" from the Cancel Command Prefix
add 4b502bf This closes #3989
add 0d8ab6c [BEAM-1017] Add RedisIO
add 9524cbd [BEAM-1017]Â This closes #1687
add d745cc9 Kafka exactly-once sink. Tested manually with direct runner and on dataflow.
add 6bf618d Disable other sink tests. Need to fix how tests use global MOCK_PRODUCER. A few more other minor changes.
add d9c95f0 1) Add a note about metadata expiration. 2) Fetch number of partitions if numShards is not set.
add 99f5541 Add basic metrics.
add 5767156 Input PCollection has the coder for elements. No need to ask for them explicitly.
add f4f6105 Close producers after one minute of inactivity. This closes producers when ranges move to other workers. Misc fixes around.
add 236484b assign a random shard id once in setup(), rather than each time in processElement().
add 3bd4085 Improve how we use MockProducer in tests. Remove global instance and use a map to use same producer instance inside beam and test verifier.
add 265e7c7 Support version 0.11 through ProducerSpEL (reflection). Tests work with both 0.11 and older versions.
add 5e280fa Add runner check. Add withReadCommit() to reader.
add 0d5a3e2 update JavaDoc for withEOS().
add 686b22f update JavaDoc for withEOS().
add 0d2a39b This closes #3612
add 0089093 Relocate everything shaded by Java SDK harness
add 0f9f47d This closes #4003: [BEAM-3049] Relocate everything shaded by Java SDK harness
add 249da9b Add missing license header in ProducerSpEL.java.
add 020ef14 This closes #4005: Add missing license header in ProducerSpEL.java.
add 3050bcc Wordcount on fnapi pipeline and IT test.
add 0f7736d This closes #3982
add c14455e Add RawPTransform.migrate(SdkComponents) for re-serialization
add 10c63e1 Add TransformPayloadTranslator.rehydrate to optionally specialize RawPTransform
add 6abf6f5 Add custom rehydration for TestStream
add 92209c3 Add custom rehydration for Combine
add 7fb3e79 Add custom rehydration for ParDo
add 187beae Add custom rehydration for WriteFiles
add 3d99d07 Better error message for RehydratedComponents.getCoder
add c0cb28c Reinstate proto round trip in Java DirectRunner
add 11368e0 Fix typo in UnboundedSource deserialization error message
add 5bc77fc Support side inputs in CombineTranslation
add d684ca0 Add NotSerializable.forUrn to key by URN for non-serializable overrides
add 01103c2 Clearer getOrDefault style in RehydratedComponents
add 505021e DirectRunner: Replace use of RawPTransform with NotSerializable.forUrn translators
add de7cc05 This closes #3938: [BEAM-2674] Add custom rehydration; reinstate proto roundtrip for Java DirectRunner
add 1e3bee1 Update PipelineTest.testReplacedNames
add 77a4d3e This closes #3929
add 2768409 [BEAM-2720] Update kafka client version to 0.11.0.1
add 2acdc74 This closes #4009: [BEAM-2720] Update kafka client version to 0.11.0.1
add 7409ca0 Improve GcsFileSystem errors messages slightly
add 9b866fe Add ability to stage explicit file list
add aea0c60 Stage the portable pipeline in Dataflow
add 090c512 Stage the pipeline without using a temp file
add cef997f Add assertion that valid jobs must have staged pipeline
add 128f3a6 Remove duplicate mocking in DataflowRunnerTest
add 1974b92 This closes #3977: [BEAM-2963] Stage pipeline in DataflowRunner
add 7155931 Pin runner harness also for official BEAM releases.
add 3360b1f This closes #4012
add 9f1db3f Created Java snippets file
add 0f096b1 This closes #3900
add acbbf1f Unit test to repro NPE in PTransformTranslation
add d879d27 Do not crash when RawPTransform has null spec
add 1039f5b This closes #4018: Do not crash when RawPTransform has null spec
add f472439 Avoids generating proto files for Windows if grpcio-tools is not installed.
add 07b6b01 This closes #4007
add 58b3369 Add standalone version of seed job
add 41f1612 This closes #4004: Add standalone version of seed job
add 21cdc85 Adding lull tracking for python sampler
add 49c3927 This closes #3936
add 8242abd [BEAM-2682] Deletes AvroIOTransformTest
add f7aa41a This closes #3999: [BEAM-2682] Deletes AvroIOTransformTest
add 241d3ce Clone source to a distinguished subdirectory of Jenkins workspace
add a29e0ad This closes #3976: Clone source to a distinguished subdirectory of Jenkins workspace
add ff54534 Add dep on Apache-licensed findbugs-annotations implementation
add 4502521 Make Java core SDK root dir NonNull by default
add e33c514 NonNull by default in sdk/coders
add 31eeb10 NonNull by default in sdk/annotations
add 51118fb NonNull by default in sdk/runners
add 524d824 NonNull by default in sdk/state
add 3a64e91 NonNull by default for sdk/testing
add 62529b7 Ignore findbugs in AutoValue generated classes
add a1311d4 NonNull by default in metrics
add 66fcda9 This closes #4021: [BEAM-3081] Findbugs: NonNull by default
add 3d6454e Unit test for label pipeline option
add a71f000 This closes #3994
add 38556b7 Implement FnApi side inputs in Python.
add 77c77ad Closes #4020
add ced7b9c Increase job_beam_PreCommit_Java_MavenInstall timeout from 2.5 to 4 hours.
add 9e93002 This closes #4028: Increase job_beam_PreCommit_Java_MavenInstall timeout from 2.5 to 4 hours
add 073601b Make the main seed job standalone
add fefe153 This closes #4019: Make the main seed job standalone
add dc0ff9a Increase seed job(s) timeout to 30 minutes
add 5b03259 This closes #4031: Increase seed job(s) timeout to 30 minutes
add 5295d02 Increase seed job(s) timeout to 60 minutes
add 2e6ed5f Fix typo in seed jobs
add c440186 Rename seed job so it is first in glob used by prior seed job
add e6d7184 This closes #4033: [BEAM-3094] Fix typo in seed job, process it first
add d54f03b Fix typo in seed job
add a180cf5 Rearrange .gitignore slightly
add 6b572ff Fix RAT exclusions
add cd6379b Manage RAT plugin more centrally; only toggle skipping
add cf6261e This closes #4027: [BEAM-3092] Fix RAT plugin configuration
add 4623109 Increase Java postcommit timeout to 240
add bf1b90d This closes #4038: Increase Java postcommit timeout to 240
add 717ab92 Fix remaining nullability errors
add 94d7f74 This closes #4039: [BEAM-3081] Findbugs stragglers
add 41417f7 Add zip to the list of accepted extra package file types.
add d5aff5d This closes #4022
add 28fbb01 [BEAM-3005] Add resource limits to provision proto
add 8370a8d [BEAM-3005] Set JVM max heap size in java container
add 8fe3a05 Declare .go and Dockerfile as text in gitattributes
add d495d1f Update rat exclusion for python and Go protos
add f56e8f6 CR: [BEAM-3005] Set JVM max heap size in java container
add 192d730 This closes #4029: [BEAM-3005] Add resource limits to provisioning
add 807b4bc Add python_requires to setup.py
add cde3b7b This closes #4026
add e5290f1 [BEAM-2566] Decouple SDK harness from Dataflow runner by elevating experiments and SDK harness configuration to java-sdk-core.
add 77d0058 Add missing @RunWith to test.
add e01c78d [BEAM-2566] Decouple SDK harness from Dataflow runner by elevating experiments and SDK harness configuration to java-sdk-core.
add ba93dd3 Many improvements to TikaIO
add c76c518 Adds ParseResult.failure()
add 0c22113 This closes #3835: [BEAM-2994] Many improvements to TikaIO
add 7d59c96 Stage the pipeline in Python DataflowRunner
add 1f68030 This closes #4010: [BEAM-3074] Stage the pipeline in Python DataflowRunner
add e127c48 Remove extraneous type arguments in PAssert
add 50de293 Remove extraneous type arguments in Latest.java
add 26a3fc5 Suppress AutoValue warnings in TextIO
add 2b8cd3f This closes #4048
add 7eb2446 Remove coveralls invocations from all jobs
add a6f69bd This closes #4043: Remove coveralls invocations from all jobs
add 1fd027b Reading spanner schema transform
add e686286 This closes #4013: [BEAM-1542] A transform for reading the Spanner schema
add 90be820 NonNull by default in sdk/transforms
add d8353b7 NonNull by default in sdk/transforms/windowing
add f9e73db NonNull by default in sdk/transforms/reflect
add e25aba8 NonNull by default in sdk/transforms/display
add 7b2edb1 NonNull by default in sdk/transforms/join
add 82fc720 NonNull by default in sdk/transforms/splittabledofn
add 5fb30ec This closes #4049: [BEAM-3081] NonNull by default for sdk/transforms and below
add f6b4045 [BEAM-3111] Upgrade Elasticsearch to 5.6.3 and clean pom
add 6b91eed This closes #4050
add 3807756 [BEAM-3112] Improve error messages in ElasticsearchIO test utils
add 16b9d58 This closes #4051
add 56b512f [BEAM-2979] Fix a race condition in getWatermark() in KafkaIO.
add aa26f4b This closes #3985
add 3ba9600 [BEAM-1542] SpannerIO: mutation encoding and size estimation improvements
add 09f6815 This closes #4014: [BEAM-1542] SpannerIO: mutation encoding and size estimation improvements
add 324dae7 Temporarily disable Dataflow pipeline_url metadata
add 710941e This closes #4056: [BEAM-3116] Temporarily disable Dataflow metadata
add 39fdace Added VoidSerializer for KafkaIO. Modified KafkaIO.Write.values() to auto add the VoidSerializer for the key.serializer config for kafka producer
add 54bc58b This closes #3969
add 18e66d0 [BEAM-3054] Uses locale-insensitive number formatting in ESIO and tests
add c53a121 This closes #4060: [BEAM-3054] Uses locale-insensitive number formatting in ESIO and tests
add ec39446 [BEAM-2468] Reading Kinesis records in the background
add 612af0a This closes #3930: [BEAM-2468] Reading Kinesis records in the background
add 6179cc7 Allows to set a Cloud Spanner host. https://batch-spanner.googleapis.com/ is set as a default host name.
add 239319b This closes #3907
add e3f6d6f Changed the mutation detector to be based on structural value only
add 84da6ca [BEAM-2482] - CodedValueMutationDetector should use the coders structural value
add b743ab1 [BEAM-2482] - CodedValueMutationDetector should use the coders structural value
add 6443754 Remove obsolete extra parameter
add b013d7c Remove obsolete extra parameter
add ac43634 Updates Python datastore wordcount example to take a dataset parameter.
add 8247b53 This closes #4067
add 6ea8afd Do not relocate generated Model Classes
add 1f37c29 This closes #4044
add 7f9cfdc Fix working dir in website precommits
add a33c717 This closes #4069: Fix working dir in website precommits
add 820f8af [BEAM-2203] Implement TIMESTAMPADD
add ee2292a This closes #4052
add d7329a0 Add License Header to SqlTypeUtils
add 5aaff7b This closes #4072
add a63345b Getting AutoValue 1.5.1 working in Beam.
add 482d178 [BEAM-2530] Updating to AutoValue 1.5.1 to support Java 9.
add b2c9fba Updates BigQueryTableSource to consider data in streaming buffer when determining estimated size.
add 4925478 This closes #4025
add f0b2b3c [BEAM-3135] Adding futures dependency to python SDK
add fbcb0ba This closes #4070
add fdd5971 Add sdks/java/fn-execution
add 927a8db Add a runners/java-fn-execution module
add 4f4632c This closes #3986
add fd58a42 [BEAM-2728] Extension for sketch-based statistics : HyperLogLog
add e25d217 This closes #3991: [BEAM-2728] Extension for sketch-based statistics : HyperLogLog
add 4210045 Add all portability protos to Go
add 6c27b82 Fix Go package comment for syscallx
add 15db1dd [BEAM-3113] Disable stack trace optimization in java container
add fcd864a [BEAM-3114] Generate text proto config properly in container boot code
add 766b4f3 This closes #4053
add 012c2e6 Migrate shared Fn Execution code to Java7
add 7550a58 This closes #4073
add bd1586f Fix Repackaging Configuration in the the DirectRunner
add 6e28892 This closes #4066
add 9e5f641 update dataflow.version
add 35952f6 This closes #4076: [BEAM-3139] Update dataflow.version on the beam root pom.xml
add 227801b Added a preprocessing step to the Cloud Spanner sink.
add 3dfcb44 This closes #3729
add 255b615 Check that bigtableWriter is non-null before calling close().
add 269bf89 This closes #4083: Check that bigtableWriter is non-null before calling close()
add 4fe7480 [BEAM-2203] Implement TIMESTAMPDIFF()
add f80bf2f This closes #4065
add 1a30e42 Ban any Java SDK dependency in Fn Execution Libs
add 13bd266 This closes #4084
add 51c782a Remove extraneous profiles from Python build
add 6339b40 Remove extraneous runner profiles from non-IT steps of pipeline build
add b761f80 This closes #4093: Remove extraneous use of runner profiles
add 70ab3e6 Fix syntax error in ReadSpannerSchema
add 29942939 This closes #4092: [BEAM-3150] Fix syntax error in ReadSpannerSchema
add 0a63eb2 [BEAM-2203] Implement 'timestamp - interval'
add e0166ce This closes #4082
add f78f455 [BEAM-2853] remove Nexmark README.md now that the doc is in the website
add 9c54542 This closes #4097
add f22da33 Added vcf file io source and modified _TextSource to optionally handle headers
add 0af9720 This closes #3979
add b413a96 Ensure Kafka sink serializers are set.
add ae45bbd This closes #4034
add 208fc38 [BEAM-2528] create table
add 5987747 This closes #4041
add 30740a5 [BEAM-3040] Disable flaky subprocess variant of ULR test.
add 29399fd Closes #4099
add 9ed655b Fork Control Clients into java-fn-execution
add 7031159 Fork Data Service Libraries to java-fn-execution
add 727253e This closes #4086
add fe387f1 Merge branch 'master' into jstorm-runner at commit 727253ee3774a3ac4f2841d4276d2bc8e14dcdec
add 050d4de jstorm-runner: update to 2.3.0-SNAPSHOT.
add 9022ef6 jstorm-runner: re-enable sdks/python module.
new ae7357a Merge pull request #4107: Merge branch 'master' into jstorm-runner at commit 727253e
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.gitattributes | 2 +
.github/PULL_REQUEST_TEMPLATE.md | 16 +-
.gitignore | 10 +-
.test-infra/jenkins/PreCommit_Pipeline.groovy | 129 +
.test-infra/jenkins/common_job_properties.groovy | 216 +-
.test-infra/jenkins/job_00_seed.groovy | 114 +
.test-infra/jenkins/job_beam_Java_Build.groovy | 74 +
.../jenkins/job_beam_Java_CodeHealth.groovy | 39 +
.../jenkins/job_beam_Java_IntegrationTest.groovy | 63 +
.test-infra/jenkins/job_beam_Java_UnitTest.groovy | 40 +
.../job_beam_PerformanceTests_Python.groovy | 58 +
...job_beam_PostCommit_Java_JDKVersionsTest.groovy | 62 +
.../job_beam_PostCommit_Java_MavenInstall.groovy | 21 +-
...eam_PostCommit_Java_MavenInstall_Windows.groovy | 5 +-
...PostCommit_Java_ValidatesRunner_Gearpump.groovy | 2 +-
...stCommit_Python_ValidatesRunner_Dataflow.groovy | 54 +
.../job_beam_PreCommit_Go_MavenInstall.groovy | 56 +
.../job_beam_PreCommit_Java_MavenInstall.groovy | 21 +-
.../jenkins/job_beam_PreCommit_Pipeline.groovy | 84 +
.../job_beam_PreCommit_Python_MavenInstall.groovy | 57 +
.../job_beam_PreCommit_Website_Merge.groovy | 62 +
.../job_beam_PreCommit_Website_Stage.groovy | 3 +
.../jenkins/job_beam_PreCommit_Website_Test.groovy | 3 +
.../jenkins/job_beam_Python_UnitTest.groovy | 40 +
.../job_beam_Release_NightlySnapshot.groovy | 22 +-
.test-infra/jenkins/job_seed.groovy | 53 -
.test-infra/jenkins/job_seed_standalone.groovy | 114 +
.../kubernetes/cassandra/LargeITCluster/setup.sh | 21 +
.../cassandra/LargeITCluster/start-up.sh | 22 -
.../cassandra/LargeITCluster/teardown.sh | 1 -
.../kubernetes/cassandra/SmallITCluster/setup.sh | 22 +
.../cassandra/SmallITCluster/start-up.sh | 23 -
.../cassandra/SmallITCluster/teardown.sh | 1 -
.../elasticsearch/LargeProductionCluster/setup.sh | 21 +
.../LargeProductionCluster/start-up.sh | 22 -
.../LargeProductionCluster/teardown.sh | 1 -
.../elasticsearch/SmallITCluster/setup.sh | 22 +
.../elasticsearch/SmallITCluster/start-up.sh | 23 -
.../elasticsearch/SmallITCluster/teardown.sh | 1 -
.../kubernetes/postgres/pkb-config-local.yml | 34 +
.test-infra/kubernetes/postgres/pkb-config.yml | 32 +
README.md | 4 +-
examples/java/pom.xml | 52 +-
.../apache/beam/examples/WindowedWordCount.java | 1 -
.../java/org/apache/beam/examples/WordCount.java | 4 +
.../examples/common/WriteOneFilePerWindow.java | 59 +-
.../org/apache/beam/examples/complete/TfIdf.java | 17 +-
.../examples/complete/TopWikipediaSessions.java | 24 +-
.../beam/examples/complete/TrafficRoutes.java | 19 +
.../beam/examples/cookbook/BigQueryTornadoes.java | 2 +-
.../examples/cookbook/CombinePerKeyExamples.java | 2 +-
.../beam/examples/cookbook/FilterExamples.java | 2 +-
.../beam/examples/cookbook/JoinExamples.java | 6 +-
.../beam/examples/cookbook/MaxPerKeyExamples.java | 2 +-
.../beam/examples/cookbook/TriggerExample.java | 6 +-
.../beam/examples/DebuggingWordCountTest.java | 11 +-
.../apache/beam/examples/WindowedWordCountIT.java | 4 +-
examples/java8/pom.xml | 55 +-
.../examples/complete/game/injector/Injector.java | 2 +-
.../complete/game/injector/InjectorUtils.java | 2 +-
.../examples/complete/game/utils/WriteToText.java | 49 +-
.../beam/examples/website_snippets/Snippets.java | 87 +
.../examples/complete/game/LeaderBoardTest.java | 2 +
.../examples/website_snippets/SnippetsTest.java | 114 +
examples/pom.xml | 2 +-
model/fn-execution/pom.xml | 114 +
.../fn-execution/src/main/proto/beam_fn_api.proto | 729 ++
.../src/main/proto/beam_provision_api.proto | 99 +
.../model/fnexecution}/v1/standard_coders.yaml | 0
model/job-management/pom.xml | 114 +
.../src/main/proto/beam_artifact_api.proto | 134 +
.../src/main/proto/beam_job_api.proto | 174 +
model/pipeline/pom.xml | 89 +
.../pipeline/src/main/proto/beam_runner_api.proto | 843 ++
model/pipeline/src/main/proto/endpoints.proto | 47 +
.../src/main/proto/standard_window_fns.proto | 54 +
model/pom.xml | 40 +
pom.xml | 601 +-
runners/apex/pom.xml | 38 +-
.../beam/runners/apex/ApexPipelineOptions.java | 2 +-
.../org/apache/beam/runners/apex/ApexRunner.java | 171 +-
.../apache/beam/runners/apex/ApexRunnerResult.java | 4 +-
.../apex/translation/ApexPipelineTranslator.java | 31 +-
.../translation/FlattenPCollectionTranslator.java | 15 +-
.../runners/apex/translation/ParDoTranslator.java | 42 +-
.../apex/translation/TranslationContext.java | 9 +-
.../operators/ApexGroupByKeyOperator.java | 10 +-
.../translation/operators/ApexParDoOperator.java | 59 +-
.../operators/ApexReadUnboundedInputOperator.java | 6 +-
.../apex/translation/utils/ApexStateInternals.java | 17 +-
.../apex/translation/utils/NoOpStepContext.java | 37 +-
.../utils/SerializablePipelineOptions.java | 78 -
.../apex/translation/utils/ValuesSource.java | 6 +-
.../beam/runners/apex/ApexYarnLauncherTest.java | 9 +-
.../runners/apex/examples/UnboundedTextSource.java | 6 +-
.../beam/runners/apex/examples/WordCountTest.java | 8 +-
.../translation/ApexGroupByKeyOperatorTest.java | 6 +-
.../FlattenPCollectionTranslatorTest.java | 1 -
.../apex/translation/GroupByKeyTranslatorTest.java | 6 +-
.../apex/translation/ParDoTranslatorTest.java | 4 +-
.../translation/utils/ApexStateInternalsTest.java | 411 +-
.../apex/translation/utils/CollectionSource.java | 6 +-
.../translation/utils/PipelineOptionsTest.java | 150 -
runners/core-construction-java/pom.xml | 46 +-
.../core/construction/ArtifactServiceStager.java | 244 +
.../core/construction/CoderTranslation.java | 183 +
.../beam/runners/core/construction/Coders.java | 193 -
.../core/construction/CombineTranslation.java | 339 +
.../CreatePCollectionViewTranslation.java | 131 +
.../core/construction/DisplayDataTranslation.java | 39 +
.../core/construction/FlattenTranslator.java | 69 +
.../core/construction/ForwardingPTransform.java | 18 +-
.../core/construction/GroupByKeyTranslation.java | 65 +
.../core/construction/PCollectionTranslation.java | 96 +
.../runners/core/construction/PCollections.java | 97 -
.../core/construction/PTransformMatchers.java | 109 +-
.../core/construction/PTransformReplacements.java | 6 +
.../core/construction/PTransformTranslation.java | 443 +
.../runners/core/construction/PTransforms.java | 110 -
.../core/construction/ParDoTranslation.java | 768 ++
.../construction/PipelineOptionsTranslation.java | 51 +
.../core/construction/PipelineTranslation.java | 195 +
.../runners/core/construction/PrimitiveCreate.java | 14 +-
.../runners/core/construction/ReadTranslation.java | 225 +
.../core/construction/RehydratedComponents.java | 160 +
.../core/construction/RunnerPCollectionView.java | 113 +
.../runners/core/construction/SdkComponents.java | 79 +-
.../construction/SerializablePipelineOptions.java | 74 +
.../runners/core/construction/SplittableParDo.java | 375 +
.../core/construction/TestStreamTranslation.java | 316 +
.../runners/core/construction/TransformInputs.java | 50 +
.../TransformPayloadTranslatorRegistrar.java | 31 +
.../core/construction/TriggerTranslation.java | 336 +
.../beam/runners/core/construction/Triggers.java | 336 -
.../core/construction/WindowIntoTranslation.java | 149 +
.../core/construction/WindowingStrategies.java | 275 -
.../construction/WindowingStrategyTranslation.java | 405 +
.../core/construction/WriteFilesTranslation.java | 335 +
.../core/construction/metrics/MetricFiltering.java | 102 +
.../core/construction/metrics/MetricKey.java | 43 +
.../core/construction/metrics/package-info.java | 22 +
.../beam/runners/core/metrics/MetricFiltering.java | 102 -
.../beam/runners/core/metrics/MetricKey.java | 43 -
.../beam/runners/core/metrics/package-info.java | 22 -
.../construction/ArtifactServiceStagerTest.java | 141 +
.../core/construction/CoderTranslationTest.java | 167 +
.../beam/runners/core/construction/CodersTest.java | 164 -
.../core/construction/CombineTranslationTest.java | 224 +
.../CreatePCollectionViewTranslationTest.java | 134 +
.../construction/ForwardingPTransformTest.java | 17 +-
.../construction/GroupByKeyTranslationTest.java | 44 +
.../InMemoryArtifactStagerService.java | 152 +
.../construction/PCollectionTranslationTest.java | 227 +
.../core/construction/PCollectionsTest.java | 189 -
.../core/construction/PTransformMatchersTest.java | 167 +-
.../construction/PTransformTranslationTest.java | 220 +
.../runners/core/construction/PTransformsTest.java | 188 -
.../core/construction/ParDoTranslationTest.java | 323 +
.../PipelineOptionsTranslationTest.java | 143 +
.../core/construction/PipelineTranslationTest.java | 199 +
.../core/construction/ReadTranslationTest.java | 173 +
.../construction/RehydratedComponentsTest.java | 96 +
.../core/construction/ReplacementOutputsTest.java | 14 +-
.../core/construction/SdkComponentsTest.java | 102 +-
.../SerializablePipelineOptionsTest.java | 89 +
.../core/construction/SplittableParDoTest.java | 158 +
.../construction/TestStreamTranslationTest.java | 130 +
.../core/construction/TransformInputsTest.java | 166 +
.../core/construction/TriggerTranslationTest.java | 112 +
.../runners/core/construction/TriggersTest.java | 111 -
.../UnboundedReadFromBoundedSourceTest.java | 2 +-
.../construction/WindowIntoTranslationTest.java | 127 +
.../core/construction/WindowingStrategiesTest.java | 110 -
.../WindowingStrategyTranslationTest.java | 124 +
.../construction/WriteFilesTranslationTest.java | 195 +
.../construction/metrics/MetricFilteringTest.java | 148 +
.../runners/core/metrics/MetricFilteringTest.java | 148 -
runners/core-java/pom.xml | 31 +-
.../beam/runners/core/BaseExecutionContext.java | 164 -
.../org/apache/beam/runners/core/DoFnRunners.java | 3 +-
.../beam/runners/core/ElementAndRestriction.java | 42 -
.../runners/core/ElementAndRestrictionCoder.java | 88 -
.../apache/beam/runners/core/ExecutionContext.java | 99 -
.../core/GroupAlsoByWindowViaWindowSetNewDoFn.java | 4 +-
.../runners/core/GroupByKeyViaGroupByKeyOnly.java | 15 +-
.../beam/runners/core/InMemoryStateInternals.java | 39 +-
.../beam/runners/core/InMemoryTimerInternals.java | 37 +-
.../runners/core/LateDataDroppingDoFnRunner.java | 33 +-
...TimeBoundedSplittableProcessElementInvoker.java | 78 +-
.../apache/beam/runners/core/ProcessFnRunner.java | 21 +-
.../apache/beam/runners/core/ReduceFnRunner.java | 135 +-
.../apache/beam/runners/core/SideInputHandler.java | 2 +-
.../apache/beam/runners/core/SimpleDoFnRunner.java | 414 +-
.../apache/beam/runners/core/SplittableParDo.java | 602 --
.../core/SplittableParDoViaKeyedWorkItems.java | 435 +
.../core/SplittableProcessElementInvoker.java | 25 +-
.../org/apache/beam/runners/core/StateTable.java | 40 +-
.../org/apache/beam/runners/core/StateTags.java | 16 +
.../org/apache/beam/runners/core/StepContext.java | 33 +
.../apache/beam/runners/core/SystemReduceFn.java | 6 +
.../runners/core/TestInMemoryStateInternals.java | 6 +-
.../apache/beam/runners/core/WatermarkHold.java | 6 +-
.../beam/runners/core/fn/FnApiControlClient.java | 152 +
.../core/fn/FnApiControlClientPoolService.java | 72 +
.../beam/runners/core/fn/FnDataReceiver.java | 37 +
.../apache/beam/runners/core/fn/FnDataService.java | 85 +
.../beam/runners/core/fn/SdkHarnessClient.java | 176 +
.../beam/runners/core/fn/SdkHarnessDoFnRunner.java | 108 +
.../apache/beam/runners/core/fn/package-info.java | 22 +
.../beam/runners/core/metrics/CounterCell.java | 10 +-
.../beam/runners/core/metrics/DirtyState.java | 4 +-
.../runners/core/metrics/DistributionCell.java | 10 +-
.../beam/runners/core/metrics/GaugeCell.java | 11 +-
.../beam/runners/core/metrics/MetricUpdates.java | 1 +
.../runners/core/metrics/MetricsContainerImpl.java | 45 +-
.../core/metrics/MetricsContainerStepMap.java | 2 +
.../core/triggers/AfterAllStateMachine.java | 25 +-
.../AfterDelayFromFirstElementStateMachine.java | 6 +-
.../core/triggers/AfterFirstStateMachine.java | 20 +-
.../core/triggers/AfterPaneStateMachine.java | 6 +-
.../core/triggers/AfterWatermarkStateMachine.java | 15 +-
.../triggers/ExecutableTriggerStateMachine.java | 23 +-
.../runners/core/triggers/NeverStateMachine.java | 5 +-
.../runners/core/triggers/TriggerStateMachine.java | 27 -
.../core/triggers/TriggerStateMachineRunner.java | 3 +-
.../core/triggers/TriggerStateMachines.java | 2 +-
.../core/ElementAndRestrictionCoderTest.java | 127 -
.../runners/core/InMemoryStateInternalsTest.java | 569 +-
...BoundedSplittableProcessElementInvokerTest.java | 47 +-
.../beam/runners/core/ReduceFnRunnerTest.java | 457 +-
.../apache/beam/runners/core/ReduceFnTester.java | 80 +-
.../beam/runners/core/SimpleDoFnRunnerTest.java | 4 +-
.../runners/core/SplittableParDoProcessFnTest.java | 594 ++
.../beam/runners/core/SplittableParDoTest.java | 606 --
.../beam/runners/core/StateInternalsTest.java | 672 ++
.../beam/runners/core/StatefulDoFnRunnerTest.java | 4 +-
.../apache/beam/runners/core/WindowMatchers.java | 15 +
.../core/fn/FnApiControlClientPoolServiceTest.java | 65 +
.../runners/core/fn/FnApiControlClientTest.java | 139 +
.../beam/runners/core/fn/SdkHarnessClientTest.java | 96 +
.../runners/core/fn/SdkHarnessDoFnRunnerTest.java | 73 +
.../core/metrics/MetricsContainerImplTest.java | 10 +
.../core/triggers/AfterFirstStateMachineTest.java | 5 +-
.../triggers/AfterWatermarkStateMachineTest.java | 36 +-
.../core/triggers/StubTriggerStateMachine.java | 7 +-
.../core/triggers/TriggerStateMachineTester.java | 13 +-
.../core/triggers/TriggerStateMachinesTest.java | 2 +-
runners/direct-java/pom.xml | 190 +-
.../direct/BoundedReadEvaluatorFactory.java | 14 +-
.../beam/runners/direct/CommittedResult.java | 12 +-
.../direct/CopyOnAccessInMemoryStateInternals.java | 10 +-
.../runners/direct/DirectExecutionContext.java | 35 +-
...DirectGBKIntoKeyedWorkItemsOverrideFactory.java | 9 +-
.../apache/beam/runners/direct/DirectGraph.java | 62 +-
.../beam/runners/direct/DirectGraphVisitor.java | 64 +-
.../beam/runners/direct/DirectGroupByKey.java | 50 +-
.../direct/DirectGroupByKeyOverrideFactory.java | 14 +-
.../apache/beam/runners/direct/DirectMetrics.java | 4 +-
.../beam/runners/direct/DirectRegistrar.java | 2 +-
.../apache/beam/runners/direct/DirectRunner.java | 119 +-
.../beam/runners/direct/DirectTestOptions.java | 42 +
.../beam/runners/direct/DirectTimerInternals.java | 9 +
.../beam/runners/direct/EmptyInputProvider.java | 8 +-
.../beam/runners/direct/EvaluationContext.java | 72 +-
.../direct/ExecutorServiceParallelExecutor.java | 27 +-
.../direct/GroupAlsoByWindowEvaluatorFactory.java | 28 +-
.../runners/direct/KeyedPValueTrackingVisitor.java | 11 +-
.../beam/runners/direct/MultiStepCombine.java | 439 +
.../apache/beam/runners/direct/ParDoEvaluator.java | 18 +-
.../beam/runners/direct/ParDoEvaluatorFactory.java | 55 +-
.../runners/direct/ParDoMultiOverrideFactory.java | 129 +-
.../beam/runners/direct/ReadEvaluatorFactory.java | 97 +
.../beam/runners/direct/RootInputProvider.java | 7 +-
.../beam/runners/direct/RootProviderRegistry.java | 28 +-
.../apache/beam/runners/direct/SourceShard.java | 33 +
.../SplittableProcessElementsEvaluatorFactory.java | 55 +-
.../direct/StatefulParDoEvaluatorFactory.java | 29 +-
.../runners/direct/TestStreamEvaluatorFactory.java | 46 +-
.../runners/direct/TransformEvaluatorRegistry.java | 135 +-
.../direct/UnboundedReadEvaluatorFactory.java | 31 +-
.../beam/runners/direct/ViewEvaluatorFactory.java | 8 +-
.../beam/runners/direct/ViewOverrideFactory.java | 71 +-
.../beam/runners/direct/WatermarkManager.java | 37 +-
.../runners/direct/WindowEvaluatorFactory.java | 5 +-
.../runners/direct/WriteWithShardingFactory.java | 51 +-
.../direct/BoundedReadEvaluatorFactoryTest.java | 6 +-
.../beam/runners/direct/CommittedResultTest.java | 43 +-
.../CopyOnAccessInMemoryStateInternalsTest.java | 74 +-
.../runners/direct/DirectGraphVisitorTest.java | 13 +-
.../apache/beam/runners/direct/DirectGraphs.java | 17 +-
.../beam/runners/direct/DirectMetricsTest.java | 2 +-
.../beam/runners/direct/DirectRegistrarTest.java | 2 +-
.../beam/runners/direct/DirectRunnerTest.java | 4 +-
.../beam/runners/direct/EvaluationContextTest.java | 31 +-
.../direct/ImmutabilityEnforcementFactoryTest.java | 4 +-
.../beam/runners/direct/MultiStepCombineTest.java | 228 +
.../beam/runners/direct/ParDoEvaluatorTest.java | 11 +-
.../direct/StatefulParDoEvaluatorFactoryTest.java | 65 +-
.../beam/runners/direct/TransformExecutorTest.java | 12 +-
.../direct/UnboundedReadEvaluatorFactoryTest.java | 5 +-
.../runners/direct/ViewEvaluatorFactoryTest.java | 8 +-
.../runners/direct/ViewOverrideFactoryTest.java | 70 +-
.../direct/WatermarkCallbackExecutorTest.java | 1 +
.../beam/runners/direct/WatermarkManagerTest.java | 16 +-
.../runners/direct/WindowEvaluatorFactoryTest.java | 10 +
.../direct/WriteWithShardingFactoryTest.java | 75 +-
runners/flink/pom.xml | 28 +-
.../runners/flink/CreateStreamingFlinkView.java | 156 +
.../flink/FlinkBatchPipelineTranslator.java | 5 +-
.../flink/FlinkBatchTranslationContext.java | 3 +-
.../flink/FlinkPipelineExecutionEnvironment.java | 6 +-
.../beam/runners/flink/FlinkPipelineOptions.java | 22 +
.../org/apache/beam/runners/flink/FlinkRunner.java | 10 +-
.../flink/FlinkStreamingPipelineTranslator.java | 87 +-
.../flink/FlinkStreamingTransformTranslators.java | 412 +-
.../flink/FlinkStreamingTranslationContext.java | 3 +-
.../runners/flink/FlinkStreamingViewOverrides.java | 372 -
.../runners/flink/FlinkTransformOverrides.java | 53 +
.../translation/functions/FlinkAssignContext.java | 17 +-
.../translation/functions/FlinkDoFnFunction.java | 24 +-
.../FlinkMergingNonShuffleReduceFunction.java | 8 +-
.../functions/FlinkNoOpStepContext.java | 36 +-
.../functions/FlinkPartialReduceFunction.java | 8 +-
.../translation/functions/FlinkReduceFunction.java | 8 +-
.../functions/FlinkStatefulDoFnFunction.java | 20 +-
.../translation/types/CoderTypeSerializer.java | 78 +
.../translation/types/EncodedValueSerializer.java | 18 +-
.../utils/SerializedPipelineOptions.java | 77 -
.../translation/wrappers/SourceInputFormat.java | 10 +-
.../wrappers/streaming/DoFnOperator.java | 570 +-
.../wrappers/streaming/SplittableDoFnOperator.java | 38 +-
.../wrappers/streaming/WindowDoFnOperator.java | 8 +-
.../streaming/io/BoundedSourceWrapper.java | 10 +-
.../streaming/io/UnboundedSocketSource.java | 2 +-
.../streaming/io/UnboundedSourceWrapper.java | 19 +-
.../state/FlinkBroadcastStateInternals.java | 29 +-
.../streaming/state/FlinkSplitStateInternals.java | 8 +-
.../streaming/state/FlinkStateInternals.java | 503 +-
.../beam/runners/flink/PipelineOptionsTest.java | 191 +-
.../runners/flink/streaming/DoFnOperatorTest.java | 332 +-
.../FlinkBroadcastStateInternalsTest.java | 242 +-
.../streaming/FlinkKeyGroupStateInternalsTest.java | 359 +-
.../streaming/FlinkSplitStateInternalsTest.java | 132 +-
.../flink/streaming/FlinkStateInternalsTest.java | 343 +-
.../flink/streaming/TestCountingSource.java | 53 +-
.../streaming/UnboundedSourceWrapperTest.java | 330 +-
.../translation/types/CoderTypeSerializerTest.java | 79 +
runners/gcp/gcemd/Dockerfile | 30 +
runners/gcp/gcemd/main.go | 85 +
runners/gcp/gcemd/pom.xml | 154 +
runners/gcp/gcsproxy/Dockerfile | 30 +
runners/gcp/gcsproxy/main.go | 91 +
runners/gcp/gcsproxy/pom.xml | 154 +
runners/gcp/pom.xml | 38 +
runners/gearpump/README.md | 61 +
runners/gearpump/pom.xml | 277 +
.../runners/gearpump/GearpumpPipelineOptions.java | 67 +
.../runners/gearpump/GearpumpPipelineResult.java | 110 +
.../beam/runners/gearpump/GearpumpRunner.java | 119 +
.../runners/gearpump/GearpumpRunnerRegistrar.java | 64 +
.../beam/runners/gearpump/TestGearpumpRunner.java | 63 +
.../apache/beam/runners/gearpump/package-info.java | 22 +
.../CreateGearpumpPCollectionViewTranslator.java | 45 +
.../translators/CreateStreamingGearpumpView.java | 156 +
.../translators/FlattenPCollectionsTranslator.java | 83 +
.../translators/GearpumpPipelineTranslator.java | 143 +
.../gearpump/translators/GroupByKeyTranslator.java | 258 +
.../translators/ParDoMultiOutputTranslator.java | 103 +
.../translators/ReadBoundedTranslator.java | 46 +
.../translators/ReadUnboundedTranslator.java | 48 +
.../gearpump/translators/TransformTranslator.java | 30 +
.../gearpump/translators/TranslationContext.java | 105 +
.../translators/WindowAssignTranslator.java | 103 +
.../translators/functions/DoFnFunction.java | 193 +
.../translators/functions/package-info.java | 22 +
.../translators/io/BoundedSourceWrapper.java | 45 +
.../gearpump/translators/io/GearpumpSource.java | 114 +
.../translators/io/UnboundedSourceWrapper.java | 46 +
.../gearpump/translators/io/ValuesSource.java | 165 +
.../gearpump/translators/io/package-info.java | 22 +
.../runners/gearpump/translators/package-info.java | 22 +
.../translators/utils/DoFnRunnerFactory.java | 83 +
.../translators/utils/NoOpStepContext.java | 41 +
.../translators/utils/TranslatorUtils.java | 198 +
.../gearpump/translators/utils/package-info.java | 22 +
.../gearpump/GearpumpRunnerRegistrarTest.java | 55 +
.../beam/runners/gearpump/PipelineOptionsTest.java | 73 +
...reateGearpumpPCollectionViewTranslatorTest.java | 56 +
.../FlattenPCollectionsTranslatorTest.java | 155 +
.../translators/GroupByKeyTranslatorTest.java | 152 +
.../translators/ReadBoundedTranslatorTest.java | 70 +
.../translators/ReadUnboundedTranslatorTest.java | 70 +
.../translators/WindowAssignTranslatorTest.java | 110 +
.../translators/io/GearpumpSourceTest.java | 99 +
.../gearpump/translators/io/ValueSoureTest.java | 79 +
.../translators/utils/TranslatorUtilsTest.java | 73 +
runners/google-cloud-dataflow-java/pom.xml | 174 +-
.../beam/runners/dataflow/AssignWindows.java | 90 -
.../dataflow/BatchStatefulParDoOverrides.java | 75 +-
.../beam/runners/dataflow/BatchViewOverrides.java | 201 +-
.../beam/runners/dataflow/CreateDataflowView.java | 7 +-
.../beam/runners/dataflow/DataflowMetrics.java | 306 +-
.../beam/runners/dataflow/DataflowPipelineJob.java | 18 +-
.../dataflow/DataflowPipelineTranslator.java | 143 +-
.../beam/runners/dataflow/DataflowRunner.java | 489 +-
.../beam/runners/dataflow/DataflowRunnerInfo.java | 91 +-
.../dataflow/PrimitiveParDoSingleFactory.java | 14 +-
.../beam/runners/dataflow/ReadTranslator.java | 7 +-
.../runners/dataflow/SplittableParDoOverrides.java | 76 +
.../runners/dataflow/StreamingViewOverrides.java | 10 +-
.../beam/runners/dataflow/TransformTranslator.java | 30 +-
.../dataflow/options/CloudDebuggerOptions.java | 3 +-
.../options/DataflowPipelineDebugOptions.java | 23 +-
.../dataflow/options/DataflowPipelineOptions.java | 8 +
.../options/DataflowPipelineWorkerPoolOptions.java | 3 +
.../dataflow/options/DataflowProfilingOptions.java | 3 +-
.../options/DataflowWorkerLoggingOptions.java | 6 +
.../runners/dataflow/util/CloudObjectKinds.java | 1 +
.../dataflow/util/CloudObjectTranslators.java | 5 +-
...DefaultCoderCloudObjectTranslatorRegistrar.java | 2 +
.../beam/runners/dataflow/util/DoFnInfo.java | 129 -
.../beam/runners/dataflow/util/GcsStager.java | 53 +-
.../beam/runners/dataflow/util/MonitoringUtil.java | 18 +-
.../beam/runners/dataflow/util/PackageUtil.java | 581 +-
.../beam/runners/dataflow/util/PropertyNames.java | 47 +-
.../runners/dataflow/util/RandomAccessData.java | 2 +-
.../apache/beam/runners/dataflow/util/Stager.java | 32 +-
.../beam/runners/dataflow/util/TimeUtil.java | 24 +-
.../dataflow/BatchStatefulParDoOverridesTest.java | 46 +-
.../beam/runners/dataflow/DataflowMetricsTest.java | 215 +-
.../dataflow/DataflowPipelineTranslatorTest.java | 214 +-
.../runners/dataflow/DataflowRunnerInfoTest.java | 9 +
.../beam/runners/dataflow/DataflowRunnerTest.java | 334 +-
.../transforms/DataflowGroupByKeyTest.java | 12 +-
.../dataflow/transforms/DataflowViewTest.java | 14 +-
.../runners/dataflow/util/MonitoringUtilTest.java | 4 +-
.../runners/dataflow/util/PackageUtilTest.java | 124 +-
.../beam/runners/dataflow/util/TimeUtilTest.java | 6 +
runners/java-fn-execution/pom.xml | 135 +
.../beam/runners/fnexecution/ServerFactory.java | 104 +
.../fnexecution/control/FnApiControlClient.java | 148 +
.../control/FnApiControlClientPoolService.java | 66 +
.../fnexecution/control/SdkHarnessClient.java | 173 +
.../runners/fnexecution/control/package-info.java | 23 +
.../runners/fnexecution/data/FnDataReceiver.java | 27 +
.../runners/fnexecution/data/package-info.java | 23 +
.../beam/runners/fnexecution/package-info.java | 23 +
.../runners/fnexecution/ServerFactoryTest.java | 153 +
.../control/FnApiControlClientPoolServiceTest.java | 65 +
.../control/FnApiControlClientTest.java | 139 +
.../fnexecution/control/SdkHarnessClientTest.java | 96 +
runners/jstorm/pom.xml | 2 +-
.../jstorm/translation/DefaultStepContext.java | 37 +-
.../jstorm/translation/JStormMetricResults.java | 4 +-
.../translation/JStormPipelineTranslator.java | 4 +-
.../jstorm/translation/JStormTimerInternals.java | 3 +
.../jstorm/translation/MetricsReporter.java | 2 +-
runners/local-artifact-service-java/pom.xml | 116 +
.../LocalFileSystemArtifactStagerService.java | 279 +
.../apache/beam/artifact/local/package-info.java | 22 +
.../LocalFileSystemArtifactStagerServiceTest.java | 301 +
runners/pom.xml | 15 +-
runners/reference/job-server/pom.xml | 82 +
.../reference/job/ReferenceRunnerJobServer.java | 77 +
.../reference/job/ReferenceRunnerJobService.java | 79 +
.../beam/runners/reference/job/package-info.java | 23 +
.../job/ReferenceRunnerJobServiceTest.java | 34 +
runners/reference/pom.xml | 39 +
runners/spark/pom.xml | 94 +-
.../runners/spark/SparkNativePipelineVisitor.java | 3 +-
.../org/apache/beam/runners/spark/SparkRunner.java | 14 +-
.../apache/beam/runners/spark/TestSparkRunner.java | 2 +-
.../spark/aggregators/NamedAggregators.java | 93 -
.../apache/beam/runners/spark/io/CreateStream.java | 115 +-
.../beam/runners/spark/io/MicrobatchSource.java | 4 +-
.../beam/runners/spark/io/SourceDStream.java | 20 +-
.../apache/beam/runners/spark/io/SourceRDD.java | 22 +-
.../runners/spark/io/SparkUnboundedSource.java | 8 +-
.../SparkGroupAlsoByWindowViaWindowSet.java | 821 +-
.../spark/stateful/SparkTimerInternals.java | 39 +-
.../runners/spark/stateful/StateSpecFunctions.java | 10 +-
.../runners/spark/translation/BoundedDataset.java | 17 +-
.../beam/runners/spark/translation/Dataset.java | 3 +-
.../spark/translation/EvaluationContext.java | 38 +-
.../spark/translation/MultiDoFnFunction.java | 126 +-
.../spark/translation/SparkAbstractCombineFn.java | 9 +-
.../spark/translation/SparkContextFactory.java | 2 -
.../spark/translation/SparkGlobalCombineFn.java | 5 +-
.../SparkGroupAlsoByWindowViaOutputBufferFn.java | 13 +-
.../spark/translation/SparkKeyedCombineFn.java | 5 +-
.../spark/translation/SparkProcessContext.java | 50 +-
.../spark/translation/SparkRuntimeContext.java | 98 -
.../spark/translation/StorageLevelPTransform.java | 43 -
.../spark/translation/TransformTranslator.java | 209 +-
.../spark/translation/TranslationUtils.java | 78 +
.../streaming/StreamingTransformTranslator.java | 110 +-
.../translation/streaming/UnboundedDataset.java | 27 +-
.../streaming/WatermarkSyncedDStream.java | 149 +
.../runners/spark/util/GlobalWatermarkHolder.java | 307 +-
.../util/SinglePrimitiveOutputPTransform.java | 51 -
.../runners/spark/GlobalWatermarkHolderTest.java | 18 +-
.../beam/runners/spark/SparkPipelineStateTest.java | 4 +-
.../runners/spark/SparkRunnerDebuggerTest.java | 31 +-
.../spark/stateful/SparkStateInternalsTest.java | 66 +
.../spark/translation/SparkRuntimeContextTest.java | 122 -
.../spark/translation/StorageLevelTest.java | 75 -
.../translation/streaming/CreateStreamTest.java | 33 +-
.../streaming/TrackStreamingSourcesTest.java | 14 +-
runners/spark/src/test/resources/log4j.properties | 11 +-
sdks/CONTAINERS.md | 162 +
sdks/common/fn-api/pom.xml | 104 -
.../common/fn-api/src/main/proto/beam_fn_api.proto | 761 --
sdks/common/pom.xml | 39 -
sdks/common/runner-api/pom.xml | 86 -
.../src/main/proto/beam_runner_api.proto | 711 --
sdks/go/BUILD.md | 63 +
sdks/go/cmd/beamctl/cmd/artifact.go | 98 +
sdks/go/cmd/beamctl/cmd/root.go | 56 +
sdks/go/cmd/beamctl/main.go | 31 +
sdks/go/descriptor.xml | 29 +
sdks/go/pkg/beam/artifact/gcsproxy/retrieval.go | 155 +
sdks/go/pkg/beam/artifact/gcsproxy/staging.go | 200 +
sdks/go/pkg/beam/artifact/materialize.go | 240 +
sdks/go/pkg/beam/artifact/materialize_test.go | 238 +
sdks/go/pkg/beam/artifact/server_test.go | 213 +
sdks/go/pkg/beam/artifact/stage.go | 238 +
sdks/go/pkg/beam/artifact/stage_test.go | 98 +
.../beam/model/fnexecution_v1/beam_fn_api.pb.go | 2729 +++++
.../model/fnexecution_v1/beam_provision_api.pb.go | 306 +
sdks/go/pkg/beam/model/gen.go | 22 +
.../model/jobmanagement_v1/beam_artifact_api.pb.go | 690 ++
.../beam/model/jobmanagement_v1/beam_job_api.pb.go | 903 ++
.../beam/model/pipeline_v1/beam_runner_api.pb.go | 3491 +++++++
sdks/go/pkg/beam/model/pipeline_v1/endpoints.pb.go | 160 +
.../model/pipeline_v1/standard_window_fns.pb.go | 120 +
sdks/go/pkg/beam/provision/provision_test.go | 54 +
sdks/go/pkg/beam/provision/provison.go | 80 +
sdks/go/pkg/beam/util/errorx/guarded.go | 47 +
sdks/go/pkg/beam/util/execx/exec.go | 33 +
sdks/go/pkg/beam/util/gcsx/gcs.go | 88 +
sdks/go/pkg/beam/util/grpcx/dial.go | 37 +
sdks/go/pkg/beam/util/grpcx/metadata.go | 55 +
sdks/go/pkg/beam/util/syscallx/syscall.go | 27 +
sdks/go/pkg/beam/util/syscallx/syscall_default.go | 28 +
sdks/go/pkg/beam/util/syscallx/syscall_linux.go | 38 +
sdks/go/pom.xml | 163 +
sdks/java/build-tools/pom.xml | 2 +-
.../src/main/resources/beam/checkstyle.xml | 8 +
.../src/main/resources/beam/findbugs-filter.xml | 74 +-
.../main/resources/docker/file/openjdk7/Dockerfile | 49 +
.../docker/file/openjdk7/docker-entrypoint.sh | 24 +
.../main/resources/docker/file/openjdk8/Dockerfile | 49 +
.../docker/file/openjdk8/docker-entrypoint.sh | 24 +
.../main/resources/docker/git/openjdk8/Dockerfile | 53 +
.../docker/git/openjdk8/docker-entrypoint.sh | 22 +
.../resources/docker/release/python2/Dockerfile | 21 +
sdks/java/container/Dockerfile | 28 +
sdks/java/container/boot.go | 134 +
sdks/java/container/pom.xml | 184 +
sdks/java/core/pom.xml | 186 +-
.../main/java/org/apache/beam/sdk/Pipeline.java | 73 +-
.../java/org/apache/beam/sdk/PipelineResult.java | 14 +-
.../apache/beam/sdk/annotations/Experimental.java | 13 +-
.../apache/beam/sdk/annotations/package-info.java | 4 +
.../apache/beam/sdk/coders/BeamRecordCoder.java | 111 +
.../org/apache/beam/sdk/coders/BooleanCoder.java | 59 +
.../java/org/apache/beam/sdk/coders/Coder.java | 12 +-
.../org/apache/beam/sdk/coders/CoderRegistry.java | 29 +
.../org/apache/beam/sdk/coders/DefaultCoder.java | 19 +-
.../org/apache/beam/sdk/coders/InstantCoder.java | 81 +-
.../apache/beam/sdk/coders/LengthPrefixCoder.java | 3 +-
.../apache/beam/sdk/coders/SerializableCoder.java | 7 +-
.../apache/beam/sdk/coders/ShardedKeyCoder.java | 66 +
.../apache/beam/sdk/coders/StructuredCoder.java | 42 +-
.../java/org/apache/beam/sdk/coders/VoidCoder.java | 12 +-
.../org/apache/beam/sdk/coders/package-info.java | 4 +
.../main/java/org/apache/beam/sdk/io/AvroIO.java | 1216 ++-
.../main/java/org/apache/beam/sdk/io/AvroSink.java | 95 +-
.../java/org/apache/beam/sdk/io/AvroSource.java | 393 +-
.../java/org/apache/beam/sdk/io/AvroUtils.java | 40 +
.../org/apache/beam/sdk/io/BlockBasedSource.java | 31 +-
.../sdk/io/BoundedReadFromUnboundedSource.java | 12 +-
.../org/apache/beam/sdk/io/CompressedSource.java | 310 +-
.../java/org/apache/beam/sdk/io/Compression.java | 228 +
.../beam/sdk/io/ConstantAvroDestination.java | 130 +
.../org/apache/beam/sdk/io/CountingSource.java | 49 +-
.../apache/beam/sdk/io/DefaultFilenamePolicy.java | 378 +-
.../beam/sdk/io/DynamicAvroDestinations.java | 46 +
.../beam/sdk/io/DynamicFileDestinations.java | 150 +
.../java/org/apache/beam/sdk/io/FileBasedSink.java | 833 +-
.../org/apache/beam/sdk/io/FileBasedSource.java | 81 +-
.../main/java/org/apache/beam/sdk/io/FileIO.java | 450 +
.../java/org/apache/beam/sdk/io/FileSystems.java | 69 +-
.../org/apache/beam/sdk/io/GenerateSequence.java | 3 +-
.../org/apache/beam/sdk/io/LocalFileSystem.java | 45 +-
.../org/apache/beam/sdk/io/OffsetBasedSource.java | 22 +-
.../src/main/java/org/apache/beam/sdk/io/Read.java | 27 +-
.../beam/sdk/io/ReadAllViaFileBasedSource.java | 114 +
.../org/apache/beam/sdk/io/ReadableFileCoder.java | 50 +
.../main/java/org/apache/beam/sdk/io/Source.java | 28 +-
.../java/org/apache/beam/sdk/io/TFRecordIO.java | 228 +-
.../main/java/org/apache/beam/sdk/io/TextIO.java | 1054 +-
.../main/java/org/apache/beam/sdk/io/TextSink.java | 22 +-
.../java/org/apache/beam/sdk/io/TextSource.java | 120 +-
.../java/org/apache/beam/sdk/io/WriteFiles.java | 782 +-
.../org/apache/beam/sdk/io/WriteFilesResult.java | 81 +
.../apache/beam/sdk/io/fs/EmptyMatchTreatment.java | 46 +
.../org/apache/beam/sdk/io/fs/MatchResult.java | 5 +-
.../org/apache/beam/sdk/io/fs/MetadataCoder.java | 63 +
.../org/apache/beam/sdk/io/fs/ResourceIdCoder.java | 56 +
.../java/org/apache/beam/sdk/io/package-info.java | 2 +-
.../beam/sdk/io/range/ByteKeyRangeTracker.java | 27 +-
.../org/apache/beam/sdk/io/range/OffsetRange.java | 101 +
.../beam/sdk/io/range/OffsetRangeTracker.java | 25 +-
.../org/apache/beam/sdk/metrics/MetricName.java | 7 +
.../apache/beam/sdk/metrics/MetricsContainer.java | 3 +-
.../org/apache/beam/sdk/metrics/package-info.java | 4 +
.../options/DefaultPipelineOptionsRegistrar.java | 2 +
.../beam/sdk/options/ExperimentalOptions.java | 38 +
.../apache/beam/sdk/options/PipelineOptions.java | 43 +-
.../beam/sdk/options/PipelineOptionsFactory.java | 63 +-
.../beam/sdk/options/PipelineOptionsValidator.java | 34 +-
.../beam/sdk/options/ProxyInvocationHandler.java | 19 +-
.../apache/beam/sdk/options/SdkHarnessOptions.java | 173 +
.../org/apache/beam/sdk/options/ValueProvider.java | 34 +-
.../apache/beam/sdk/options/ValueProviders.java | 25 +-
.../java/org/apache/beam/sdk/package-info.java | 4 +
.../beam/sdk/runners/TransformHierarchy.java | 247 +-
.../org/apache/beam/sdk/runners/package-info.java | 4 +
.../java/org/apache/beam/sdk/state/BagState.java | 6 +
.../org/apache/beam/sdk/state/CombiningState.java | 5 +
.../org/apache/beam/sdk/state/GroupingState.java | 12 +-
.../java/org/apache/beam/sdk/state/MapState.java | 20 +-
.../org/apache/beam/sdk/state/ReadableState.java | 6 +
.../org/apache/beam/sdk/state/ReadableStates.java | 4 +-
.../java/org/apache/beam/sdk/state/SetState.java | 10 +-
.../java/org/apache/beam/sdk/state/StateSpec.java | 53 +
.../java/org/apache/beam/sdk/state/StateSpecs.java | 57 +-
.../org/apache/beam/sdk/state/package-info.java | 4 +
.../apache/beam/sdk/testing/CombineFnTester.java | 147 +
.../beam/sdk/testing/FileChecksumMatcher.java | 41 +-
.../org/apache/beam/sdk/testing/LargeKeys.java | 48 +
.../java/org/apache/beam/sdk/testing/PAssert.java | 79 +-
.../apache/beam/sdk/testing/PaneExtractors.java | 25 +-
.../apache/beam/sdk/testing/SourceTestUtils.java | 23 +-
.../org/apache/beam/sdk/testing/StaticWindows.java | 17 +
.../org/apache/beam/sdk/testing/StreamingIT.java | 4 +
.../apache/beam/sdk/testing/SuccessOrFailure.java | 24 +-
.../org/apache/beam/sdk/testing/TestPipeline.java | 121 +-
.../org/apache/beam/sdk/testing/TestStream.java | 77 +-
.../beam/sdk/testing/UsesCustomWindowMerging.java | 23 +
.../apache/beam/sdk/testing/WindowFnTestUtils.java | 141 +-
.../apache/beam/sdk/testing/WindowSupplier.java | 4 +-
.../org/apache/beam/sdk/testing/package-info.java | 5 +
.../beam/sdk/transforms/ApproximateQuantiles.java | 14 +-
.../beam/sdk/transforms/ApproximateUnique.java | 4 +-
.../org/apache/beam/sdk/transforms/Combine.java | 63 +-
.../org/apache/beam/sdk/transforms/CombineFns.java | 2 +
.../org/apache/beam/sdk/transforms/Contextful.java | 127 +
.../java/org/apache/beam/sdk/transforms/Count.java | 10 +
.../org/apache/beam/sdk/transforms/Create.java | 119 +-
.../org/apache/beam/sdk/transforms/Distinct.java | 82 +-
.../java/org/apache/beam/sdk/transforms/DoFn.java | 66 +-
.../org/apache/beam/sdk/transforms/DoFnTester.java | 48 +-
.../org/apache/beam/sdk/transforms/Filter.java | 26 +-
.../beam/sdk/transforms/FlatMapElements.java | 150 +-
.../org/apache/beam/sdk/transforms/Flatten.java | 22 +-
.../org/apache/beam/sdk/transforms/GroupByKey.java | 12 +-
.../beam/sdk/transforms/GroupIntoBatches.java | 12 +-
.../org/apache/beam/sdk/transforms/Latest.java | 2 +-
.../apache/beam/sdk/transforms/MapElements.java | 77 +-
.../java/org/apache/beam/sdk/transforms/Max.java | 9 +-
.../java/org/apache/beam/sdk/transforms/Min.java | 12 +-
.../org/apache/beam/sdk/transforms/PTransform.java | 28 +-
.../java/org/apache/beam/sdk/transforms/ParDo.java | 110 +-
.../apache/beam/sdk/transforms/Requirements.java | 61 +
.../org/apache/beam/sdk/transforms/Reshuffle.java | 47 +
.../beam/sdk/transforms/SerializableFunctions.java | 50 +
.../java/org/apache/beam/sdk/transforms/Sum.java | 30 +
.../java/org/apache/beam/sdk/transforms/Top.java | 41 +-
.../java/org/apache/beam/sdk/transforms/View.java | 55 +-
.../java/org/apache/beam/sdk/transforms/Watch.java | 1066 ++
.../org/apache/beam/sdk/transforms/WithKeys.java | 12 +-
.../beam/sdk/transforms/display/DisplayData.java | 25 +-
.../beam/sdk/transforms/display/package-info.java | 4 +
.../beam/sdk/transforms/join/CoGbkResult.java | 15 +-
.../sdk/transforms/join/KeyedPCollectionTuple.java | 10 +-
.../beam/sdk/transforms/join/RawUnionValue.java | 8 +-
.../beam/sdk/transforms/join/package-info.java | 4 +
.../apache/beam/sdk/transforms/package-info.java | 4 +
.../reflect/ByteBuddyDoFnInvokerFactory.java | 30 +-
.../reflect/ByteBuddyOnTimerInvokerFactory.java | 73 +-
.../beam/sdk/transforms/reflect/DoFnInvoker.java | 65 +-
.../beam/sdk/transforms/reflect/DoFnInvokers.java | 9 -
.../beam/sdk/transforms/reflect/DoFnSignature.java | 33 +-
.../sdk/transforms/reflect/DoFnSignatures.java | 45 +-
.../transforms/reflect/OnTimerMethodSpecifier.java | 37 +
.../beam/sdk/transforms/reflect/package-info.java | 3 +
.../sdk/transforms/splittabledofn/OffsetRange.java | 77 -
.../splittabledofn/OffsetRangeTracker.java | 16 +-
.../splittabledofn/RestrictionTracker.java | 11 +-
.../transforms/splittabledofn/package-info.java | 4 +
.../sdk/transforms/windowing/BoundedWindow.java | 20 +-
.../sdk/transforms/windowing/CalendarWindows.java | 36 +
.../sdk/transforms/windowing/FixedWindows.java | 11 +
.../sdk/transforms/windowing/GlobalWindows.java | 22 +-
.../windowing/IncompatibleWindowException.java | 38 +
.../sdk/transforms/windowing/InvalidWindows.java | 11 +
.../windowing/MergeOverlappingIntervalWindows.java | 4 +-
.../transforms/windowing/PartitioningWindowFn.java | 5 +
.../beam/sdk/transforms/windowing/Sessions.java | 11 +
.../sdk/transforms/windowing/SlidingWindows.java | 16 +
.../beam/sdk/transforms/windowing/Trigger.java | 20 +-
.../beam/sdk/transforms/windowing/Window.java | 47 +-
.../beam/sdk/transforms/windowing/WindowFn.java | 32 +
.../sdk/transforms/windowing/package-info.java | 4 +
.../java/org/apache/beam/sdk/util/ApiSurface.java | 2 +
.../java/org/apache/beam/sdk/util/ClassPath.java | 8 +-
.../java/org/apache/beam/sdk/util/CoderUtils.java | 2 +-
.../java/org/apache/beam/sdk/util/DoFnInfo.java | 104 +
.../org/apache/beam/sdk/util/IdentityWindowFn.java | 17 +-
.../apache/beam/sdk/util/MutationDetectors.java | 79 +-
.../beam/sdk/util/SerializableThrowable.java | 49 +
.../apache/beam/sdk/util/SerializableUtils.java | 69 +-
.../org/apache/beam/sdk/values/BeamRecord.java | 319 +
.../org/apache/beam/sdk/values/BeamRecordType.java | 96 +
.../org/apache/beam/sdk/values/PCollection.java | 21 +-
.../apache/beam/sdk/values/PCollectionTuple.java | 10 +-
.../apache/beam/sdk/values/PCollectionViews.java | 37 +
.../java/org/apache/beam/sdk/values/PValue.java | 4 +-
.../org/apache/beam/sdk/values/PValueBase.java | 12 -
.../org/apache/beam/sdk/values/ShardedKey.java | 65 +
.../org/apache/beam/sdk/values/TypeDescriptor.java | 64 +-
.../apache/beam/sdk/values/TypeDescriptors.java | 127 +-
.../apache/beam/sdk/values/WindowingStrategy.java | 46 +-
.../java/org/apache/beam/sdk/PipelineTest.java | 51 +-
.../test/java/org/apache/beam/sdk/TestUtils.java | 88 -
.../org/apache/beam/sdk/coders/AvroCoderTest.java | 39 +-
.../apache/beam/sdk/coders/CoderRegistryTest.java | 69 +
.../apache/beam/sdk/coders/CommonCoderTest.java | 4 +-
.../apache/beam/sdk/coders/DefaultCoderTest.java | 3 +-
.../org/apache/beam/sdk/coders/VoidCoderTest.java | 11 +-
.../java/org/apache/beam/sdk/io/AvroIOTest.java | 917 +-
.../apache/beam/sdk/io/AvroIOTransformTest.java | 324 -
.../org/apache/beam/sdk/io/AvroSourceTest.java | 59 +-
.../apache/beam/sdk/io/CompressedSourceTest.java | 138 +-
.../beam/sdk/io/DefaultFilenamePolicyTest.java | 108 +-
.../sdk/io/DrunkWritableByteChannelFactory.java | 2 +-
.../org/apache/beam/sdk/io/FileBasedSinkTest.java | 202 +-
.../apache/beam/sdk/io/FileBasedSourceTest.java | 56 +-
.../java/org/apache/beam/sdk/io/FileIOTest.java | 313 +
.../org/apache/beam/sdk/io/FileSystemsTest.java | 15 +-
.../apache/beam/sdk/io/LocalFileSystemTest.java | 49 +
.../apache/beam/sdk/io/OffsetBasedSourceTest.java | 5 +-
.../test/java/org/apache/beam/sdk/io/ReadTest.java | 10 +-
.../java/org/apache/beam/sdk/io/SimpleSink.java | 77 +-
.../org/apache/beam/sdk/io/TFRecordIOTest.java | 35 +-
.../org/apache/beam/sdk/io/TextIOReadTest.java | 860 ++
.../java/org/apache/beam/sdk/io/TextIOTest.java | 1104 +-
.../org/apache/beam/sdk/io/TextIOWriteTest.java | 643 ++
.../org/apache/beam/sdk/io/WriteFilesTest.java | 479 +-
.../beam/sdk/io/range/ByteKeyRangeTrackerTest.java | 23 +
.../beam/sdk/metrics/MetricResultsMatchers.java | 2 +-
.../org/apache/beam/sdk/metrics/MetricsTest.java | 28 +
.../sdk/options/PipelineOptionsFactoryTest.java | 39 +
.../beam/sdk/options/PipelineOptionsTest.java | 11 +
.../sdk/options/PipelineOptionsValidatorTest.java | 44 +
.../sdk/options/ProxyInvocationHandlerTest.java | 23 +-
.../beam/sdk/options/SdkHarnessOptionsTest.java | 76 +
.../apache/beam/sdk/options/ValueProviderTest.java | 38 +-
.../beam/sdk/runners/TransformHierarchyTest.java | 216 +-
.../apache/beam/sdk/runners/TransformTreeTest.java | 19 +-
.../sdk/runners/dataflow/TestCountingSource.java | 5 +-
.../beam/sdk/testing/CombineFnTesterTest.java | 276 +
.../sdk/testing/InterceptingUrlClassLoader.java | 57 +
.../org/apache/beam/sdk/testing/PAssertTest.java | 41 +-
.../beam/sdk/testing/PCollectionViewTesting.java | 8 +
.../beam/sdk/testing/PaneExtractorsTest.java | 7 +-
.../apache/beam/sdk/testing/StaticWindowsTest.java | 12 +
.../apache/beam/sdk/testing/TestPipelineTest.java | 75 +-
.../sdk/transforms/ApproximateQuantilesTest.java | 530 +-
.../apache/beam/sdk/transforms/CombineTest.java | 377 +-
.../org/apache/beam/sdk/transforms/CreateTest.java | 70 +-
.../apache/beam/sdk/transforms/DistinctTest.java | 130 +-
.../apache/beam/sdk/transforms/DoFnTesterTest.java | 32 +
.../beam/sdk/transforms/FlatMapElementsTest.java | 35 +-
.../apache/beam/sdk/transforms/FlattenTest.java | 37 +-
.../apache/beam/sdk/transforms/GroupByKeyTest.java | 280 +-
.../beam/sdk/transforms/GroupIntoBatchesTest.java | 4 +-
.../beam/sdk/transforms/MapElementsTest.java | 42 +-
.../org/apache/beam/sdk/transforms/MaxTest.java | 10 +-
.../org/apache/beam/sdk/transforms/MeanTest.java | 4 +-
.../org/apache/beam/sdk/transforms/MinTest.java | 10 +-
.../org/apache/beam/sdk/transforms/ParDoTest.java | 322 +-
.../beam/sdk/transforms/SplittableDoFnTest.java | 155 +-
.../org/apache/beam/sdk/transforms/SumTest.java | 8 +-
.../org/apache/beam/sdk/transforms/TopTest.java | 10 +-
.../org/apache/beam/sdk/transforms/ViewTest.java | 11 +-
.../org/apache/beam/sdk/transforms/WatchTest.java | 797 ++
.../apache/beam/sdk/transforms/WithKeysTest.java | 30 +-
.../sdk/transforms/reflect/DoFnInvokersTest.java | 103 +-
.../reflect/DoFnSignaturesProcessElementTest.java | 2 +-
.../reflect/DoFnSignaturesSplittableDoFnTest.java | 83 +-
.../sdk/transforms/reflect/DoFnSignaturesTest.java | 14 +
.../splittabledofn/OffsetRangeTrackerTest.java | 1 +
.../transforms/windowing/CalendarWindowsTest.java | 24 +
.../sdk/transforms/windowing/FixedWindowsTest.java | 7 +
.../sdk/transforms/windowing/SessionsTest.java | 14 +
.../transforms/windowing/SlidingWindowsTest.java | 41 +-
.../beam/sdk/transforms/windowing/WindowTest.java | 225 +
.../beam/sdk/util/IdentitySideInputWindowFn.java | 4 +
.../beam/sdk/util/MutationDetectorsTest.java | 56 +
.../beam/sdk/util/SerializableUtilsTest.java | 60 +
.../beam/sdk/values/PCollectionTupleTest.java | 7 +-
.../beam/sdk/values/TypeDescriptorsTest.java | 60 +
.../extensions/google-cloud-platform-core/pom.xml | 2 +-
.../sdk/extensions/gcp/storage/GcsFileSystem.java | 5 +-
.../java/org/apache/beam/sdk/util/GcsUtil.java | 16 +-
.../beam/sdk/util/RetryHttpRequestInitializer.java | 147 +-
.../sdk/extensions/gcp/GcpCoreApiSurfaceTest.java | 48 +-
.../java/org/apache/beam/sdk/util/GcsUtilTest.java | 61 +-
.../sdk/util/RetryHttpRequestInitializerTest.java | 31 +-
sdks/java/extensions/jackson/pom.xml | 2 +-
sdks/java/extensions/join-library/pom.xml | 2 +-
.../beam/sdk/extensions/joinlibrary/Join.java | 65 +-
.../extensions/joinlibrary/OuterFullJoinTest.java | 179 +
sdks/java/extensions/pom.xml | 4 +-
sdks/java/extensions/protobuf/pom.xml | 2 +-
sdks/java/extensions/sketching/pom.xml | 104 +
.../extensions/sketching/ApproximateDistinct.java | 573 ++
.../sdk/extensions/sketching/package-info.java | 22 +
.../sketching/ApproximateDistinctTest.java | 209 +
sdks/java/extensions/sorter/pom.xml | 8 +-
.../beam/sdk/extensions/sorter/SortValues.java | 20 +-
sdks/java/extensions/sql/NOTICE | 45 +
sdks/java/extensions/sql/pom.xml | 425 +
.../extensions/sql/src/main/codegen/config.fmpp | 23 +
.../sql/src/main/codegen/data/Parser.tdd | 75 +
.../sql/src/main/codegen/includes/license.ftl | 17 +
.../sql/src/main/codegen/includes/parserImpls.ftl | 89 +
.../beam/sdk/extensions/sql/BeamRecordSqlType.java | 186 +
.../apache/beam/sdk/extensions/sql/BeamSql.java | 250 +
.../apache/beam/sdk/extensions/sql/BeamSqlCli.java | 115 +
.../sdk/extensions/sql/BeamSqlRecordHelper.java | 217 +
.../beam/sdk/extensions/sql/BeamSqlTable.java | 54 +
.../apache/beam/sdk/extensions/sql/BeamSqlUdf.java | 43 +
.../sdk/extensions/sql/example/BeamSqlExample.java | 104 +
.../sdk/extensions/sql/example/package-info.java | 23 +
.../beam/sdk/extensions/sql/impl/BeamSqlEnv.java | 136 +
.../interpreter/BeamSqlExpressionExecutor.java | 44 +
.../sql/impl/interpreter/BeamSqlFnExecutor.java | 463 +
.../operator/BeamSqlCaseExpression.java | 64 +
.../operator/BeamSqlCastExpression.java | 132 +
.../interpreter/operator/BeamSqlExpression.java | 79 +
.../operator/BeamSqlInputRefExpression.java | 48 +
.../interpreter/operator/BeamSqlPrimitive.java | 157 +
.../interpreter/operator/BeamSqlUdfExpression.java | 92 +
.../operator/BeamSqlWindowEndExpression.java | 48 +
.../operator/BeamSqlWindowExpression.java | 51 +
.../operator/BeamSqlWindowStartExpression.java | 49 +
.../sql/impl/interpreter/operator/UdafImpl.java | 87 +
.../arithmetic/BeamSqlArithmeticExpression.java | 124 +
.../arithmetic/BeamSqlDivideExpression.java | 37 +
.../arithmetic/BeamSqlMinusExpression.java | 36 +
.../operator/arithmetic/BeamSqlModExpression.java | 36 +
.../arithmetic/BeamSqlMultiplyExpression.java | 36 +
.../operator/arithmetic/BeamSqlPlusExpression.java | 36 +
.../operator/arithmetic/package-info.java | 22 +
.../comparison/BeamSqlCompareExpression.java | 97 +
.../comparison/BeamSqlEqualsExpression.java | 49 +
.../comparison/BeamSqlGreaterThanExpression.java | 49 +
.../BeamSqlGreaterThanOrEqualsExpression.java | 49 +
.../comparison/BeamSqlIsNotNullExpression.java | 54 +
.../comparison/BeamSqlIsNullExpression.java | 54 +
.../comparison/BeamSqlLessThanExpression.java | 49 +
.../BeamSqlLessThanOrEqualsExpression.java | 49 +
.../comparison/BeamSqlNotEqualsExpression.java | 49 +
.../operator/comparison/package-info.java | 22 +
.../date/BeamSqlCurrentDateExpression.java | 45 +
.../date/BeamSqlCurrentTimeExpression.java | 53 +
.../date/BeamSqlCurrentTimestampExpression.java | 49 +
.../operator/date/BeamSqlDateCeilExpression.java | 55 +
.../operator/date/BeamSqlDateFloorExpression.java | 55 +
.../date/BeamSqlDatetimeMinusExpression.java | 107 +
.../date/BeamSqlDatetimePlusExpression.java | 129 +
.../operator/date/BeamSqlExtractExpression.java | 102 +
.../date/BeamSqlIntervalMultiplyExpression.java | 103 +
.../BeamSqlTimestampMinusIntervalExpression.java | 83 +
.../BeamSqlTimestampMinusTimestampExpression.java | 97 +
.../interpreter/operator/date/TimeUnitUtils.java | 54 +
.../interpreter/operator/date/package-info.java | 22 +
.../operator/logical/BeamSqlAndExpression.java | 48 +
.../operator/logical/BeamSqlLogicalExpression.java | 46 +
.../operator/logical/BeamSqlNotExpression.java | 54 +
.../operator/logical/BeamSqlOrExpression.java | 48 +
.../interpreter/operator/logical/package-info.java | 22 +
.../operator/math/BeamSqlAbsExpression.java | 74 +
.../operator/math/BeamSqlAcosExpression.java | 40 +
.../operator/math/BeamSqlAsinExpression.java | 40 +
.../operator/math/BeamSqlAtan2Expression.java | 42 +
.../operator/math/BeamSqlAtanExpression.java | 40 +
.../operator/math/BeamSqlCeilExpression.java | 45 +
.../operator/math/BeamSqlCosExpression.java | 40 +
.../operator/math/BeamSqlCotExpression.java | 40 +
.../operator/math/BeamSqlDegreesExpression.java | 40 +
.../operator/math/BeamSqlExpExpression.java | 40 +
.../operator/math/BeamSqlFloorExpression.java | 45 +
.../operator/math/BeamSqlLnExpression.java | 40 +
.../operator/math/BeamSqlLogExpression.java | 40 +
.../operator/math/BeamSqlMathBinaryExpression.java | 65 +
.../operator/math/BeamSqlMathUnaryExpression.java | 60 +
.../operator/math/BeamSqlPiExpression.java | 43 +
.../operator/math/BeamSqlPowerExpression.java | 44 +
.../operator/math/BeamSqlRadiansExpression.java | 40 +
.../operator/math/BeamSqlRandExpression.java | 55 +
.../math/BeamSqlRandIntegerExpression.java | 59 +
.../operator/math/BeamSqlRoundExpression.java | 107 +
.../operator/math/BeamSqlSignExpression.java | 72 +
.../operator/math/BeamSqlSinExpression.java | 40 +
.../operator/math/BeamSqlTanExpression.java | 40 +
.../operator/math/BeamSqlTruncateExpression.java | 75 +
.../interpreter/operator/math/package-info.java | 22 +
.../impl/interpreter/operator/package-info.java | 22 +
.../reinterpret/BeamSqlReinterpretExpression.java | 58 +
.../DatetimeReinterpretConversions.java | 59 +
.../reinterpret/IntegerReinterpretConversions.java | 44 +
.../reinterpret/ReinterpretConversion.java | 114 +
.../operator/reinterpret/Reinterpreter.java | 101 +
.../operator/reinterpret/package-info.java | 22 +
.../string/BeamSqlCharLengthExpression.java | 40 +
.../operator/string/BeamSqlConcatExpression.java | 63 +
.../operator/string/BeamSqlInitCapExpression.java | 56 +
.../operator/string/BeamSqlLowerExpression.java | 40 +
.../operator/string/BeamSqlOverlayExpression.java | 77 +
.../operator/string/BeamSqlPositionExpression.java | 73 +
.../string/BeamSqlStringUnaryExpression.java | 44 +
.../string/BeamSqlSubstringExpression.java | 83 +
.../operator/string/BeamSqlTrimExpression.java | 102 +
.../operator/string/BeamSqlUpperExpression.java | 40 +
.../interpreter/operator/string/package-info.java | 22 +
.../sql/impl/interpreter/package-info.java | 22 +
.../beam/sdk/extensions/sql/impl/package-info.java | 22 +
.../extensions/sql/impl/parser/BeamSqlParser.java | 50 +
.../sql/impl/parser/ColumnConstraint.java | 42 +
.../sql/impl/parser/ColumnDefinition.java | 56 +
.../extensions/sql/impl/parser/ParserUtils.java | 64 +
.../extensions/sql/impl/parser/SqlCreateTable.java | 141 +
.../extensions/sql/impl/parser/SqlDDLKeywords.java | 27 +
.../extensions/sql/impl/parser/UnparseUtil.java | 59 +
.../extensions/sql/impl/parser/package-info.java | 22 +
.../sql/impl/planner/BeamQueryPlanner.java | 168 +
.../sql/impl/planner/BeamRelDataTypeSystem.java | 40 +
.../extensions/sql/impl/planner/BeamRuleSets.java | 75 +
.../extensions/sql/impl/planner/package-info.java | 24 +
.../sql/impl/rel/BeamAggregationRel.java | 182 +
.../sdk/extensions/sql/impl/rel/BeamFilterRel.java | 69 +
.../sdk/extensions/sql/impl/rel/BeamIOSinkRel.java | 75 +
.../extensions/sql/impl/rel/BeamIOSourceRel.java | 62 +
.../extensions/sql/impl/rel/BeamIntersectRel.java | 58 +
.../sdk/extensions/sql/impl/rel/BeamJoinRel.java | 298 +
.../sql/impl/rel/BeamLogicalConvention.java | 72 +
.../sdk/extensions/sql/impl/rel/BeamMinusRel.java | 56 +
.../extensions/sql/impl/rel/BeamProjectRel.java | 80 +
.../sdk/extensions/sql/impl/rel/BeamRelNode.java | 39 +
.../sql/impl/rel/BeamSetOperatorRelBase.java | 98 +
.../sdk/extensions/sql/impl/rel/BeamSortRel.java | 235 +
.../extensions/sql/impl/rel/BeamSqlRelUtils.java | 72 +
.../sdk/extensions/sql/impl/rel/BeamUnionRel.java | 88 +
.../sdk/extensions/sql/impl/rel/BeamValuesRel.java | 79 +
.../sdk/extensions/sql/impl/rel/package-info.java | 23 +
.../sql/impl/rule/BeamAggregationRule.java | 162 +
.../extensions/sql/impl/rule/BeamFilterRule.java | 49 +
.../extensions/sql/impl/rule/BeamIOSinkRule.java | 81 +
.../extensions/sql/impl/rule/BeamIOSourceRule.java | 49 +
.../sql/impl/rule/BeamIntersectRule.java | 50 +
.../sdk/extensions/sql/impl/rule/BeamJoinRule.java | 53 +
.../extensions/sql/impl/rule/BeamMinusRule.java | 50 +
.../extensions/sql/impl/rule/BeamProjectRule.java | 50 +
.../sdk/extensions/sql/impl/rule/BeamSortRule.java | 51 +
.../extensions/sql/impl/rule/BeamUnionRule.java | 50 +
.../extensions/sql/impl/rule/BeamValuesRule.java | 48 +
.../sdk/extensions/sql/impl/rule/package-info.java | 23 +
.../extensions/sql/impl/schema/BaseBeamTable.java | 36 +
.../sdk/extensions/sql/impl/schema/BeamIOType.java | 28 +
.../sql/impl/schema/BeamPCollectionTable.java | 63 +
.../extensions/sql/impl/schema/BeamTableUtils.java | 118 +
.../extensions/sql/impl/schema/package-info.java | 22 +
.../impl/transform/BeamAggregationTransforms.java | 311 +
.../impl/transform/BeamBuiltinAggregations.java | 557 ++
.../sql/impl/transform/BeamJoinTransforms.java | 161 +
.../impl/transform/BeamSetOperatorsTransforms.java | 111 +
.../sql/impl/transform/BeamSqlFilterFn.java | 63 +
.../impl/transform/BeamSqlOutputToConsoleFn.java | 41 +
.../sql/impl/transform/BeamSqlProjectFn.java | 72 +
.../sql/impl/transform/package-info.java | 22 +
.../extensions/sql/impl/utils/CalciteUtils.java | 113 +
.../extensions/sql/impl/utils/SqlTypeUtils.java | 63 +
.../extensions/sql/impl/utils/package-info.java | 22 +
.../beam/sdk/extensions/sql/meta/Column.java | 51 +
.../apache/beam/sdk/extensions/sql/meta/Table.java | 69 +
.../beam/sdk/extensions/sql/meta/package-info.java | 22 +
.../extensions/sql/meta/provider/MetaUtils.java | 40 +
.../sql/meta/provider/TableProvider.java | 61 +
.../sql/meta/provider/kafka/BeamKafkaCSVTable.java | 111 +
.../sql/meta/provider/kafka/BeamKafkaTable.java | 115 +
.../meta/provider/kafka/KafkaTableProvider.java | 82 +
.../sql/meta/provider/kafka/package-info.java | 22 +
.../extensions/sql/meta/provider/package-info.java | 22 +
.../sql/meta/provider/text/BeamTextCSVTable.java | 80 +
.../provider/text/BeamTextCSVTableIOReader.java | 59 +
.../provider/text/BeamTextCSVTableIOWriter.java | 59 +
.../sql/meta/provider/text/BeamTextTable.java | 41 +
.../sql/meta/provider/text/TextTableProvider.java | 83 +
.../sql/meta/provider/text/package-info.java | 22 +
.../sql/meta/store/InMemoryMetaStore.java | 113 +
.../sdk/extensions/sql/meta/store/MetaStore.java | 56 +
.../extensions/sql/meta/store/package-info.java | 22 +
.../beam/sdk/extensions/sql/package-info.java | 22 +
.../sql/src/main/resources/log4j.properties | 23 +
...ckaged.org.codehaus.commons.compiler.properties | 18 +
.../sdk/extensions/sql/BeamSqlApiSurfaceTest.java | 62 +
.../beam/sdk/extensions/sql/BeamSqlCliTest.java | 75 +
.../extensions/sql/BeamSqlDslAggregationTest.java | 400 +
.../beam/sdk/extensions/sql/BeamSqlDslBase.java | 136 +
.../sdk/extensions/sql/BeamSqlDslFilterTest.java | 155 +
.../sdk/extensions/sql/BeamSqlDslJoinTest.java | 188 +
.../sdk/extensions/sql/BeamSqlDslProjectTest.java | 227 +
.../sdk/extensions/sql/BeamSqlDslUdfUdafTest.java | 139 +
.../apache/beam/sdk/extensions/sql/TestUtils.java | 190 +
.../impl/interpreter/BeamSqlFnExecutorTest.java | 460 +
.../interpreter/BeamSqlFnExecutorTestBase.java | 88 +
.../operator/BeamNullExperssionTest.java | 55 +
.../operator/BeamSqlAndOrExpressionTest.java | 61 +
.../operator/BeamSqlCaseExpressionTest.java | 93 +
.../operator/BeamSqlCastExpressionTest.java | 129 +
.../operator/BeamSqlCompareExpressionTest.java | 115 +
.../operator/BeamSqlInputRefExpressionTest.java | 57 +
.../interpreter/operator/BeamSqlPrimitiveTest.java | 59 +
.../operator/BeamSqlReinterpretExpressionTest.java | 133 +
.../operator/BeamSqlUdfExpressionTest.java | 51 +
.../BeamSqlArithmeticExpressionTest.java | 237 +
.../date/BeamSqlCurrentDateExpressionTest.java | 38 +
.../date/BeamSqlCurrentTimeExpressionTest.java | 39 +
.../BeamSqlCurrentTimestampExpressionTest.java | 39 +
.../date/BeamSqlDateCeilExpressionTest.java | 50 +
.../date/BeamSqlDateExpressionTestBase.java | 52 +
.../date/BeamSqlDateFloorExpressionTest.java | 49 +
.../date/BeamSqlDatetimeMinusExpressionTest.java | 133 +
.../date/BeamSqlDatetimePlusExpressionTest.java | 155 +
.../date/BeamSqlExtractExpressionTest.java | 103 +
.../BeamSqlIntervalMultiplyExpressionTest.java | 107 +
...eamSqlTimestampMinusIntervalExpressionTest.java | 163 +
...amSqlTimestampMinusTimestampExpressionTest.java | 233 +
.../operator/date/TimeUnitUtilsTest.java | 54 +
.../operator/logical/BeamSqlNotExpressionTest.java | 47 +
.../math/BeamSqlMathBinaryExpressionTest.java | 215 +
.../math/BeamSqlMathUnaryExpressionTest.java | 312 +
.../DatetimeReinterpretConversionsTest.java | 73 +
.../IntegerReinterpretConversionsTest.java | 81 +
.../reinterpret/ReinterpretConversionTest.java | 116 +
.../operator/reinterpret/ReinterpreterTest.java | 155 +
.../string/BeamSqlCharLengthExpressionTest.java | 44 +
.../string/BeamSqlConcatExpressionTest.java | 66 +
.../string/BeamSqlInitCapExpressionTest.java | 54 +
.../string/BeamSqlLowerExpressionTest.java | 44 +
.../string/BeamSqlOverlayExpressionTest.java | 87 +
.../string/BeamSqlPositionExpressionTest.java | 84 +
.../string/BeamSqlStringUnaryExpressionTest.java | 52 +
.../string/BeamSqlSubstringExpressionTest.java | 101 +
.../operator/string/BeamSqlTrimExpressionTest.java | 103 +
.../string/BeamSqlUpperExpressionTest.java | 44 +
.../sql/impl/parser/BeamSqlParserTest.java | 167 +
.../sdk/extensions/sql/impl/rel/BaseRelTest.java | 34 +
.../sql/impl/rel/BeamIntersectRelTest.java | 118 +
.../impl/rel/BeamJoinRelBoundedVsBoundedTest.java | 203 +
.../rel/BeamJoinRelUnboundedVsBoundedTest.java | 240 +
.../rel/BeamJoinRelUnboundedVsUnboundedTest.java | 218 +
.../extensions/sql/impl/rel/BeamMinusRelTest.java | 117 +
.../sql/impl/rel/BeamSetOperatorRelBaseTest.java | 105 +
.../extensions/sql/impl/rel/BeamSortRelTest.java | 257 +
.../extensions/sql/impl/rel/BeamUnionRelTest.java | 103 +
.../extensions/sql/impl/rel/BeamValuesRelTest.java | 104 +
.../sdk/extensions/sql/impl/rel/CheckSize.java | 41 +
.../sql/impl/schema/BeamSqlRowCoderTest.java | 77 +
.../transform/BeamAggregationTransformTest.java | 453 +
.../schema/transform/BeamTransformBaseTest.java | 97 +
.../sql/impl/utils/SqlTypeUtilsTest.java | 76 +
.../BeamSqlArithmeticOperatorsIntegrationTest.java | 165 +
...BeamSqlBuiltinFunctionsIntegrationTestBase.java | 168 +
.../BeamSqlComparisonOperatorsIntegrationTest.java | 329 +
...BeamSqlConditionalFunctionsIntegrationTest.java | 60 +
.../BeamSqlDateFunctionsIntegrationTest.java | 195 +
.../BeamSqlLogicalFunctionsIntegrationTest.java | 43 +
.../BeamSqlMathFunctionsIntegrationTest.java | 351 +
.../BeamSqlStringFunctionsIntegrationTest.java | 51 +
.../meta/provider/kafka/BeamKafkaCSVTableTest.java | 107 +
.../provider/kafka/KafkaTableProviderTest.java | 76 +
.../meta/provider/text/BeamTextCSVTableTest.java | 176 +
.../meta/provider/text/TextTableProviderTest.java | 87 +
.../sql/meta/store/InMemoryMetaStoreTest.java | 185 +
.../extensions/sql/mock/MockedBoundedTable.java | 134 +
.../beam/sdk/extensions/sql/mock/MockedTable.java | 42 +
.../extensions/sql/mock/MockedUnboundedTable.java | 110 +
sdks/java/fn-execution/pom.xml | 115 +
.../harness/channel/ManagedChannelFactory.java | 82 +
.../beam/harness/channel/SocketAddressFactory.java | 64 +
.../apache/beam/harness/channel/package-info.java | 22 +
.../harness/channel/ManagedChannelFactoryTest.java | 71 +
.../harness/channel/SocketAddressFactoryTest.java | 56 +
.../org/apache/beam/harness/test/Consumer.java | 26 +
.../org/apache/beam/harness/test/Supplier.java | 26 +
.../apache/beam/harness/test/TestExecutors.java | 93 +
.../beam/harness/test/TestExecutorsTest.java | 175 +
.../org/apache/beam/harness/test/TestStreams.java | 185 +
.../apache/beam/harness/test/TestStreamsTest.java | 109 +
sdks/java/harness/pom.xml | 133 +-
.../beam/fn/harness/BeamFnDataReadRunner.java | 171 +
.../beam/fn/harness/BeamFnDataWriteRunner.java | 156 +
.../beam/fn/harness/BoundedSourceRunner.java | 165 +
.../apache/beam/fn/harness/FnApiDoFnRunner.java | 918 ++
.../java/org/apache/beam/fn/harness/FnHarness.java | 64 +-
.../org/apache/beam/fn/harness/IdGenerator.java | 33 +
.../beam/fn/harness/PTransformRunnerFactory.java | 84 +
.../fn/harness/channel/ManagedChannelFactory.java | 80 -
.../fn/harness/channel/SocketAddressFactory.java | 64 -
.../beam/fn/harness/channel/package-info.java | 22 -
.../fn/harness/control/BeamFnControlClient.java | 18 +-
.../fn/harness/control/ProcessBundleHandler.java | 455 +-
.../beam/fn/harness/control/RegisterHandler.java | 18 +-
.../data/BeamFnDataBufferingOutboundObserver.java | 8 +-
.../beam/fn/harness/data/BeamFnDataClient.java | 7 +-
.../beam/fn/harness/data/BeamFnDataGrpcClient.java | 24 +-
.../fn/harness/data/BeamFnDataGrpcMultiplexer.java | 33 +-
.../fn/harness/data/BeamFnDataInboundObserver.java | 4 +-
.../beam/fn/harness/fake/FakeStepContext.java | 70 -
.../apache/beam/fn/harness/fake/package-info.java | 22 -
.../beam/fn/harness/fn/ThrowingBiConsumer.java | 33 +
.../fn/harness/logging/BeamFnLoggingClient.java | 238 +-
.../apache/beam/fn/harness/state/BagUserState.java | 121 +
.../beam/fn/harness/state/BeamFnStateClient.java | 39 +
.../harness/state/BeamFnStateGrpcClientCache.java | 173 +
.../state/LazyCachingIteratorToIterable.java | 72 +
.../fn/harness/state/StateFetchingIterators.java | 126 +
.../apache/beam/fn/harness/state/package-info.java | 22 +
.../fn/harness/stream/BufferingStreamObserver.java | 16 +-
.../apache/beam/fn/harness/stream/DataStreams.java | 229 +
.../fn/harness/stream/StreamObserverFactory.java | 4 +-
.../beam/runners/core/BeamFnDataReadRunner.java | 113 -
.../beam/runners/core/BeamFnDataWriteRunner.java | 96 -
.../beam/runners/core/BoundedSourceRunner.java | 105 -
.../org/apache/beam/runners/core/package-info.java | 22 -
.../beam/fn/harness/BeamFnDataReadRunnerTest.java | 280 +
.../beam/fn/harness/BeamFnDataWriteRunnerTest.java | 264 +
.../beam/fn/harness/BoundedSourceRunnerTest.java | 184 +
.../beam/fn/harness/FnApiDoFnRunnerTest.java | 408 +
.../org/apache/beam/fn/harness/FnHarnessTest.java | 25 +-
.../apache/beam/fn/harness/IdGeneratorTest.java | 40 +
.../harness/channel/ManagedChannelFactoryTest.java | 74 -
.../harness/channel/SocketAddressFactoryTest.java | 56 -
.../harness/control/BeamFnControlClientTest.java | 13 +-
.../harness/control/ProcessBundleHandlerTest.java | 777 +-
.../fn/harness/control/RegisterHandlerTest.java | 34 +-
.../BeamFnDataBufferingOutboundObserverTest.java | 4 +-
.../fn/harness/data/BeamFnDataGrpcClientTest.java | 30 +-
.../data/BeamFnDataGrpcMultiplexerTest.java | 9 +-
.../data/BeamFnDataInboundObserverTest.java | 2 +-
.../harness/logging/BeamFnLoggingClientTest.java | 136 +-
.../beam/fn/harness/state/BagUserStateTest.java | 106 +
.../state/BeamFnStateGrpcClientCacheTest.java | 234 +
.../fn/harness/state/FakeBeamFnStateClient.java | 110 +
.../state/LazyCachingIteratorToIterableTest.java | 76 +
.../harness/state/StateFetchingIteratorsTest.java | 99 +
.../stream/BufferingStreamObserverTest.java | 12 +-
.../beam/fn/harness/stream/DataStreamsTest.java | 167 +
.../harness/stream/DirectStreamObserverTest.java | 8 +-
.../apache/beam/fn/harness/test/TestExecutors.java | 85 -
.../beam/fn/harness/test/TestExecutorsTest.java | 160 -
.../apache/beam/fn/harness/test/TestStreams.java | 162 -
.../beam/fn/harness/test/TestStreamsTest.java | 84 -
.../runners/core/BeamFnDataReadRunnerTest.java | 189 -
.../runners/core/BeamFnDataWriteRunnerTest.java | 157 -
.../beam/runners/core/BoundedSourceRunnerTest.java | 113 -
sdks/java/io/amqp/pom.xml | 123 +
.../java/org/apache/beam/sdk/io/amqp/AmqpIO.java | 380 +
.../apache/beam/sdk/io/amqp/AmqpMessageCoder.java | 79 +
.../io/amqp/AmqpMessageCoderProviderRegistrar.java | 44 +
.../org/apache/beam/sdk/io/amqp/package-info.java | 22 +
.../org/apache/beam/sdk/io/amqp/AmqpIOTest.java | 124 +
.../beam/sdk/io/amqp/AmqpMessageCoderTest.java | 89 +
sdks/java/io/cassandra/pom.xml | 113 +
.../apache/beam/sdk/io/cassandra/CassandraIO.java | 486 +
.../beam/sdk/io/cassandra/CassandraService.java | 66 +
.../sdk/io/cassandra/CassandraServiceImpl.java | 398 +
.../apache/beam/sdk/io/cassandra/package-info.java | 22 +
.../beam/sdk/io/cassandra/CassandraIOIT.java | 254 +
.../beam/sdk/io/cassandra/CassandraIOTest.java | 279 +
.../sdk/io/cassandra/CassandraServiceImplTest.java | 138 +
.../sdk/io/cassandra/CassandraTestDataSet.java | 153 +
sdks/java/io/common/pom.xml | 12 +-
.../beam/sdk/io/common/IOTestPipelineOptions.java | 20 +-
.../org/apache/beam/sdk/io/common/TestRow.java | 114 +
.../elasticsearch-tests-2/pom.xml | 60 +
.../src/test/contrib/create_elk_container.sh | 0
.../sdk/io/elasticsearch/ElasticsearchIOIT.java | 123 +
.../sdk/io/elasticsearch/ElasticsearchIOTest.java | 185 +
.../elasticsearch-tests-5/pom.xml | 124 +
.../src/test/contrib/create_elk_container.sh | 0
.../sdk/io/elasticsearch/ElasticsearchIOIT.java | 122 +
.../sdk/io/elasticsearch/ElasticsearchIOTest.java | 185 +
.../java/org/elasticsearch/bootstrap/JarHell.java | 39 +
.../elasticsearch-tests-common/pom.xml | 77 +
.../io/elasticsearch/ElasticSearchIOTestUtils.java | 141 +
.../io/elasticsearch/ElasticsearchIOITCommon.java | 92 +
.../elasticsearch/ElasticsearchIOTestCommon.java | 306 +
sdks/java/io/elasticsearch-tests/pom.xml | 144 +
sdks/java/io/elasticsearch/pom.xml | 224 +-
.../beam/sdk/io/elasticsearch/ElasticsearchIO.java | 479 +-
.../beam/sdk/io/elasticsearch/package-info.java | 1 -
.../io/elasticsearch/ElasticSearchIOTestUtils.java | 129 -
.../sdk/io/elasticsearch/ElasticsearchIOIT.java | 155 -
.../sdk/io/elasticsearch/ElasticsearchIOTest.java | 353 -
.../io/elasticsearch/ElasticsearchTestDataSet.java | 111 -
sdks/java/io/google-cloud-platform/pom.xml | 188 +-
.../beam/sdk/io/gcp/bigquery/BatchLoads.java | 480 +-
.../beam/sdk/io/gcp/bigquery/BigQueryHelpers.java | 28 +-
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 903 +-
.../sdk/io/gcp/bigquery/BigQueryQuerySource.java | 49 +-
.../beam/sdk/io/gcp/bigquery/BigQueryServices.java | 61 +-
.../sdk/io/gcp/bigquery/BigQueryServicesImpl.java | 121 +-
.../sdk/io/gcp/bigquery/BigQuerySourceBase.java | 153 +-
.../io/gcp/bigquery/BigQueryTableRowIterator.java | 501 -
.../sdk/io/gcp/bigquery/BigQueryTableSource.java | 40 +-
.../beam/sdk/io/gcp/bigquery/CalculateSchemas.java | 78 -
.../beam/sdk/io/gcp/bigquery/CreateTables.java | 23 +-
.../sdk/io/gcp/bigquery/DynamicDestinations.java | 34 +-
.../gcp/bigquery/DynamicDestinationsHelpers.java | 27 +-
.../sdk/io/gcp/bigquery/GenerateShardedTable.java | 1 +
.../sdk/io/gcp/bigquery/InsertRetryPolicy.java | 86 +
.../io/gcp/bigquery/PassThroughThenCleanup.java | 48 +-
.../beam/sdk/io/gcp/bigquery/ReifyAsIterable.java | 51 +
.../beam/sdk/io/gcp/bigquery/SchemaAndRecord.java | 43 +
.../beam/sdk/io/gcp/bigquery/ShardedKey.java | 67 -
.../beam/sdk/io/gcp/bigquery/ShardedKeyCoder.java | 74 -
.../beam/sdk/io/gcp/bigquery/StreamingInserts.java | 39 +-
.../beam/sdk/io/gcp/bigquery/StreamingWriteFn.java | 51 +-
.../sdk/io/gcp/bigquery/StreamingWriteTables.java | 38 +-
.../beam/sdk/io/gcp/bigquery/TableDestination.java | 43 +-
.../sdk/io/gcp/bigquery/TableDestinationCoder.java | 2 +
.../io/gcp/bigquery/TableDestinationCoderV2.java | 59 +
.../beam/sdk/io/gcp/bigquery/TagWithUniqueIds.java | 1 +
.../sdk/io/gcp/bigquery/TransformingSource.java | 136 -
.../sdk/io/gcp/bigquery/WriteBundlesToFiles.java | 57 +-
.../gcp/bigquery/WriteGroupedRecordsToFiles.java | 8 +-
.../beam/sdk/io/gcp/bigquery/WritePartition.java | 21 +-
.../beam/sdk/io/gcp/bigquery/WriteRename.java | 112 +-
.../beam/sdk/io/gcp/bigquery/WriteResult.java | 27 +-
.../beam/sdk/io/gcp/bigquery/WriteTables.java | 190 +-
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 171 +-
.../sdk/io/gcp/bigtable/BigtableServiceImpl.java | 25 +-
.../sdk/io/gcp/datastore/AdaptiveThrottler.java | 103 +
.../beam/sdk/io/gcp/datastore/DatastoreV1.java | 339 +-
.../beam/sdk/io/gcp/datastore/MovingAverage.java | 50 +
.../apache/beam/sdk/io/gcp/pubsub/PubsubIO.java | 53 +-
.../sdk/io/gcp/pubsub/PubsubUnboundedSink.java | 6 +-
.../sdk/io/gcp/pubsub/PubsubUnboundedSource.java | 43 +-
.../sdk/io/gcp/spanner/CreateTransactionFn.java | 61 +
.../beam/sdk/io/gcp/spanner/MutationGroup.java | 67 +
.../sdk/io/gcp/spanner/MutationGroupEncoder.java | 660 ++
.../sdk/io/gcp/spanner/MutationSizeEstimator.java | 155 +
.../sdk/io/gcp/spanner/NaiveSpannerReadFn.java | 85 +
.../beam/sdk/io/gcp/spanner/OrderedCode.java | 764 ++
.../beam/sdk/io/gcp/spanner/ReadOperation.java | 96 +
.../beam/sdk/io/gcp/spanner/ReadSpannerSchema.java | 94 +
.../sdk/io/gcp/spanner/SerializedMutation.java | 35 +
.../io/gcp/spanner/SerializedMutationCoder.java | 60 +
.../beam/sdk/io/gcp/spanner/SpannerAccessor.java | 43 +
.../beam/sdk/io/gcp/spanner/SpannerConfig.java | 161 +
.../apache/beam/sdk/io/gcp/spanner/SpannerIO.java | 991 ++
.../beam/sdk/io/gcp/spanner/SpannerSchema.java | 144 +
.../beam/sdk/io/gcp/spanner/Transaction.java | 33 +
.../beam/sdk/io/gcp/spanner/package-info.java | 23 +
.../apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java | 18 +-
.../beam/sdk/io/gcp/bigquery/BigQueryIOTest.java | 1023 +-
.../io/gcp/bigquery/BigQueryServicesImplTest.java | 88 +-
.../gcp/bigquery/BigQueryTableRowIteratorTest.java | 358 -
.../beam/sdk/io/gcp/bigquery/BigQueryUtilTest.java | 198 +-
.../sdk/io/gcp/bigquery/FakeBigQueryServices.java | 78 -
.../sdk/io/gcp/bigquery/FakeDatasetService.java | 101 +-
.../beam/sdk/io/gcp/bigquery/FakeJobService.java | 46 +-
.../sdk/io/gcp/bigquery/InsertRetryPolicyTest.java | 79 +
.../beam/sdk/io/gcp/bigquery/TableContainer.java | 2 +
.../beam/sdk/io/gcp/bigtable/BigtableIOTest.java | 30 +-
.../beam/sdk/io/gcp/bigtable/BigtableReadIT.java | 5 +-
.../sdk/io/gcp/bigtable/BigtableTestOptions.java | 5 -
.../beam/sdk/io/gcp/bigtable/BigtableWriteIT.java | 4 +-
.../io/gcp/datastore/AdaptiveThrottlerTest.java | 111 +
.../beam/sdk/io/gcp/datastore/DatastoreV1Test.java | 326 +-
.../beam/sdk/io/gcp/datastore/SplitQueryFnIT.java | 5 +-
.../apache/beam/sdk/io/gcp/datastore/V1ReadIT.java | 2 +-
.../beam/sdk/io/gcp/datastore/V1TestUtil.java | 17 +-
.../beam/sdk/io/gcp/datastore/V1WriteIT.java | 36 +-
.../beam/sdk/io/gcp/pubsub/PubsubIOTest.java | 55 +
.../sdk/io/gcp/spanner/FakeServiceFactory.java | 82 +
.../io/gcp/spanner/MutationGroupEncoderTest.java | 636 ++
.../io/gcp/spanner/MutationSizeEstimatorTest.java | 150 +
.../beam/sdk/io/gcp/spanner/OrderedCodeTest.java | 890 ++
.../beam/sdk/io/gcp/spanner/RandomUtils.java | 41 +
.../sdk/io/gcp/spanner/ReadSpannerSchemaTest.java | 134 +
.../beam/sdk/io/gcp/spanner/SpannerIOReadTest.java | 263 +
.../sdk/io/gcp/spanner/SpannerIOWriteTest.java | 497 +
.../beam/sdk/io/gcp/spanner/SpannerReadIT.java | 166 +
.../beam/sdk/io/gcp/spanner/SpannerSchemaTest.java | 61 +
.../beam/sdk/io/gcp/spanner/SpannerWriteIT.java | 170 +
sdks/java/io/hadoop-common/pom.xml | 2 +-
.../sdk/io/hadoop/SerializableConfiguration.java | 18 +-
sdks/java/io/hadoop-file-system/pom.xml | 47 +-
.../apache/beam/sdk/io/hdfs/HadoopFileSystem.java | 27 +-
.../beam/sdk/io/hdfs/HadoopFileSystemTest.java | 95 +-
sdks/java/io/hadoop/input-format/pom.xml | 7 +-
.../io/hadoop/inputformat/HadoopInputFormatIO.java | 105 +-
.../inputformat/HadoopInputFormatIOTest.java | 94 +-
sdks/java/io/hadoop/jdk1.8-tests/pom.xml | 5 +-
.../hadoop/inputformat/HIFIOWithElasticTest.java | 21 +-
sdks/java/io/hadoop/pom.xml | 2 +-
sdks/java/io/hbase/pom.xml | 26 +-
.../sdk/io/hbase/HBaseCoderProviderRegistrar.java | 38 +
.../java/org/apache/beam/sdk/io/hbase/HBaseIO.java | 1020 +-
.../beam/sdk/io/hbase/HBaseMutationCoder.java | 41 +
.../apache/beam/sdk/io/hbase/HBaseResultCoder.java | 6 +-
.../apache/beam/sdk/io/hbase/SerializableScan.java | 37 +-
.../io/hbase/HBaseCoderProviderRegistrarTest.java | 43 +
.../org/apache/beam/sdk/io/hbase/HBaseIOTest.java | 740 +-
.../beam/sdk/io/hbase/HBaseMutationCoderTest.java | 4 +-
.../beam/sdk/io/hbase/HBaseResultCoderTest.java | 4 +-
.../beam/sdk/io/hbase/SerializableScanTest.java | 6 +-
sdks/java/io/hcatalog/pom.xml | 179 +
.../apache/beam/sdk/io/hcatalog/HCatalogIO.java | 479 +
.../apache/beam/sdk/io/hcatalog/package-info.java | 22 +
.../sdk/io/hcatalog/EmbeddedMetastoreService.java | 87 +
.../beam/sdk/io/hcatalog/HCatalogIOTest.java | 277 +
.../beam/sdk/io/hcatalog/HCatalogIOTestUtils.java | 108 +
.../io/hcatalog/src/test/resources/hive-site.xml | 301 +
sdks/java/io/jdbc/pom.xml | 196 +-
.../java/org/apache/beam/sdk/io/jdbc/JdbcIO.java | 322 +-
.../java/org/apache/beam/sdk/io/jdbc/JdbcIOIT.java | 203 +-
.../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java | 121 +-
.../apache/beam/sdk/io/jdbc/JdbcTestDataSet.java | 130 -
.../apache/beam/sdk/io/jdbc/JdbcTestHelper.java | 81 +
sdks/java/io/jms/pom.xml | 2 +-
.../java/org/apache/beam/sdk/io/jms/JmsIO.java | 80 +-
.../java/org/apache/beam/sdk/io/jms/JmsIOTest.java | 78 +
sdks/java/io/kafka/README.md | 36 +
sdks/java/io/kafka/pom.xml | 31 +-
.../org/apache/beam/sdk/io/kafka/ConsumerSpEL.java | 56 +
.../java/org/apache/beam/sdk/io/kafka/KafkaIO.java | 956 +-
.../org/apache/beam/sdk/io/kafka/ProducerSpEL.java | 135 +
.../org/apache/beam/sdk/io/kafka/KafkaIOTest.java | 385 +-
sdks/java/io/kinesis/pom.xml | 18 +-
.../beam/sdk/io/kinesis/AWSClientsProvider.java | 36 +
.../beam/sdk/io/kinesis/CheckpointGenerator.java | 6 +-
.../apache/beam/sdk/io/kinesis/CustomOptional.java | 111 +-
.../sdk/io/kinesis/DynamicCheckpointGenerator.java | 52 +-
.../sdk/io/kinesis/GetKinesisRecordsResult.java | 55 +-
.../beam/sdk/io/kinesis/KinesisClientProvider.java | 31 -
.../org/apache/beam/sdk/io/kinesis/KinesisIO.java | 348 +-
.../apache/beam/sdk/io/kinesis/KinesisReader.java | 274 +-
.../sdk/io/kinesis/KinesisReaderCheckpoint.java | 83 +-
.../apache/beam/sdk/io/kinesis/KinesisRecord.java | 177 +-
.../beam/sdk/io/kinesis/KinesisRecordCoder.java | 68 +-
.../apache/beam/sdk/io/kinesis/KinesisSource.java | 161 +-
.../apache/beam/sdk/io/kinesis/RecordFilter.java | 18 +-
.../org/apache/beam/sdk/io/kinesis/RoundRobin.java | 53 -
.../beam/sdk/io/kinesis/ShardCheckpoint.java | 235 +-
.../beam/sdk/io/kinesis/ShardReadersPool.java | 162 +
.../beam/sdk/io/kinesis/ShardRecordsIterator.java | 120 +-
.../sdk/io/kinesis/SimplifiedKinesisClient.java | 291 +-
.../apache/beam/sdk/io/kinesis/StartingPoint.java | 84 +-
.../sdk/io/kinesis/StaticCheckpointGenerator.java | 27 +-
.../sdk/io/kinesis/TransientKinesisException.java | 9 +-
.../beam/sdk/io/kinesis/AmazonKinesisMock.java | 532 +-
.../beam/sdk/io/kinesis/CustomOptionalTest.java | 27 +-
.../io/kinesis/DynamicCheckpointGeneratorTest.java | 33 +-
.../beam/sdk/io/kinesis/KinesisMockReadTest.java | 98 +-
.../io/kinesis/KinesisReaderCheckpointTest.java | 52 +-
.../beam/sdk/io/kinesis/KinesisReaderIT.java | 128 +-
.../beam/sdk/io/kinesis/KinesisReaderTest.java | 254 +-
.../sdk/io/kinesis/KinesisRecordCoderTest.java | 34 +-
.../beam/sdk/io/kinesis/KinesisTestOptions.java | 43 +-
.../beam/sdk/io/kinesis/KinesisUploader.java | 70 +-
.../beam/sdk/io/kinesis/RecordFilterTest.java | 52 +-
.../apache/beam/sdk/io/kinesis/RoundRobinTest.java | 57 -
.../beam/sdk/io/kinesis/ShardCheckpointTest.java | 203 +-
.../beam/sdk/io/kinesis/ShardReadersPoolTest.java | 185 +
.../sdk/io/kinesis/ShardRecordsIteratorTest.java | 221 +-
.../io/kinesis/SimplifiedKinesisClientTest.java | 420 +-
sdks/java/io/mongodb/pom.xml | 2 +-
.../beam/sdk/io/mongodb/MongoDbGridFSIO.java | 8 +-
.../org/apache/beam/sdk/io/mongodb/MongoDbIO.java | 321 +-
.../beam/sdk/io/mongodb/MongoDBGridFSIOTest.java | 2 +-
.../apache/beam/sdk/io/mongodb/MongoDbIOTest.java | 48 +
sdks/java/io/mqtt/pom.xml | 2 +-
.../java/org/apache/beam/sdk/io/mqtt/MqttIO.java | 53 +-
sdks/java/io/pom.xml | 66 +-
sdks/java/io/redis/pom.xml | 90 +
.../sdk/io/redis/RedisConnectionConfiguration.java | 122 +
.../java/org/apache/beam/sdk/io/redis/RedisIO.java | 451 +
.../org/apache/beam/sdk/io/redis/package-info.java | 22 +
.../org/apache/beam/sdk/io/redis/RedisIOTest.java | 109 +
sdks/java/io/solr/pom.xml | 145 +
.../beam/sdk/io/solr/AuthorizedSolrClient.java | 91 +
.../apache/beam/sdk/io/solr/JavaBinCodecCoder.java | 98 +
.../java/org/apache/beam/sdk/io/solr/SolrIO.java | 705 ++
.../org/apache/beam/sdk/io/solr/package-info.java | 20 +
.../beam/sdk/io/solr/JavaBinCodecCoderTest.java | 81 +
.../org/apache/beam/sdk/io/solr/SolrIOTest.java | 269 +
.../apache/beam/sdk/io/solr/SolrIOTestUtils.java | 132 +
.../test/resources/cloud-minimal/conf/schema.xml | 29 +
.../resources/cloud-minimal/conf/solrconfig.xml | 48 +
sdks/java/io/tika/pom.xml | 113 +
.../org/apache/beam/sdk/io/tika/ParseResult.java | 144 +
.../java/org/apache/beam/sdk/io/tika/TikaIO.java | 284 +
.../org/apache/beam/sdk/io/tika/package-info.java | 22 +
.../apache/beam/sdk/io/tika/ParseResultTest.java | 83 +
.../org/apache/beam/sdk/io/tika/TikaIOTest.java | 149 +
sdks/java/io/tika/src/test/resources/damaged.pdf | 2 +
.../test/resources/valid/apache-beam-tika-pdf.zip | Bin 0 -> 11685 bytes
.../src/test/resources/valid/apache-beam-tika.odt | Bin 0 -> 12540 bytes
sdks/java/io/xml/pom.xml | 2 +-
.../java/org/apache/beam/sdk/io/xml/XmlIO.java | 375 +-
.../java/org/apache/beam/sdk/io/xml/XmlSink.java | 33 +-
.../java/org/apache/beam/sdk/io/xml/XmlSource.java | 65 +-
.../org/apache/beam/sdk/io/xml/XmlSinkTest.java | 16 +-
.../org/apache/beam/sdk/io/xml/XmlSourceTest.java | 74 +-
sdks/java/java8tests/pom.xml | 2 +-
sdks/java/javadoc/ant.xml | 1 +
sdks/java/javadoc/pom.xml | 44 +-
.../examples-java8/generate-sources.sh | 2 +-
sdks/java/maven-archetypes/examples-java8/pom.xml | 2 +-
.../src/main/resources/archetype-resources/pom.xml | 65 +-
.../maven-archetypes/examples/generate-sources.sh | 2 +-
sdks/java/maven-archetypes/examples/pom.xml | 2 +-
.../src/main/resources/archetype-resources/pom.xml | 49 +-
sdks/java/maven-archetypes/pom.xml | 2 +-
sdks/java/maven-archetypes/starter/pom.xml | 2 +-
.../src/main/resources/archetype-resources/pom.xml | 12 +-
.../resources/projects/basic/reference/pom.xml | 12 +-
sdks/java/nexmark/pom.xml | 265 +
.../java/org/apache/beam/sdk/nexmark/Main.java | 303 +
.../java/org/apache/beam/sdk/nexmark/Monitor.java | 78 +
.../beam/sdk/nexmark/NexmarkConfiguration.java | 721 ++
.../apache/beam/sdk/nexmark/NexmarkLauncher.java | 1157 +++
.../apache/beam/sdk/nexmark/NexmarkOptions.java | 403 +
.../org/apache/beam/sdk/nexmark/NexmarkPerf.java | 207 +
.../org/apache/beam/sdk/nexmark/NexmarkSuite.java | 112 +
.../org/apache/beam/sdk/nexmark/NexmarkUtils.java | 674 ++
.../org/apache/beam/sdk/nexmark/model/Auction.java | 187 +
.../apache/beam/sdk/nexmark/model/AuctionBid.java | 85 +
.../beam/sdk/nexmark/model/AuctionCount.java | 84 +
.../beam/sdk/nexmark/model/AuctionPrice.java | 88 +
.../org/apache/beam/sdk/nexmark/model/Bid.java | 177 +
.../beam/sdk/nexmark/model/BidsPerSession.java | 87 +
.../beam/sdk/nexmark/model/CategoryPrice.java | 97 +
.../org/apache/beam/sdk/nexmark/model/Done.java | 80 +
.../org/apache/beam/sdk/nexmark/model/Event.java | 171 +
.../beam/sdk/nexmark/model/IdNameReserve.java | 98 +
.../apache/beam/sdk/nexmark/model/KnownSize.java | 26 +
.../beam/sdk/nexmark/model/NameCityStateId.java | 103 +
.../org/apache/beam/sdk/nexmark/model/Person.java | 163 +
.../apache/beam/sdk/nexmark/model/SellerPrice.java | 89 +
.../beam/sdk/nexmark/model/package-info.java | 22 +
.../org/apache/beam/sdk/nexmark/package-info.java | 21 +
.../sdk/nexmark/queries/AbstractSimulator.java | 211 +
.../beam/sdk/nexmark/queries/NexmarkQuery.java | 270 +
.../sdk/nexmark/queries/NexmarkQueryModel.java | 117 +
.../apache/beam/sdk/nexmark/queries/Query0.java | 70 +
.../beam/sdk/nexmark/queries/Query0Model.java | 64 +
.../apache/beam/sdk/nexmark/queries/Query1.java | 67 +
.../apache/beam/sdk/nexmark/queries/Query10.java | 367 +
.../apache/beam/sdk/nexmark/queries/Query11.java | 79 +
.../apache/beam/sdk/nexmark/queries/Query12.java | 80 +
.../beam/sdk/nexmark/queries/Query1Model.java | 76 +
.../apache/beam/sdk/nexmark/queries/Query2.java | 79 +
.../beam/sdk/nexmark/queries/Query2Model.java | 80 +
.../apache/beam/sdk/nexmark/queries/Query3.java | 301 +
.../beam/sdk/nexmark/queries/Query3Model.java | 124 +
.../apache/beam/sdk/nexmark/queries/Query4.java | 116 +
.../beam/sdk/nexmark/queries/Query4Model.java | 186 +
.../apache/beam/sdk/nexmark/queries/Query5.java | 138 +
.../beam/sdk/nexmark/queries/Query5Model.java | 176 +
.../apache/beam/sdk/nexmark/queries/Query6.java | 155 +
.../beam/sdk/nexmark/queries/Query6Model.java | 133 +
.../apache/beam/sdk/nexmark/queries/Query7.java | 90 +
.../beam/sdk/nexmark/queries/Query7Model.java | 130 +
.../apache/beam/sdk/nexmark/queries/Query8.java | 98 +
.../beam/sdk/nexmark/queries/Query8Model.java | 148 +
.../apache/beam/sdk/nexmark/queries/Query9.java | 44 +
.../beam/sdk/nexmark/queries/Query9Model.java | 44 +
.../beam/sdk/nexmark/queries/WinningBids.java | 418 +
.../sdk/nexmark/queries/WinningBidsSimulator.java | 206 +
.../beam/sdk/nexmark/queries/package-info.java | 22 +
.../sdk/nexmark/sources/BoundedEventSource.java | 190 +
.../apache/beam/sdk/nexmark/sources/Generator.java | 609 ++
.../beam/sdk/nexmark/sources/GeneratorConfig.java | 298 +
.../sdk/nexmark/sources/UnboundedEventSource.java | 329 +
.../beam/sdk/nexmark/sources/package-info.java | 22 +
.../nexmark/src/main/resources/log4j.properties | 55 +
.../apache/beam/sdk/nexmark/queries/QueryTest.java | 185 +
.../nexmark/sources/BoundedEventSourceTest.java | 70 +
.../beam/sdk/nexmark/sources/GeneratorTest.java | 110 +
.../nexmark/sources/UnboundedEventSourceTest.java | 105 +
sdks/java/pom.xml | 5 +-
sdks/pom.xml | 6 +-
sdks/python/MANIFEST.in | 1 +
sdks/python/apache_beam/__init__.py | 47 +-
sdks/python/apache_beam/coders/coder_impl.py | 26 +-
sdks/python/apache_beam/coders/coders.py | 134 +-
sdks/python/apache_beam/coders/coders_test.py | 2 +-
.../apache_beam/coders/coders_test_common.py | 21 +-
sdks/python/apache_beam/coders/observable_test.py | 1 -
.../coders/proto2_coder_test_messages_pb2.py | 6 +-
.../apache_beam/coders/standard_coders_test.py | 11 +-
sdks/python/apache_beam/coders/stream.pxd | 2 +-
sdks/python/apache_beam/coders/stream.pyx | 2 +-
sdks/python/apache_beam/coders/stream_test.py | 1 -
sdks/python/apache_beam/coders/typecoders.py | 1 -
.../apache_beam/examples/complete/autocomplete.py | 21 +-
.../examples/complete/autocomplete_test.py | 31 +-
.../apache_beam/examples/complete/estimate_pi.py | 16 +-
.../examples/complete/estimate_pi_test.py | 14 +-
.../examples/complete/game/game_stats.py | 393 +
.../examples/complete/game/game_stats_test.py | 81 +
.../examples/complete/game/hourly_team_score.py | 256 +-
.../examples/complete/game/leader_board.py | 349 +
.../examples/complete/game/leader_board_test.py | 69 +
.../examples/complete/game/user_score.py | 195 +-
.../complete/juliaset/juliaset/juliaset.py | 48 +-
.../complete/juliaset/juliaset/juliaset_test.py | 6 +-
.../examples/complete/juliaset/juliaset_main.py | 2 -
.../examples/complete/juliaset/setup.py | 7 +-
sdks/python/apache_beam/examples/complete/tfidf.py | 38 +-
.../apache_beam/examples/complete/tfidf_test.py | 38 +-
.../examples/complete/top_wikipedia_sessions.py | 17 +-
.../complete/top_wikipedia_sessions_test.py | 10 +-
.../examples/cookbook/bigquery_schema.py | 159 +-
.../examples/cookbook/bigquery_side_input.py | 56 +-
.../examples/cookbook/bigquery_side_input_test.py | 39 +-
.../examples/cookbook/bigquery_tornadoes.py | 28 +-
.../cookbook/bigquery_tornadoes_it_test.py | 14 +-
.../examples/cookbook/bigquery_tornadoes_test.py | 19 +-
.../python/apache_beam/examples/cookbook/coders.py | 16 +-
.../apache_beam/examples/cookbook/coders_test.py | 14 +-
.../examples/cookbook/custom_ptransform.py | 29 +-
.../examples/cookbook/custom_ptransform_test.py | 11 +-
.../examples/cookbook/datastore_wordcount.py | 57 +-
.../apache_beam/examples/cookbook/filters.py | 21 +-
.../examples/cookbook/group_with_coder.py | 47 +-
.../examples/cookbook/group_with_coder_test.py | 10 +-
.../apache_beam/examples/cookbook/mergecontacts.py | 121 +-
.../examples/cookbook/mergecontacts_test.py | 6 +-
.../examples/cookbook/multiple_output_pardo.py | 84 +-
.../cookbook/multiple_output_pardo_test.py | 13 +-
.../apache_beam/examples/snippets/snippets.py | 576 +-
.../apache_beam/examples/snippets/snippets_test.py | 470 +-
.../apache_beam/examples/streaming_wordcap.py | 22 +-
.../apache_beam/examples/streaming_wordcount.py | 66 +-
.../apache_beam/examples/windowed_wordcount.py | 96 +
sdks/python/apache_beam/examples/wordcount.py | 13 +-
.../apache_beam/examples/wordcount_debugging.py | 65 +-
.../examples/wordcount_debugging_test.py | 3 +-
.../python/apache_beam/examples/wordcount_fnapi.py | 151 +
.../apache_beam/examples/wordcount_it_test.py | 16 +
.../apache_beam/examples/wordcount_minimal.py | 35 +-
.../apache_beam/examples/wordcount_minimal_test.py | 3 +-
sdks/python/apache_beam/examples/wordcount_test.py | 3 +-
sdks/python/apache_beam/internal/gcp/auth.py | 1 -
sdks/python/apache_beam/internal/gcp/json_value.py | 46 +-
.../apache_beam/internal/gcp/json_value_test.py | 3 +-
sdks/python/apache_beam/internal/pickler.py | 2 +-
sdks/python/apache_beam/internal/util.py | 6 +-
sdks/python/apache_beam/io/avroio.py | 164 +-
sdks/python/apache_beam/io/avroio_test.py | 54 +-
sdks/python/apache_beam/io/concat_source_test.py | 1 -
sdks/python/apache_beam/io/filebasedsink.py | 22 +-
sdks/python/apache_beam/io/filebasedsink_test.py | 19 +-
sdks/python/apache_beam/io/filebasedsource.py | 234 +-
sdks/python/apache_beam/io/filebasedsource_test.py | 8 +-
sdks/python/apache_beam/io/filesystem.py | 78 +-
sdks/python/apache_beam/io/filesystem_test.py | 5 +-
sdks/python/apache_beam/io/filesystems.py | 1 -
sdks/python/apache_beam/io/filesystems_test.py | 4 +-
sdks/python/apache_beam/io/gcp/bigquery.py | 458 +-
sdks/python/apache_beam/io/gcp/bigquery_test.py | 266 +-
.../io/gcp/datastore/v1/adaptive_throttler.py | 94 +
.../io/gcp/datastore/v1/adaptive_throttler_test.py | 95 +
.../apache_beam/io/gcp/datastore/v1/datastoreio.py | 136 +-
.../io/gcp/datastore/v1/datastoreio_test.py | 78 +-
.../apache_beam/io/gcp/datastore/v1/helper.py | 64 +-
.../apache_beam/io/gcp/datastore/v1/helper_test.py | 27 +-
.../python/apache_beam/io/gcp/datastore/v1/util.py | 95 +
.../apache_beam/io/gcp/datastore/v1/util_test.py | 67 +
.../apache_beam/io/gcp/gcsfilesystem_test.py | 1 +
sdks/python/apache_beam/io/gcp/gcsio.py | 29 +-
sdks/python/apache_beam/io/gcp/gcsio_test.py | 22 +-
.../clients/bigquery/bigquery_v2_messages.py | 3 -
.../clients/storage/storage_v1_messages.py | 3 -
sdks/python/apache_beam/io/gcp/pubsub.py | 167 +-
sdks/python/apache_beam/io/gcp/pubsub_test.py | 97 +-
.../apache_beam/io/gcp/tests/bigquery_matcher.py | 6 +-
.../io/gcp/tests/bigquery_matcher_test.py | 5 +-
sdks/python/apache_beam/io/gcp/tests/utils.py | 63 +
sdks/python/apache_beam/io/gcp/tests/utils_test.py | 110 +
sdks/python/apache_beam/io/iobase.py | 110 +-
sdks/python/apache_beam/io/localfilesystem_test.py | 4 +-
sdks/python/apache_beam/io/range_trackers.py | 184 +-
sdks/python/apache_beam/io/range_trackers_test.py | 223 +-
sdks/python/apache_beam/io/source_test_utils.py | 95 +-
.../apache_beam/io/source_test_utils_test.py | 2 +-
sdks/python/apache_beam/io/sources_test.py | 1 -
sdks/python/apache_beam/io/textio.py | 245 +-
sdks/python/apache_beam/io/textio_test.py | 499 +-
sdks/python/apache_beam/io/tfrecordio.py | 5 +-
sdks/python/apache_beam/io/tfrecordio_test.py | 8 +-
sdks/python/apache_beam/io/vcfio.py | 436 +
sdks/python/apache_beam/io/vcfio_test.py | 519 +
sdks/python/apache_beam/metrics/cells_test.py | 2 +-
sdks/python/apache_beam/metrics/execution.py | 5 +-
sdks/python/apache_beam/metrics/execution_test.py | 8 +-
sdks/python/apache_beam/metrics/metric.py | 3 +-
sdks/python/apache_beam/metrics/metric_test.py | 18 +-
sdks/python/apache_beam/metrics/metricbase.py | 4 +
.../python/apache_beam/options/pipeline_options.py | 66 +-
.../apache_beam/options/pipeline_options_test.py | 46 +-
.../options/pipeline_options_validator_test.py | 11 +-
sdks/python/apache_beam/options/value_provider.py | 2 -
.../apache_beam/options/value_provider_test.py | 93 +-
sdks/python/apache_beam/pipeline.py | 385 +-
sdks/python/apache_beam/pipeline_test.py | 143 +-
.../{runners/worker => portability}/__init__.py | 0
.../python/apache_beam/portability/api/__init__.py | 21 +
sdks/python/apache_beam/pvalue.py | 111 +-
sdks/python/apache_beam/pvalue_test.py | 8 +
sdks/python/apache_beam/runners/api/__init__.py | 32 -
.../apache_beam/runners/api/beam_fn_api_pb2.py | 2742 -----
.../runners/api/beam_fn_api_pb2_grpc.py | 205 -
.../apache_beam/runners/api/beam_runner_api_pb2.py | 2872 ------
sdks/python/apache_beam/runners/common.py | 6 +-
sdks/python/apache_beam/runners/common_test.py | 2 +-
.../runners/dataflow/dataflow_metrics.py | 2 +-
.../runners/dataflow/dataflow_runner.py | 320 +-
.../runners/dataflow/dataflow_runner_test.py | 142 +-
.../runners/dataflow/internal/apiclient.py | 148 +-
.../runners/dataflow/internal/apiclient_test.py | 135 +-
.../clients/dataflow/dataflow_v1b3_client.py | 259 +-
.../clients/dataflow/dataflow_v1b3_messages.py | 92 +-
.../internal/clients/dataflow/message_matchers.py | 1 -
.../clients/dataflow/message_matchers_test.py | 3 +-
.../runners/dataflow/internal/dependency.py | 127 +-
.../runners/dataflow/internal/dependency_test.py | 22 +-
.../apache_beam/runners/dataflow/internal/names.py | 2 +
.../runners/dataflow/native_io/iobase.py | 6 +-
.../runners/dataflow/native_io/iobase_test.py | 59 +-
.../runners/dataflow/native_io/streaming_create.py | 72 +
.../runners/dataflow/ptransform_overrides.py | 52 +
.../runners/dataflow/template_runner_test.py | 5 +-
.../runners/dataflow/test_dataflow_runner.py | 10 +-
.../apache_beam/runners/direct/bundle_factory.py | 2 +-
.../consumer_tracking_pipeline_visitor_test.py | 2 +-
.../apache_beam/runners/direct/direct_metrics.py | 2 +-
.../runners/direct/direct_metrics_test.py | 8 +-
.../apache_beam/runners/direct/direct_runner.py | 146 +-
.../runners/direct/direct_runner_test.py | 41 +
.../runners/direct/evaluation_context.py | 90 +-
sdks/python/apache_beam/runners/direct/executor.py | 242 +-
.../runners/direct/helper_transforms.py | 2 +-
.../runners/direct/transform_evaluator.py | 462 +-
.../apache_beam/runners/direct/transform_result.py | 41 -
sdks/python/apache_beam/runners/direct/util.py | 70 +
.../runners/direct/watermark_manager.py | 100 +-
.../{testing => runners/experimental}/__init__.py | 0
.../experimental/python_rpc_direct/__init__.py | 22 +
.../python_rpc_direct/python_rpc_direct_runner.py | 110 +
.../experimental/python_rpc_direct/server.py | 111 +
.../{testing => runners/job}/__init__.py | 0
sdks/python/apache_beam/runners/job/manager.py | 52 +
sdks/python/apache_beam/runners/job/utils.py | 32 +
.../python/apache_beam/runners/pipeline_context.py | 31 +-
.../runners/portability/fn_api_runner.py | 1096 +-
.../runners/portability/fn_api_runner_test.py | 134 +-
.../runners/portability/maptask_executor_runner.py | 19 +-
.../portability/maptask_executor_runner_test.py | 12 +-
.../runners/portability/universal_local_runner.py | 409 +
.../portability/universal_local_runner_main.py | 44 +
.../portability/universal_local_runner_test.py | 86 +
sdks/python/apache_beam/runners/runner.py | 61 +-
sdks/python/apache_beam/runners/runner_test.py | 2 +-
.../apache_beam/runners/worker/bundle_processor.py | 590 ++
.../apache_beam/runners/worker/data_plane.py | 71 +-
.../apache_beam/runners/worker/data_plane_test.py | 9 +-
.../apache_beam/runners/worker/log_handler.py | 17 +-
.../apache_beam/runners/worker/log_handler_test.py | 22 +-
.../apache_beam/runners/worker/opcounters.py | 10 +-
.../apache_beam/runners/worker/opcounters_test.py | 1 -
.../apache_beam/runners/worker/operation_specs.py | 20 +-
.../apache_beam/runners/worker/operations.pxd | 4 +-
.../apache_beam/runners/worker/operations.py | 87 +-
.../apache_beam/runners/worker/sdk_worker.py | 560 +-
.../apache_beam/runners/worker/sdk_worker_main.py | 30 +-
.../apache_beam/runners/worker/sdk_worker_test.py | 106 +-
.../apache_beam/runners/worker/statesampler.pyx | 68 +-
.../runners/worker/statesampler_fake.py | 17 +-
.../runners/worker/statesampler_test.py | 5 +-
.../apache_beam/testing/data/vcf/valid-4.0.vcf | 23 +
.../apache_beam/testing/data/vcf/valid-4.0.vcf.bz2 | Bin 0 -> 781 bytes
.../apache_beam/testing/data/vcf/valid-4.0.vcf.gz | Bin 0 -> 727 bytes
.../testing/data/vcf/valid-4.1-large.vcf | 10000 +++++++++++++++++++
.../testing/data/vcf/valid-4.1-large.vcf.gz | Bin 0 -> 156715 bytes
.../apache_beam/testing/data/vcf/valid-4.2.vcf | 42 +
.../apache_beam/testing/data/vcf/valid-4.2.vcf.gz | Bin 0 -> 1240 bytes
.../apache_beam/testing/pipeline_verifiers.py | 1 -
.../apache_beam/testing/pipeline_verifiers_test.py | 5 +-
sdks/python/apache_beam/testing/test_pipeline.py | 54 +-
.../apache_beam/testing/test_pipeline_test.py | 4 +-
sdks/python/apache_beam/testing/test_stream.py | 6 +-
.../python/apache_beam/testing/test_stream_test.py | 69 +
sdks/python/apache_beam/testing/test_utils.py | 63 +-
sdks/python/apache_beam/testing/test_utils_test.py | 86 +
sdks/python/apache_beam/testing/util.py | 21 +-
sdks/python/apache_beam/testing/util_test.py | 4 +-
sdks/python/apache_beam/transforms/combiners.py | 17 +-
.../apache_beam/transforms/combiners_test.py | 70 +-
sdks/python/apache_beam/transforms/core.py | 561 +-
sdks/python/apache_beam/transforms/create_test.py | 6 +-
sdks/python/apache_beam/transforms/display.py | 91 +-
sdks/python/apache_beam/transforms/display_test.py | 6 +-
sdks/python/apache_beam/transforms/ptransform.py | 202 +-
.../apache_beam/transforms/ptransform_test.py | 78 +-
.../apache_beam/transforms/sideinputs_test.py | 5 +-
sdks/python/apache_beam/transforms/timeutil.py | 1 -
sdks/python/apache_beam/transforms/trigger.py | 79 +-
sdks/python/apache_beam/transforms/trigger_test.py | 19 +-
sdks/python/apache_beam/transforms/util.py | 273 +-
sdks/python/apache_beam/transforms/util_test.py | 108 +
sdks/python/apache_beam/transforms/window.py | 68 +-
sdks/python/apache_beam/transforms/window_test.py | 158 +-
.../transforms/write_ptransform_test.py | 11 +-
sdks/python/apache_beam/typehints/decorators.py | 129 +-
.../typehints/native_type_compatibility.py | 166 +
.../typehints/native_type_compatibility_test.py | 92 +
sdks/python/apache_beam/typehints/opcodes.py | 19 +-
.../apache_beam/typehints/trivial_inference.py | 83 +-
.../typehints/trivial_inference_test.py | 16 +-
sdks/python/apache_beam/typehints/typecheck.py | 6 +-
.../apache_beam/typehints/typed_pipeline_test.py | 55 +-
sdks/python/apache_beam/typehints/typehints.py | 79 +-
.../python/apache_beam/typehints/typehints_test.py | 3 +-
sdks/python/apache_beam/utils/annotations_test.py | 1 +
sdks/python/apache_beam/utils/counters.py | 107 +-
sdks/python/apache_beam/utils/counters_test.py | 78 +
sdks/python/apache_beam/utils/plugin.py | 42 +
sdks/python/apache_beam/utils/processes_test.py | 1 -
sdks/python/apache_beam/utils/proto_utils.py | 17 +
sdks/python/apache_beam/utils/retry.py | 9 +-
sdks/python/apache_beam/utils/retry_test.py | 5 +-
sdks/python/apache_beam/utils/timestamp.py | 5 +
sdks/python/apache_beam/utils/urns.py | 49 +-
sdks/python/apache_beam/version.py | 2 +-
sdks/python/container/Dockerfile | 27 +
sdks/python/container/boot.go | 123 +
sdks/python/container/pom.xml | 154 +
sdks/python/gen_protos.py | 141 +
sdks/python/generate_pydoc.sh | 140 +-
sdks/python/pom.xml | 9 +-
sdks/python/run_postcommit.sh | 20 -
sdks/python/run_pylint.sh | 49 +-
sdks/python/run_validatesrunner.sh | 71 +
sdks/python/setup.py | 65 +-
sdks/python/tox.ini | 32 +-
1777 files changed, 159991 insertions(+), 41448 deletions(-)
create mode 100644 .test-infra/jenkins/PreCommit_Pipeline.groovy
create mode 100644 .test-infra/jenkins/job_00_seed.groovy
create mode 100644 .test-infra/jenkins/job_beam_Java_Build.groovy
create mode 100644 .test-infra/jenkins/job_beam_Java_CodeHealth.groovy
create mode 100644 .test-infra/jenkins/job_beam_Java_IntegrationTest.groovy
create mode 100644 .test-infra/jenkins/job_beam_Java_UnitTest.groovy
create mode 100644 .test-infra/jenkins/job_beam_PerformanceTests_Python.groovy
create mode 100644 .test-infra/jenkins/job_beam_PostCommit_Java_JDKVersionsTest.groovy
create mode 100644 .test-infra/jenkins/job_beam_PostCommit_Python_ValidatesRunner_Dataflow.groovy
create mode 100644 .test-infra/jenkins/job_beam_PreCommit_Go_MavenInstall.groovy
create mode 100644 .test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy
create mode 100644 .test-infra/jenkins/job_beam_PreCommit_Python_MavenInstall.groovy
create mode 100644 .test-infra/jenkins/job_beam_PreCommit_Website_Merge.groovy
create mode 100644 .test-infra/jenkins/job_beam_Python_UnitTest.groovy
delete mode 100644 .test-infra/jenkins/job_seed.groovy
create mode 100644 .test-infra/jenkins/job_seed_standalone.groovy
create mode 100644 .test-infra/kubernetes/cassandra/LargeITCluster/setup.sh
delete mode 100644 .test-infra/kubernetes/cassandra/LargeITCluster/start-up.sh
create mode 100644 .test-infra/kubernetes/cassandra/SmallITCluster/setup.sh
delete mode 100644 .test-infra/kubernetes/cassandra/SmallITCluster/start-up.sh
create mode 100644 .test-infra/kubernetes/elasticsearch/LargeProductionCluster/setup.sh
delete mode 100644 .test-infra/kubernetes/elasticsearch/LargeProductionCluster/start-up.sh
create mode 100644 .test-infra/kubernetes/elasticsearch/SmallITCluster/setup.sh
delete mode 100644 .test-infra/kubernetes/elasticsearch/SmallITCluster/start-up.sh
create mode 100644 .test-infra/kubernetes/postgres/pkb-config-local.yml
create mode 100644 .test-infra/kubernetes/postgres/pkb-config.yml
create mode 100644 examples/java8/src/main/java/org/apache/beam/examples/website_snippets/Snippets.java
create mode 100644 examples/java8/src/test/java/org/apache/beam/examples/website_snippets/SnippetsTest.java
create mode 100644 model/fn-execution/pom.xml
create mode 100644 model/fn-execution/src/main/proto/beam_fn_api.proto
create mode 100644 model/fn-execution/src/main/proto/beam_provision_api.proto
rename {sdks/common/fn-api/src/test/resources/org/apache/beam/fn => model/fn-execution/src/test/resources/org/apache/beam/model/fnexecution}/v1/standard_coders.yaml (100%)
create mode 100644 model/job-management/pom.xml
create mode 100644 model/job-management/src/main/proto/beam_artifact_api.proto
create mode 100644 model/job-management/src/main/proto/beam_job_api.proto
create mode 100644 model/pipeline/pom.xml
create mode 100644 model/pipeline/src/main/proto/beam_runner_api.proto
create mode 100644 model/pipeline/src/main/proto/endpoints.proto
create mode 100644 model/pipeline/src/main/proto/standard_window_fns.proto
create mode 100644 model/pom.xml
delete mode 100644 runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/SerializablePipelineOptions.java
delete mode 100644 runners/apex/src/test/java/org/apache/beam/runners/apex/translation/utils/PipelineOptionsTest.java
create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ArtifactServiceStager.java
create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CoderTranslation.java
delete mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Coders.java
create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CombineTranslation.java
create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CreatePCollectionViewTranslation.java
create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/DisplayDataTranslation.java
create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/FlattenTranslator.java
create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/GroupByKeyTranslation.java
create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollectionTranslation.java
delete mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollections.java
create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java
delete mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransforms.java
create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDoTranslation.java
create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PipelineOptionsTranslation.java
create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PipelineTranslation.java
create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReadTranslation.java
create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/RehydratedComponents.java
create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/RunnerPCollectionView.java
create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SerializablePipelineOptions.java
create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SplittableParDo.java
create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TestStreamTranslation.java
create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformInputs.java
create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformPayloadTranslatorRegistrar.java
create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TriggerTranslation.java
delete mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Triggers.java
create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslation.java
delete mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowingStrategies.java
create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowingStrategyTranslation.java
create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WriteFilesTranslation.java
create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/metrics/MetricFiltering.java
create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/metrics/MetricKey.java
create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/metrics/package-info.java
delete mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/metrics/MetricFiltering.java
delete mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/metrics/MetricKey.java
delete mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/metrics/package-info.java
create mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ArtifactServiceStagerTest.java
create mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CoderTranslationTest.java
delete mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CodersTest.java
create mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CombineTranslationTest.java
create mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CreatePCollectionViewTranslationTest.java
create mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/GroupByKeyTranslationTest.java
create mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/InMemoryArtifactStagerService.java
create mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionTranslationTest.java
delete mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionsTest.java
create mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformTranslationTest.java
delete mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformsTest.java
create mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ParDoTranslationTest.java
create mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PipelineOptionsTranslationTest.java
create mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PipelineTranslationTest.java
create mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ReadTranslationTest.java
create mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/RehydratedComponentsTest.java
create mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SerializablePipelineOptionsTest.java
create mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SplittableParDoTest.java
create mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TestStreamTranslationTest.java
create mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TransformInputsTest.java
create mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TriggerTranslationTest.java
delete mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TriggersTest.java
create mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowIntoTranslationTest.java
delete mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowingStrategiesTest.java
create mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowingStrategyTranslationTest.java
create mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WriteFilesTranslationTest.java
create mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/metrics/MetricFilteringTest.java
delete mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/metrics/MetricFilteringTest.java
delete mode 100644 runners/core-java/src/main/java/org/apache/beam/runners/core/BaseExecutionContext.java
delete mode 100644 runners/core-java/src/main/java/org/apache/beam/runners/core/ElementAndRestriction.java
delete mode 100644 runners/core-java/src/main/java/org/apache/beam/runners/core/ElementAndRestrictionCoder.java
delete mode 100644 runners/core-java/src/main/java/org/apache/beam/runners/core/ExecutionContext.java
delete mode 100644 runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDo.java
create mode 100644 runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDoViaKeyedWorkItems.java
create mode 100644 runners/core-java/src/main/java/org/apache/beam/runners/core/StepContext.java
create mode 100644 runners/core-java/src/main/java/org/apache/beam/runners/core/fn/FnApiControlClient.java
create mode 100644 runners/core-java/src/main/java/org/apache/beam/runners/core/fn/FnApiControlClientPoolService.java
create mode 100644 runners/core-java/src/main/java/org/apache/beam/runners/core/fn/FnDataReceiver.java
create mode 100644 runners/core-java/src/main/java/org/apache/beam/runners/core/fn/FnDataService.java
create mode 100644 runners/core-java/src/main/java/org/apache/beam/runners/core/fn/SdkHarnessClient.java
create mode 100644 runners/core-java/src/main/java/org/apache/beam/runners/core/fn/SdkHarnessDoFnRunner.java
create mode 100644 runners/core-java/src/main/java/org/apache/beam/runners/core/fn/package-info.java
delete mode 100644 runners/core-java/src/test/java/org/apache/beam/runners/core/ElementAndRestrictionCoderTest.java
create mode 100644 runners/core-java/src/test/java/org/apache/beam/runners/core/SplittableParDoProcessFnTest.java
delete mode 100644 runners/core-java/src/test/java/org/apache/beam/runners/core/SplittableParDoTest.java
create mode 100644 runners/core-java/src/test/java/org/apache/beam/runners/core/StateInternalsTest.java
create mode 100644 runners/core-java/src/test/java/org/apache/beam/runners/core/fn/FnApiControlClientPoolServiceTest.java
create mode 100644 runners/core-java/src/test/java/org/apache/beam/runners/core/fn/FnApiControlClientTest.java
create mode 100644 runners/core-java/src/test/java/org/apache/beam/runners/core/fn/SdkHarnessClientTest.java
create mode 100644 runners/core-java/src/test/java/org/apache/beam/runners/core/fn/SdkHarnessDoFnRunnerTest.java
create mode 100644 runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectTestOptions.java
create mode 100644 runners/direct-java/src/main/java/org/apache/beam/runners/direct/MultiStepCombine.java
create mode 100644 runners/direct-java/src/main/java/org/apache/beam/runners/direct/ReadEvaluatorFactory.java
create mode 100644 runners/direct-java/src/main/java/org/apache/beam/runners/direct/SourceShard.java
create mode 100644 runners/direct-java/src/test/java/org/apache/beam/runners/direct/MultiStepCombineTest.java
create mode 100644 runners/flink/src/main/java/org/apache/beam/runners/flink/CreateStreamingFlinkView.java
delete mode 100644 runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingViewOverrides.java
create mode 100644 runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkTransformOverrides.java
delete mode 100644 runners/flink/src/main/java/org/apache/beam/runners/flink/translation/utils/SerializedPipelineOptions.java
create mode 100644 runners/flink/src/test/java/org/apache/beam/runners/flink/translation/types/CoderTypeSerializerTest.java
create mode 100644 runners/gcp/gcemd/Dockerfile
create mode 100644 runners/gcp/gcemd/main.go
create mode 100644 runners/gcp/gcemd/pom.xml
create mode 100644 runners/gcp/gcsproxy/Dockerfile
create mode 100644 runners/gcp/gcsproxy/main.go
create mode 100644 runners/gcp/gcsproxy/pom.xml
create mode 100644 runners/gcp/pom.xml
create mode 100644 runners/gearpump/README.md
create mode 100644 runners/gearpump/pom.xml
create mode 100644 runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineOptions.java
create mode 100644 runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineResult.java
create mode 100644 runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java
create mode 100644 runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunnerRegistrar.java
create mode 100644 runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/TestGearpumpRunner.java
create mode 100644 runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/package-info.java
create mode 100644 runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateGearpumpPCollectionViewTranslator.java
create mode 100644 runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateStreamingGearpumpView.java
create mode 100644 runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslator.java
create mode 100644 runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GearpumpPipelineTranslator.java
create mode 100644 runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java
create mode 100644 runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoMultiOutputTranslator.java
create mode 100644 runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ReadBoundedTranslator.java
create mode 100644 runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ReadUnboundedTranslator.java
create mode 100644 runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TransformTranslator.java
create mode 100644 runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java
create mode 100644 runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowAssignTranslator.java
create mode 100644 runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
create mode 100644 runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/package-info.java
create mode 100644 runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/BoundedSourceWrapper.java
create mode 100644 runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
create mode 100644 runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/UnboundedSourceWrapper.java
create mode 100644 runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/ValuesSource.java
create mode 100644 runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/package-info.java
create mode 100644 runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/package-info.java
create mode 100644 runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java
create mode 100644 runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/NoOpStepContext.java
create mode 100644 runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
create mode 100644 runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/package-info.java
create mode 100644 runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/GearpumpRunnerRegistrarTest.java
create mode 100644 runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/PipelineOptionsTest.java
create mode 100644 runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/CreateGearpumpPCollectionViewTranslatorTest.java
create mode 100644 runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslatorTest.java
create mode 100644 runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslatorTest.java
create mode 100644 runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/ReadBoundedTranslatorTest.java
create mode 100644 runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/ReadUnboundedTranslatorTest.java
create mode 100644 runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/WindowAssignTranslatorTest.java
create mode 100644 runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSourceTest.java
create mode 100644 runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/io/ValueSoureTest.java
create mode 100644 runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtilsTest.java
delete mode 100644 runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/AssignWindows.java
create mode 100644 runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/SplittableParDoOverrides.java
delete mode 100644 runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DoFnInfo.java
create mode 100644 runners/java-fn-execution/pom.xml
create mode 100644 runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/ServerFactory.java
create mode 100644 runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/FnApiControlClient.java
create mode 100644 runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/FnApiControlClientPoolService.java
create mode 100644 runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/SdkHarnessClient.java
create mode 100644 runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/package-info.java
create mode 100644 runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/data/FnDataReceiver.java
create mode 100644 runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/data/package-info.java
create mode 100644 runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/package-info.java
create mode 100644 runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/ServerFactoryTest.java
create mode 100644 runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/FnApiControlClientPoolServiceTest.java
create mode 100644 runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/FnApiControlClientTest.java
create mode 100644 runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/SdkHarnessClientTest.java
create mode 100644 runners/local-artifact-service-java/pom.xml
create mode 100644 runners/local-artifact-service-java/src/main/java/org/apache/beam/artifact/local/LocalFileSystemArtifactStagerService.java
create mode 100644 runners/local-artifact-service-java/src/main/java/org/apache/beam/artifact/local/package-info.java
create mode 100644 runners/local-artifact-service-java/src/test/java/org/apache/beam/artifact/local/LocalFileSystemArtifactStagerServiceTest.java
create mode 100644 runners/reference/job-server/pom.xml
create mode 100644 runners/reference/job-server/src/main/java/org/apache/beam/runners/reference/job/ReferenceRunnerJobServer.java
create mode 100644 runners/reference/job-server/src/main/java/org/apache/beam/runners/reference/job/ReferenceRunnerJobService.java
create mode 100644 runners/reference/job-server/src/main/java/org/apache/beam/runners/reference/job/package-info.java
create mode 100644 runners/reference/job-server/src/test/java/org/apache/beam/runners/reference/job/ReferenceRunnerJobServiceTest.java
create mode 100644 runners/reference/pom.xml
delete mode 100644 runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkRuntimeContext.java
delete mode 100644 runners/spark/src/main/java/org/apache/beam/runners/spark/translation/StorageLevelPTransform.java
create mode 100644 runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/WatermarkSyncedDStream.java
delete mode 100644 runners/spark/src/main/java/org/apache/beam/runners/spark/util/SinglePrimitiveOutputPTransform.java
create mode 100644 runners/spark/src/test/java/org/apache/beam/runners/spark/stateful/SparkStateInternalsTest.java
delete mode 100644 runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SparkRuntimeContextTest.java
delete mode 100644 runners/spark/src/test/java/org/apache/beam/runners/spark/translation/StorageLevelTest.java
create mode 100644 sdks/CONTAINERS.md
delete mode 100644 sdks/common/fn-api/pom.xml
delete mode 100644 sdks/common/fn-api/src/main/proto/beam_fn_api.proto
delete mode 100644 sdks/common/pom.xml
delete mode 100644 sdks/common/runner-api/pom.xml
delete mode 100644 sdks/common/runner-api/src/main/proto/beam_runner_api.proto
create mode 100644 sdks/go/BUILD.md
create mode 100644 sdks/go/cmd/beamctl/cmd/artifact.go
create mode 100644 sdks/go/cmd/beamctl/cmd/root.go
create mode 100644 sdks/go/cmd/beamctl/main.go
create mode 100644 sdks/go/descriptor.xml
create mode 100644 sdks/go/pkg/beam/artifact/gcsproxy/retrieval.go
create mode 100644 sdks/go/pkg/beam/artifact/gcsproxy/staging.go
create mode 100644 sdks/go/pkg/beam/artifact/materialize.go
create mode 100644 sdks/go/pkg/beam/artifact/materialize_test.go
create mode 100644 sdks/go/pkg/beam/artifact/server_test.go
create mode 100644 sdks/go/pkg/beam/artifact/stage.go
create mode 100644 sdks/go/pkg/beam/artifact/stage_test.go
create mode 100644 sdks/go/pkg/beam/model/fnexecution_v1/beam_fn_api.pb.go
create mode 100644 sdks/go/pkg/beam/model/fnexecution_v1/beam_provision_api.pb.go
create mode 100644 sdks/go/pkg/beam/model/gen.go
create mode 100644 sdks/go/pkg/beam/model/jobmanagement_v1/beam_artifact_api.pb.go
create mode 100644 sdks/go/pkg/beam/model/jobmanagement_v1/beam_job_api.pb.go
create mode 100644 sdks/go/pkg/beam/model/pipeline_v1/beam_runner_api.pb.go
create mode 100644 sdks/go/pkg/beam/model/pipeline_v1/endpoints.pb.go
create mode 100644 sdks/go/pkg/beam/model/pipeline_v1/standard_window_fns.pb.go
create mode 100644 sdks/go/pkg/beam/provision/provision_test.go
create mode 100644 sdks/go/pkg/beam/provision/provison.go
create mode 100644 sdks/go/pkg/beam/util/errorx/guarded.go
create mode 100644 sdks/go/pkg/beam/util/execx/exec.go
create mode 100644 sdks/go/pkg/beam/util/gcsx/gcs.go
create mode 100644 sdks/go/pkg/beam/util/grpcx/dial.go
create mode 100644 sdks/go/pkg/beam/util/grpcx/metadata.go
create mode 100644 sdks/go/pkg/beam/util/syscallx/syscall.go
create mode 100644 sdks/go/pkg/beam/util/syscallx/syscall_default.go
create mode 100644 sdks/go/pkg/beam/util/syscallx/syscall_linux.go
create mode 100644 sdks/go/pom.xml
create mode 100644 sdks/java/build-tools/src/main/resources/docker/file/openjdk7/Dockerfile
create mode 100755 sdks/java/build-tools/src/main/resources/docker/file/openjdk7/docker-entrypoint.sh
create mode 100644 sdks/java/build-tools/src/main/resources/docker/file/openjdk8/Dockerfile
create mode 100755 sdks/java/build-tools/src/main/resources/docker/file/openjdk8/docker-entrypoint.sh
create mode 100644 sdks/java/build-tools/src/main/resources/docker/git/openjdk8/Dockerfile
create mode 100755 sdks/java/build-tools/src/main/resources/docker/git/openjdk8/docker-entrypoint.sh
create mode 100644 sdks/java/build-tools/src/main/resources/docker/release/python2/Dockerfile
create mode 100644 sdks/java/container/Dockerfile
create mode 100644 sdks/java/container/boot.go
create mode 100644 sdks/java/container/pom.xml
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BeamRecordCoder.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BooleanCoder.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ShardedKeyCoder.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroUtils.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/io/Compression.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/io/ConstantAvroDestination.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/io/DynamicAvroDestinations.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/io/DynamicFileDestinations.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileIO.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/io/ReadAllViaFileBasedSource.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/io/ReadableFileCoder.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFilesResult.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/EmptyMatchTreatment.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/MetadataCoder.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/ResourceIdCoder.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/OffsetRange.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/options/ExperimentalOptions.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/options/SdkHarnessOptions.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/testing/CombineFnTester.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/testing/LargeKeys.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesCustomWindowMerging.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Contextful.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Requirements.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/SerializableFunctions.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Watch.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/OnTimerMethodSpecifier.java
delete mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/OffsetRange.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/IncompatibleWindowException.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/util/DoFnInfo.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/util/SerializableThrowable.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/values/BeamRecord.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/values/BeamRecordType.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/values/ShardedKey.java
delete mode 100644 sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroIOTransformTest.java
create mode 100644 sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileIOTest.java
create mode 100644 sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOReadTest.java
create mode 100644 sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOWriteTest.java
create mode 100644 sdks/java/core/src/test/java/org/apache/beam/sdk/options/SdkHarnessOptionsTest.java
create mode 100644 sdks/java/core/src/test/java/org/apache/beam/sdk/testing/CombineFnTesterTest.java
create mode 100644 sdks/java/core/src/test/java/org/apache/beam/sdk/testing/InterceptingUrlClassLoader.java
create mode 100644 sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/WatchTest.java
create mode 100644 sdks/java/extensions/join-library/src/test/java/org/apache/beam/sdk/extensions/joinlibrary/OuterFullJoinTest.java
create mode 100755 sdks/java/extensions/sketching/pom.xml
create mode 100644 sdks/java/extensions/sketching/src/main/java/org/apache/beam/sdk/extensions/sketching/ApproximateDistinct.java
create mode 100755 sdks/java/extensions/sketching/src/main/java/org/apache/beam/sdk/extensions/sketching/package-info.java
create mode 100644 sdks/java/extensions/sketching/src/test/java/org/apache/beam/sdk/extensions/sketching/ApproximateDistinctTest.java
create mode 100644 sdks/java/extensions/sql/NOTICE
create mode 100644 sdks/java/extensions/sql/pom.xml
create mode 100644 sdks/java/extensions/sql/src/main/codegen/config.fmpp
create mode 100644 sdks/java/extensions/sql/src/main/codegen/data/Parser.tdd
create mode 100644 sdks/java/extensions/sql/src/main/codegen/includes/license.ftl
create mode 100644 sdks/java/extensions/sql/src/main/codegen/includes/parserImpls.ftl
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamRecordSqlType.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSql.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSqlCli.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSqlRecordHelper.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSqlTable.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSqlUdf.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/example/BeamSqlExample.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/example/package-info.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamSqlEnv.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/BeamSqlExpressionExecutor.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/BeamSqlFnExecutor.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlCaseExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlCastExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlInputRefExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlPrimitive.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlUdfExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlWindowEndExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlWindowExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlWindowStartExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/UdafImpl.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlArithmeticExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlDivideExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlMinusExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlModExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlMultiplyExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlPlusExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/package-info.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/BeamSqlCompareExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/BeamSqlEqualsExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/BeamSqlGreaterThanExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/BeamSqlGreaterThanOrEqualsExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/BeamSqlIsNotNullExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/BeamSqlIsNullExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/BeamSqlLessThanExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/BeamSqlLessThanOrEqualsExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/BeamSqlNotEqualsExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/comparison/package-info.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlCurrentDateExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlCurrentTimeExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlCurrentTimestampExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlDateCeilExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlDateFloorExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlDatetimeMinusExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlDatetimePlusExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlExtractExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlIntervalMultiplyExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlTimestampMinusIntervalExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlTimestampMinusTimestampExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/TimeUnitUtils.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/package-info.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/logical/BeamSqlAndExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/logical/BeamSqlLogicalExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/logical/BeamSqlNotExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/logical/BeamSqlOrExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/logical/package-info.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlAbsExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlAcosExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlAsinExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlAtan2Expression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlAtanExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlCeilExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlCosExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlCotExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlDegreesExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlExpExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlFloorExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlLnExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlLogExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlMathBinaryExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlMathUnaryExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlPiExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlPowerExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlRadiansExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlRandExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlRandIntegerExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlRoundExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlSignExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlSinExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlTanExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlTruncateExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/package-info.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/package-info.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/BeamSqlReinterpretExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/DatetimeReinterpretConversions.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/IntegerReinterpretConversions.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/ReinterpretConversion.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/Reinterpreter.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/package-info.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlCharLengthExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlConcatExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlInitCapExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlLowerExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlOverlayExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlPositionExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlStringUnaryExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlSubstringExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlTrimExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlUpperExpression.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/package-info.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/package-info.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/package-info.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/BeamSqlParser.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/ColumnConstraint.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/ColumnDefinition.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/ParserUtils.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/SqlCreateTable.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/SqlDDLKeywords.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/UnparseUtil.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/package-info.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/planner/BeamQueryPlanner.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/planner/BeamRelDataTypeSystem.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/planner/BeamRuleSets.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/planner/package-info.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamAggregationRel.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamFilterRel.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamIOSinkRel.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamIOSourceRel.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamIntersectRel.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamJoinRel.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamLogicalConvention.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamMinusRel.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamProjectRel.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamRelNode.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSetOperatorRelBase.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSortRel.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSqlRelUtils.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamUnionRel.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamValuesRel.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/package-info.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamAggregationRule.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamFilterRule.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamIOSinkRule.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamIOSourceRule.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamIntersectRule.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamJoinRule.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamMinusRule.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamProjectRule.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamSortRule.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamUnionRule.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamValuesRule.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/package-info.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/schema/BaseBeamTable.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/schema/BeamIOType.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/schema/BeamPCollectionTable.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/schema/BeamTableUtils.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/schema/package-info.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamAggregationTransforms.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamBuiltinAggregations.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamJoinTransforms.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamSetOperatorsTransforms.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamSqlFilterFn.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamSqlOutputToConsoleFn.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamSqlProjectFn.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/package-info.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/utils/CalciteUtils.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/utils/SqlTypeUtils.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/utils/package-info.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/Column.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/Table.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/package-info.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/MetaUtils.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/TableProvider.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaCSVTable.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTable.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaTableProvider.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/package-info.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/package-info.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/BeamTextCSVTable.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/BeamTextCSVTableIOReader.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/BeamTextCSVTableIOWriter.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/BeamTextTable.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/TextTableProvider.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/package-info.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/store/InMemoryMetaStore.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/store/MetaStore.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/store/package-info.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/package-info.java
create mode 100644 sdks/java/extensions/sql/src/main/resources/log4j.properties
create mode 100644 sdks/java/extensions/sql/src/main/resources/org.apache.beam.sdks.java.extensions.sql.repackaged.org.codehaus.commons.compiler.properties
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlApiSurfaceTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlCliTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslAggregationTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslBase.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslFilterTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslJoinTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslProjectTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslUdfUdafTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/TestUtils.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/BeamSqlFnExecutorTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/BeamSqlFnExecutorTestBase.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamNullExperssionTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlAndOrExpressionTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlCaseExpressionTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlCastExpressionTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlCompareExpressionTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlInputRefExpressionTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlPrimitiveTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlReinterpretExpressionTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlUdfExpressionTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlArithmeticExpressionTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlCurrentDateExpressionTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlCurrentTimeExpressionTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlCurrentTimestampExpressionTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlDateCeilExpressionTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlDateExpressionTestBase.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlDateFloorExpressionTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlDatetimeMinusExpressionTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlDatetimePlusExpressionTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlExtractExpressionTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlIntervalMultiplyExpressionTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlTimestampMinusIntervalExpressionTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlTimestampMinusTimestampExpressionTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/TimeUnitUtilsTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/logical/BeamSqlNotExpressionTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlMathBinaryExpressionTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlMathUnaryExpressionTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/DatetimeReinterpretConversionsTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/IntegerReinterpretConversionsTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/ReinterpretConversionTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/reinterpret/ReinterpreterTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlCharLengthExpressionTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlConcatExpressionTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlInitCapExpressionTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlLowerExpressionTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlOverlayExpressionTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlPositionExpressionTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlStringUnaryExpressionTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlSubstringExpressionTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlTrimExpressionTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlUpperExpressionTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/parser/BeamSqlParserTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BaseRelTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamIntersectRelTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamJoinRelBoundedVsBoundedTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamJoinRelUnboundedVsBoundedTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamJoinRelUnboundedVsUnboundedTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamMinusRelTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSetOperatorRelBaseTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSortRelTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamUnionRelTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamValuesRelTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/CheckSize.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/schema/BeamSqlRowCoderTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/schema/transform/BeamAggregationTransformTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/schema/transform/BeamTransformBaseTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/utils/SqlTypeUtilsTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlArithmeticOperatorsIntegrationTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlBuiltinFunctionsIntegrationTestBase.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlComparisonOperatorsIntegrationTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlConditionalFunctionsIntegrationTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlDateFunctionsIntegrationTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlLogicalFunctionsIntegrationTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlMathFunctionsIntegrationTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlStringFunctionsIntegrationTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaCSVTableTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaTableProviderTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/BeamTextCSVTableTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/TextTableProviderTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/store/InMemoryMetaStoreTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/mock/MockedBoundedTable.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/mock/MockedTable.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/mock/MockedUnboundedTable.java
create mode 100644 sdks/java/fn-execution/pom.xml
create mode 100644 sdks/java/fn-execution/src/main/java/org/apache/beam/harness/channel/ManagedChannelFactory.java
create mode 100644 sdks/java/fn-execution/src/main/java/org/apache/beam/harness/channel/SocketAddressFactory.java
create mode 100644 sdks/java/fn-execution/src/main/java/org/apache/beam/harness/channel/package-info.java
create mode 100644 sdks/java/fn-execution/src/test/java/org/apache/beam/harness/channel/ManagedChannelFactoryTest.java
create mode 100644 sdks/java/fn-execution/src/test/java/org/apache/beam/harness/channel/SocketAddressFactoryTest.java
create mode 100644 sdks/java/fn-execution/src/test/java/org/apache/beam/harness/test/Consumer.java
create mode 100644 sdks/java/fn-execution/src/test/java/org/apache/beam/harness/test/Supplier.java
create mode 100644 sdks/java/fn-execution/src/test/java/org/apache/beam/harness/test/TestExecutors.java
create mode 100644 sdks/java/fn-execution/src/test/java/org/apache/beam/harness/test/TestExecutorsTest.java
create mode 100644 sdks/java/fn-execution/src/test/java/org/apache/beam/harness/test/TestStreams.java
create mode 100644 sdks/java/fn-execution/src/test/java/org/apache/beam/harness/test/TestStreamsTest.java
create mode 100644 sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataReadRunner.java
create mode 100644 sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataWriteRunner.java
create mode 100644 sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BoundedSourceRunner.java
create mode 100644 sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnApiDoFnRunner.java
create mode 100644 sdks/java/harness/src/main/java/org/apache/beam/fn/harness/IdGenerator.java
create mode 100644 sdks/java/harness/src/main/java/org/apache/beam/fn/harness/PTransformRunnerFactory.java
delete mode 100644 sdks/java/harness/src/main/java/org/apache/beam/fn/harness/channel/ManagedChannelFactory.java
delete mode 100644 sdks/java/harness/src/main/java/org/apache/beam/fn/harness/channel/SocketAddressFactory.java
delete mode 100644 sdks/java/harness/src/main/java/org/apache/beam/fn/harness/channel/package-info.java
delete mode 100644 sdks/java/harness/src/main/java/org/apache/beam/fn/harness/fake/FakeStepContext.java
delete mode 100644 sdks/java/harness/src/main/java/org/apache/beam/fn/harness/fake/package-info.java
create mode 100644 sdks/java/harness/src/main/java/org/apache/beam/fn/harness/fn/ThrowingBiConsumer.java
create mode 100644 sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BagUserState.java
create mode 100644 sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BeamFnStateClient.java
create mode 100644 sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BeamFnStateGrpcClientCache.java
create mode 100644 sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/LazyCachingIteratorToIterable.java
create mode 100644 sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/StateFetchingIterators.java
create mode 100644 sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/package-info.java
create mode 100644 sdks/java/harness/src/main/java/org/apache/beam/fn/harness/stream/DataStreams.java
delete mode 100644 sdks/java/harness/src/main/java/org/apache/beam/runners/core/BeamFnDataReadRunner.java
delete mode 100644 sdks/java/harness/src/main/java/org/apache/beam/runners/core/BeamFnDataWriteRunner.java
delete mode 100644 sdks/java/harness/src/main/java/org/apache/beam/runners/core/BoundedSourceRunner.java
delete mode 100644 sdks/java/harness/src/main/java/org/apache/beam/runners/core/package-info.java
create mode 100644 sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataReadRunnerTest.java
create mode 100644 sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataWriteRunnerTest.java
create mode 100644 sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BoundedSourceRunnerTest.java
create mode 100644 sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnApiDoFnRunnerTest.java
create mode 100644 sdks/java/harness/src/test/java/org/apache/beam/fn/harness/IdGeneratorTest.java
delete mode 100644 sdks/java/harness/src/test/java/org/apache/beam/fn/harness/channel/ManagedChannelFactoryTest.java
delete mode 100644 sdks/java/harness/src/test/java/org/apache/beam/fn/harness/channel/SocketAddressFactoryTest.java
create mode 100644 sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/BagUserStateTest.java
create mode 100644 sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/BeamFnStateGrpcClientCacheTest.java
create mode 100644 sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/FakeBeamFnStateClient.java
create mode 100644 sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/LazyCachingIteratorToIterableTest.java
create mode 100644 sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/StateFetchingIteratorsTest.java
create mode 100644 sdks/java/harness/src/test/java/org/apache/beam/fn/harness/stream/DataStreamsTest.java
delete mode 100644 sdks/java/harness/src/test/java/org/apache/beam/fn/harness/test/TestExecutors.java
delete mode 100644 sdks/java/harness/src/test/java/org/apache/beam/fn/harness/test/TestExecutorsTest.java
delete mode 100644 sdks/java/harness/src/test/java/org/apache/beam/fn/harness/test/TestStreams.java
delete mode 100644 sdks/java/harness/src/test/java/org/apache/beam/fn/harness/test/TestStreamsTest.java
delete mode 100644 sdks/java/harness/src/test/java/org/apache/beam/runners/core/BeamFnDataReadRunnerTest.java
delete mode 100644 sdks/java/harness/src/test/java/org/apache/beam/runners/core/BeamFnDataWriteRunnerTest.java
delete mode 100644 sdks/java/harness/src/test/java/org/apache/beam/runners/core/BoundedSourceRunnerTest.java
create mode 100644 sdks/java/io/amqp/pom.xml
create mode 100644 sdks/java/io/amqp/src/main/java/org/apache/beam/sdk/io/amqp/AmqpIO.java
create mode 100644 sdks/java/io/amqp/src/main/java/org/apache/beam/sdk/io/amqp/AmqpMessageCoder.java
create mode 100644 sdks/java/io/amqp/src/main/java/org/apache/beam/sdk/io/amqp/AmqpMessageCoderProviderRegistrar.java
create mode 100644 sdks/java/io/amqp/src/main/java/org/apache/beam/sdk/io/amqp/package-info.java
create mode 100644 sdks/java/io/amqp/src/test/java/org/apache/beam/sdk/io/amqp/AmqpIOTest.java
create mode 100644 sdks/java/io/amqp/src/test/java/org/apache/beam/sdk/io/amqp/AmqpMessageCoderTest.java
create mode 100644 sdks/java/io/cassandra/pom.xml
create mode 100644 sdks/java/io/cassandra/src/main/java/org/apache/beam/sdk/io/cassandra/CassandraIO.java
create mode 100644 sdks/java/io/cassandra/src/main/java/org/apache/beam/sdk/io/cassandra/CassandraService.java
create mode 100644 sdks/java/io/cassandra/src/main/java/org/apache/beam/sdk/io/cassandra/CassandraServiceImpl.java
create mode 100644 sdks/java/io/cassandra/src/main/java/org/apache/beam/sdk/io/cassandra/package-info.java
create mode 100644 sdks/java/io/cassandra/src/test/java/org/apache/beam/sdk/io/cassandra/CassandraIOIT.java
create mode 100644 sdks/java/io/cassandra/src/test/java/org/apache/beam/sdk/io/cassandra/CassandraIOTest.java
create mode 100644 sdks/java/io/cassandra/src/test/java/org/apache/beam/sdk/io/cassandra/CassandraServiceImplTest.java
create mode 100644 sdks/java/io/cassandra/src/test/java/org/apache/beam/sdk/io/cassandra/CassandraTestDataSet.java
create mode 100644 sdks/java/io/common/src/test/java/org/apache/beam/sdk/io/common/TestRow.java
create mode 100644 sdks/java/io/elasticsearch-tests/elasticsearch-tests-2/pom.xml
copy sdks/java/io/{elasticsearch => elasticsearch-tests/elasticsearch-tests-2}/src/test/contrib/create_elk_container.sh (100%)
create mode 100644 sdks/java/io/elasticsearch-tests/elasticsearch-tests-2/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOIT.java
create mode 100644 sdks/java/io/elasticsearch-tests/elasticsearch-tests-2/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java
create mode 100644 sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/pom.xml
rename sdks/java/io/{elasticsearch => elasticsearch-tests/elasticsearch-tests-5}/src/test/contrib/create_elk_container.sh (100%)
create mode 100644 sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOIT.java
create mode 100644 sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java
create mode 100644 sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/src/test/java/org/elasticsearch/bootstrap/JarHell.java
create mode 100644 sdks/java/io/elasticsearch-tests/elasticsearch-tests-common/pom.xml
create mode 100644 sdks/java/io/elasticsearch-tests/elasticsearch-tests-common/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticSearchIOTestUtils.java
create mode 100644 sdks/java/io/elasticsearch-tests/elasticsearch-tests-common/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOITCommon.java
create mode 100644 sdks/java/io/elasticsearch-tests/elasticsearch-tests-common/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTestCommon.java
create mode 100644 sdks/java/io/elasticsearch-tests/pom.xml
delete mode 100644 sdks/java/io/elasticsearch/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticSearchIOTestUtils.java
delete mode 100644 sdks/java/io/elasticsearch/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOIT.java
delete mode 100644 sdks/java/io/elasticsearch/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java
delete mode 100644 sdks/java/io/elasticsearch/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchTestDataSet.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableRowIterator.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/CalculateSchemas.java
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/InsertRetryPolicy.java
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ReifyAsIterable.java
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/SchemaAndRecord.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ShardedKey.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ShardedKeyCoder.java
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestinationCoderV2.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TransformingSource.java
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/AdaptiveThrottler.java
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/MovingAverage.java
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/CreateTransactionFn.java
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationGroup.java
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationGroupEncoder.java
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationSizeEstimator.java
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/NaiveSpannerReadFn.java
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/OrderedCode.java
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/ReadOperation.java
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/ReadSpannerSchema.java
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SerializedMutation.java
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SerializedMutationCoder.java
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerAccessor.java
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerConfig.java
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIO.java
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchema.java
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/Transaction.java
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/package-info.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableRowIteratorTest.java
create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/InsertRetryPolicyTest.java
create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/AdaptiveThrottlerTest.java
create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/FakeServiceFactory.java
create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/MutationGroupEncoderTest.java
create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/MutationSizeEstimatorTest.java
create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/OrderedCodeTest.java
create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/RandomUtils.java
create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/ReadSpannerSchemaTest.java
create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIOReadTest.java
create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIOWriteTest.java
create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerReadIT.java
create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchemaTest.java
create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteIT.java
create mode 100644 sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseCoderProviderRegistrar.java
create mode 100644 sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseCoderProviderRegistrarTest.java
create mode 100644 sdks/java/io/hcatalog/pom.xml
create mode 100644 sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/HCatalogIO.java
create mode 100644 sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/package-info.java
create mode 100644 sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/EmbeddedMetastoreService.java
create mode 100644 sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/HCatalogIOTest.java
create mode 100644 sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/HCatalogIOTestUtils.java
create mode 100644 sdks/java/io/hcatalog/src/test/resources/hive-site.xml
delete mode 100644 sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcTestDataSet.java
create mode 100644 sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcTestHelper.java
create mode 100644 sdks/java/io/kafka/README.md
create mode 100644 sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/ProducerSpEL.java
create mode 100644 sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/AWSClientsProvider.java
delete mode 100644 sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisClientProvider.java
delete mode 100644 sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/RoundRobin.java
create mode 100644 sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/ShardReadersPool.java
delete mode 100644 sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/RoundRobinTest.java
create mode 100644 sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/ShardReadersPoolTest.java
create mode 100644 sdks/java/io/redis/pom.xml
create mode 100644 sdks/java/io/redis/src/main/java/org/apache/beam/sdk/io/redis/RedisConnectionConfiguration.java
create mode 100644 sdks/java/io/redis/src/main/java/org/apache/beam/sdk/io/redis/RedisIO.java
create mode 100644 sdks/java/io/redis/src/main/java/org/apache/beam/sdk/io/redis/package-info.java
create mode 100644 sdks/java/io/redis/src/test/java/org/apache/beam/sdk/io/redis/RedisIOTest.java
create mode 100644 sdks/java/io/solr/pom.xml
create mode 100644 sdks/java/io/solr/src/main/java/org/apache/beam/sdk/io/solr/AuthorizedSolrClient.java
create mode 100644 sdks/java/io/solr/src/main/java/org/apache/beam/sdk/io/solr/JavaBinCodecCoder.java
create mode 100644 sdks/java/io/solr/src/main/java/org/apache/beam/sdk/io/solr/SolrIO.java
create mode 100644 sdks/java/io/solr/src/main/java/org/apache/beam/sdk/io/solr/package-info.java
create mode 100644 sdks/java/io/solr/src/test/java/org/apache/beam/sdk/io/solr/JavaBinCodecCoderTest.java
create mode 100644 sdks/java/io/solr/src/test/java/org/apache/beam/sdk/io/solr/SolrIOTest.java
create mode 100644 sdks/java/io/solr/src/test/java/org/apache/beam/sdk/io/solr/SolrIOTestUtils.java
create mode 100644 sdks/java/io/solr/src/test/resources/cloud-minimal/conf/schema.xml
create mode 100644 sdks/java/io/solr/src/test/resources/cloud-minimal/conf/solrconfig.xml
create mode 100644 sdks/java/io/tika/pom.xml
create mode 100644 sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/ParseResult.java
create mode 100644 sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaIO.java
create mode 100644 sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/package-info.java
create mode 100644 sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/ParseResultTest.java
create mode 100644 sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaIOTest.java
create mode 100644 sdks/java/io/tika/src/test/resources/damaged.pdf
create mode 100644 sdks/java/io/tika/src/test/resources/valid/apache-beam-tika-pdf.zip
create mode 100644 sdks/java/io/tika/src/test/resources/valid/apache-beam-tika.odt
create mode 100644 sdks/java/nexmark/pom.xml
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/Main.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/Monitor.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkConfiguration.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkLauncher.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkOptions.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkPerf.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkSuite.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkUtils.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/Auction.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/AuctionBid.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/AuctionCount.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/AuctionPrice.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/Bid.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/BidsPerSession.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/CategoryPrice.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/Done.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/Event.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/IdNameReserve.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/KnownSize.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/NameCityStateId.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/Person.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/SellerPrice.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/package-info.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/package-info.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/AbstractSimulator.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/NexmarkQuery.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/NexmarkQueryModel.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query0.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query0Model.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query1.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query10.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query11.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query12.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query1Model.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query2.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query2Model.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query3.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query3Model.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query4.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query4Model.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query5.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query5Model.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query6.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query6Model.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query7.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query7Model.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query8.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query8Model.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query9.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query9Model.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/WinningBids.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/WinningBidsSimulator.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/package-info.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/sources/BoundedEventSource.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/sources/Generator.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/sources/GeneratorConfig.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/sources/UnboundedEventSource.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/sources/package-info.java
create mode 100644 sdks/java/nexmark/src/main/resources/log4j.properties
create mode 100644 sdks/java/nexmark/src/test/java/org/apache/beam/sdk/nexmark/queries/QueryTest.java
create mode 100644 sdks/java/nexmark/src/test/java/org/apache/beam/sdk/nexmark/sources/BoundedEventSourceTest.java
create mode 100644 sdks/java/nexmark/src/test/java/org/apache/beam/sdk/nexmark/sources/GeneratorTest.java
create mode 100644 sdks/java/nexmark/src/test/java/org/apache/beam/sdk/nexmark/sources/UnboundedEventSourceTest.java
create mode 100644 sdks/python/apache_beam/examples/complete/game/game_stats.py
create mode 100644 sdks/python/apache_beam/examples/complete/game/game_stats_test.py
create mode 100644 sdks/python/apache_beam/examples/complete/game/leader_board.py
create mode 100644 sdks/python/apache_beam/examples/complete/game/leader_board_test.py
create mode 100644 sdks/python/apache_beam/examples/windowed_wordcount.py
create mode 100644 sdks/python/apache_beam/examples/wordcount_fnapi.py
create mode 100644 sdks/python/apache_beam/io/gcp/datastore/v1/adaptive_throttler.py
create mode 100644 sdks/python/apache_beam/io/gcp/datastore/v1/adaptive_throttler_test.py
create mode 100644 sdks/python/apache_beam/io/gcp/datastore/v1/util.py
create mode 100644 sdks/python/apache_beam/io/gcp/datastore/v1/util_test.py
create mode 100644 sdks/python/apache_beam/io/gcp/tests/utils.py
create mode 100644 sdks/python/apache_beam/io/gcp/tests/utils_test.py
create mode 100644 sdks/python/apache_beam/io/vcfio.py
create mode 100644 sdks/python/apache_beam/io/vcfio_test.py
copy sdks/python/apache_beam/{runners/worker => portability}/__init__.py (100%)
create mode 100644 sdks/python/apache_beam/portability/api/__init__.py
delete mode 100644 sdks/python/apache_beam/runners/api/__init__.py
delete mode 100644 sdks/python/apache_beam/runners/api/beam_fn_api_pb2.py
delete mode 100644 sdks/python/apache_beam/runners/api/beam_fn_api_pb2_grpc.py
delete mode 100644 sdks/python/apache_beam/runners/api/beam_runner_api_pb2.py
create mode 100644 sdks/python/apache_beam/runners/dataflow/native_io/streaming_create.py
create mode 100644 sdks/python/apache_beam/runners/dataflow/ptransform_overrides.py
create mode 100644 sdks/python/apache_beam/runners/direct/direct_runner_test.py
delete mode 100644 sdks/python/apache_beam/runners/direct/transform_result.py
create mode 100644 sdks/python/apache_beam/runners/direct/util.py
copy sdks/python/apache_beam/{testing => runners/experimental}/__init__.py (100%)
create mode 100644 sdks/python/apache_beam/runners/experimental/python_rpc_direct/__init__.py
create mode 100644 sdks/python/apache_beam/runners/experimental/python_rpc_direct/python_rpc_direct_runner.py
create mode 100644 sdks/python/apache_beam/runners/experimental/python_rpc_direct/server.py
copy sdks/python/apache_beam/{testing => runners/job}/__init__.py (100%)
create mode 100644 sdks/python/apache_beam/runners/job/manager.py
create mode 100644 sdks/python/apache_beam/runners/job/utils.py
create mode 100644 sdks/python/apache_beam/runners/portability/universal_local_runner.py
create mode 100644 sdks/python/apache_beam/runners/portability/universal_local_runner_main.py
create mode 100644 sdks/python/apache_beam/runners/portability/universal_local_runner_test.py
create mode 100644 sdks/python/apache_beam/runners/worker/bundle_processor.py
create mode 100644 sdks/python/apache_beam/testing/data/vcf/valid-4.0.vcf
create mode 100644 sdks/python/apache_beam/testing/data/vcf/valid-4.0.vcf.bz2
create mode 100644 sdks/python/apache_beam/testing/data/vcf/valid-4.0.vcf.gz
create mode 100644 sdks/python/apache_beam/testing/data/vcf/valid-4.1-large.vcf
create mode 100644 sdks/python/apache_beam/testing/data/vcf/valid-4.1-large.vcf.gz
create mode 100644 sdks/python/apache_beam/testing/data/vcf/valid-4.2.vcf
create mode 100644 sdks/python/apache_beam/testing/data/vcf/valid-4.2.vcf.gz
create mode 100644 sdks/python/apache_beam/testing/test_utils_test.py
create mode 100644 sdks/python/apache_beam/transforms/util_test.py
create mode 100644 sdks/python/apache_beam/typehints/native_type_compatibility.py
create mode 100644 sdks/python/apache_beam/typehints/native_type_compatibility_test.py
create mode 100644 sdks/python/apache_beam/utils/counters_test.py
create mode 100644 sdks/python/apache_beam/utils/plugin.py
create mode 100644 sdks/python/container/Dockerfile
create mode 100644 sdks/python/container/boot.go
create mode 100644 sdks/python/container/pom.xml
create mode 100644 sdks/python/gen_protos.py
create mode 100755 sdks/python/run_validatesrunner.sh
--
To stop receiving notification emails like this one, please contact
['"commits@beam.apache.org" <co...@beam.apache.org>'].
[beam] 01/01: Merge pull request #4107: Merge branch 'master' into
jstorm-runner at commit 727253e
Posted by ke...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kenn pushed a commit to branch jstorm-runner
in repository https://gitbox.apache.org/repos/asf/beam.git
commit ae7357a77e6d75fcabff4ccf7c9373adf12bd28b
Merge: cd9c548 9022ef6
Author: Kenn Knowles <ke...@kennknowles.com>
AuthorDate: Tue Dec 5 00:28:09 2017 -0800
Merge pull request #4107: Merge branch 'master' into jstorm-runner at commit 727253e
.gitattributes | 2 +
.github/PULL_REQUEST_TEMPLATE.md | 16 +-
.gitignore | 10 +-
.test-infra/jenkins/PreCommit_Pipeline.groovy | 129 +
.test-infra/jenkins/common_job_properties.groovy | 216 +-
.test-infra/jenkins/job_00_seed.groovy | 114 +
.test-infra/jenkins/job_beam_Java_Build.groovy | 74 +
.../jenkins/job_beam_Java_CodeHealth.groovy | 39 +
.../jenkins/job_beam_Java_IntegrationTest.groovy | 63 +
.test-infra/jenkins/job_beam_Java_UnitTest.groovy | 40 +
.../job_beam_PerformanceTests_Python.groovy | 58 +
...job_beam_PostCommit_Java_JDKVersionsTest.groovy | 62 +
.../job_beam_PostCommit_Java_MavenInstall.groovy | 21 +-
...eam_PostCommit_Java_MavenInstall_Windows.groovy | 5 +-
...PostCommit_Java_ValidatesRunner_Gearpump.groovy | 2 +-
...stCommit_Python_ValidatesRunner_Dataflow.groovy | 54 +
.../job_beam_PreCommit_Go_MavenInstall.groovy | 56 +
.../job_beam_PreCommit_Java_MavenInstall.groovy | 21 +-
.../jenkins/job_beam_PreCommit_Pipeline.groovy | 84 +
.../job_beam_PreCommit_Python_MavenInstall.groovy | 57 +
.../job_beam_PreCommit_Website_Merge.groovy | 62 +
.../job_beam_PreCommit_Website_Stage.groovy | 3 +
.../jenkins/job_beam_PreCommit_Website_Test.groovy | 3 +
.../jenkins/job_beam_Python_UnitTest.groovy | 40 +
.../job_beam_Release_NightlySnapshot.groovy | 22 +-
.test-infra/jenkins/job_seed.groovy | 53 -
.test-infra/jenkins/job_seed_standalone.groovy | 114 +
.../kubernetes/cassandra/LargeITCluster/setup.sh | 21 +
.../cassandra/LargeITCluster/start-up.sh | 22 -
.../cassandra/LargeITCluster/teardown.sh | 1 -
.../kubernetes/cassandra/SmallITCluster/setup.sh | 22 +
.../cassandra/SmallITCluster/start-up.sh | 23 -
.../cassandra/SmallITCluster/teardown.sh | 1 -
.../elasticsearch/LargeProductionCluster/setup.sh | 21 +
.../LargeProductionCluster/start-up.sh | 22 -
.../LargeProductionCluster/teardown.sh | 1 -
.../elasticsearch/SmallITCluster/setup.sh | 22 +
.../elasticsearch/SmallITCluster/start-up.sh | 23 -
.../elasticsearch/SmallITCluster/teardown.sh | 1 -
.../kubernetes/postgres/pkb-config-local.yml | 34 +
.test-infra/kubernetes/postgres/pkb-config.yml | 32 +
README.md | 4 +-
examples/java/pom.xml | 52 +-
.../apache/beam/examples/WindowedWordCount.java | 1 -
.../java/org/apache/beam/examples/WordCount.java | 4 +
.../examples/common/WriteOneFilePerWindow.java | 59 +-
.../org/apache/beam/examples/complete/TfIdf.java | 17 +-
.../examples/complete/TopWikipediaSessions.java | 24 +-
.../beam/examples/complete/TrafficRoutes.java | 19 +
.../beam/examples/cookbook/BigQueryTornadoes.java | 2 +-
.../examples/cookbook/CombinePerKeyExamples.java | 2 +-
.../beam/examples/cookbook/FilterExamples.java | 2 +-
.../beam/examples/cookbook/JoinExamples.java | 6 +-
.../beam/examples/cookbook/MaxPerKeyExamples.java | 2 +-
.../beam/examples/cookbook/TriggerExample.java | 6 +-
.../beam/examples/DebuggingWordCountTest.java | 11 +-
.../apache/beam/examples/WindowedWordCountIT.java | 4 +-
examples/java8/pom.xml | 55 +-
.../examples/complete/game/injector/Injector.java | 2 +-
.../complete/game/injector/InjectorUtils.java | 2 +-
.../examples/complete/game/utils/WriteToText.java | 49 +-
.../beam/examples/website_snippets/Snippets.java | 87 +
.../examples/complete/game/LeaderBoardTest.java | 2 +
.../examples/website_snippets/SnippetsTest.java | 114 +
examples/pom.xml | 2 +-
model/fn-execution/pom.xml | 114 +
.../fn-execution/src/main/proto/beam_fn_api.proto | 729 ++
.../src/main/proto/beam_provision_api.proto | 99 +
.../model/fnexecution}/v1/standard_coders.yaml | 0
model/job-management/pom.xml | 114 +
.../src/main/proto/beam_artifact_api.proto | 134 +
.../src/main/proto/beam_job_api.proto | 174 +
model/pipeline/pom.xml | 89 +
.../pipeline/src/main/proto/beam_runner_api.proto | 843 ++
model/pipeline/src/main/proto/endpoints.proto | 47 +
.../src/main/proto/standard_window_fns.proto | 54 +
model/pom.xml | 40 +
pom.xml | 601 +-
runners/apex/pom.xml | 38 +-
.../beam/runners/apex/ApexPipelineOptions.java | 2 +-
.../org/apache/beam/runners/apex/ApexRunner.java | 171 +-
.../apache/beam/runners/apex/ApexRunnerResult.java | 4 +-
.../apex/translation/ApexPipelineTranslator.java | 31 +-
.../translation/FlattenPCollectionTranslator.java | 15 +-
.../runners/apex/translation/ParDoTranslator.java | 42 +-
.../apex/translation/TranslationContext.java | 9 +-
.../operators/ApexGroupByKeyOperator.java | 10 +-
.../translation/operators/ApexParDoOperator.java | 59 +-
.../operators/ApexReadUnboundedInputOperator.java | 6 +-
.../apex/translation/utils/ApexStateInternals.java | 17 +-
.../apex/translation/utils/NoOpStepContext.java | 37 +-
.../utils/SerializablePipelineOptions.java | 78 -
.../apex/translation/utils/ValuesSource.java | 6 +-
.../beam/runners/apex/ApexYarnLauncherTest.java | 9 +-
.../runners/apex/examples/UnboundedTextSource.java | 6 +-
.../beam/runners/apex/examples/WordCountTest.java | 8 +-
.../translation/ApexGroupByKeyOperatorTest.java | 6 +-
.../FlattenPCollectionTranslatorTest.java | 1 -
.../apex/translation/GroupByKeyTranslatorTest.java | 6 +-
.../apex/translation/ParDoTranslatorTest.java | 4 +-
.../translation/utils/ApexStateInternalsTest.java | 411 +-
.../apex/translation/utils/CollectionSource.java | 6 +-
.../translation/utils/PipelineOptionsTest.java | 150 -
runners/core-construction-java/pom.xml | 46 +-
.../core/construction/ArtifactServiceStager.java | 244 +
.../core/construction/CoderTranslation.java | 183 +
.../beam/runners/core/construction/Coders.java | 193 -
.../core/construction/CombineTranslation.java | 339 +
.../CreatePCollectionViewTranslation.java | 131 +
.../core/construction/DisplayDataTranslation.java | 39 +
.../core/construction/FlattenTranslator.java | 69 +
.../core/construction/ForwardingPTransform.java | 18 +-
.../core/construction/GroupByKeyTranslation.java | 65 +
.../core/construction/PCollectionTranslation.java | 96 +
.../runners/core/construction/PCollections.java | 97 -
.../core/construction/PTransformMatchers.java | 109 +-
.../core/construction/PTransformReplacements.java | 6 +
.../core/construction/PTransformTranslation.java | 443 +
.../runners/core/construction/PTransforms.java | 110 -
.../core/construction/ParDoTranslation.java | 768 ++
.../construction/PipelineOptionsTranslation.java | 51 +
.../core/construction/PipelineTranslation.java | 195 +
.../runners/core/construction/PrimitiveCreate.java | 14 +-
.../runners/core/construction/ReadTranslation.java | 225 +
.../core/construction/RehydratedComponents.java | 160 +
.../core/construction/RunnerPCollectionView.java | 113 +
.../runners/core/construction/SdkComponents.java | 79 +-
.../construction/SerializablePipelineOptions.java | 74 +
.../runners/core/construction/SplittableParDo.java | 375 +
.../core/construction/TestStreamTranslation.java | 316 +
.../runners/core/construction/TransformInputs.java | 50 +
.../TransformPayloadTranslatorRegistrar.java | 31 +
.../core/construction/TriggerTranslation.java | 336 +
.../beam/runners/core/construction/Triggers.java | 336 -
.../core/construction/WindowIntoTranslation.java | 149 +
.../core/construction/WindowingStrategies.java | 275 -
.../construction/WindowingStrategyTranslation.java | 405 +
.../core/construction/WriteFilesTranslation.java | 335 +
.../core/construction/metrics/MetricFiltering.java | 102 +
.../core/construction/metrics/MetricKey.java | 43 +
.../core/construction/metrics/package-info.java | 22 +
.../beam/runners/core/metrics/MetricFiltering.java | 102 -
.../beam/runners/core/metrics/MetricKey.java | 43 -
.../beam/runners/core/metrics/package-info.java | 22 -
.../construction/ArtifactServiceStagerTest.java | 141 +
.../core/construction/CoderTranslationTest.java | 167 +
.../beam/runners/core/construction/CodersTest.java | 164 -
.../core/construction/CombineTranslationTest.java | 224 +
.../CreatePCollectionViewTranslationTest.java | 134 +
.../construction/ForwardingPTransformTest.java | 17 +-
.../construction/GroupByKeyTranslationTest.java | 44 +
.../InMemoryArtifactStagerService.java | 152 +
.../construction/PCollectionTranslationTest.java | 227 +
.../core/construction/PCollectionsTest.java | 189 -
.../core/construction/PTransformMatchersTest.java | 167 +-
.../construction/PTransformTranslationTest.java | 220 +
.../runners/core/construction/PTransformsTest.java | 188 -
.../core/construction/ParDoTranslationTest.java | 323 +
.../PipelineOptionsTranslationTest.java | 143 +
.../core/construction/PipelineTranslationTest.java | 199 +
.../core/construction/ReadTranslationTest.java | 173 +
.../construction/RehydratedComponentsTest.java | 96 +
.../core/construction/ReplacementOutputsTest.java | 14 +-
.../core/construction/SdkComponentsTest.java | 102 +-
.../SerializablePipelineOptionsTest.java | 89 +
.../core/construction/SplittableParDoTest.java | 158 +
.../construction/TestStreamTranslationTest.java | 130 +
.../core/construction/TransformInputsTest.java | 166 +
.../core/construction/TriggerTranslationTest.java | 112 +
.../runners/core/construction/TriggersTest.java | 111 -
.../UnboundedReadFromBoundedSourceTest.java | 2 +-
.../construction/WindowIntoTranslationTest.java | 127 +
.../core/construction/WindowingStrategiesTest.java | 110 -
.../WindowingStrategyTranslationTest.java | 124 +
.../construction/WriteFilesTranslationTest.java | 195 +
.../construction/metrics/MetricFilteringTest.java | 148 +
.../runners/core/metrics/MetricFilteringTest.java | 148 -
runners/core-java/pom.xml | 31 +-
.../beam/runners/core/BaseExecutionContext.java | 164 -
.../org/apache/beam/runners/core/DoFnRunners.java | 3 +-
.../beam/runners/core/ElementAndRestriction.java | 42 -
.../runners/core/ElementAndRestrictionCoder.java | 88 -
.../apache/beam/runners/core/ExecutionContext.java | 99 -
.../core/GroupAlsoByWindowViaWindowSetNewDoFn.java | 4 +-
.../runners/core/GroupByKeyViaGroupByKeyOnly.java | 15 +-
.../beam/runners/core/InMemoryStateInternals.java | 39 +-
.../beam/runners/core/InMemoryTimerInternals.java | 37 +-
.../runners/core/LateDataDroppingDoFnRunner.java | 33 +-
...TimeBoundedSplittableProcessElementInvoker.java | 78 +-
.../apache/beam/runners/core/ProcessFnRunner.java | 21 +-
.../apache/beam/runners/core/ReduceFnRunner.java | 135 +-
.../apache/beam/runners/core/SideInputHandler.java | 2 +-
.../apache/beam/runners/core/SimpleDoFnRunner.java | 414 +-
.../apache/beam/runners/core/SplittableParDo.java | 602 --
.../core/SplittableParDoViaKeyedWorkItems.java | 435 +
.../core/SplittableProcessElementInvoker.java | 25 +-
.../org/apache/beam/runners/core/StateTable.java | 40 +-
.../org/apache/beam/runners/core/StateTags.java | 16 +
.../org/apache/beam/runners/core/StepContext.java | 33 +
.../apache/beam/runners/core/SystemReduceFn.java | 6 +
.../runners/core/TestInMemoryStateInternals.java | 6 +-
.../apache/beam/runners/core/WatermarkHold.java | 6 +-
.../beam/runners/core/fn/FnApiControlClient.java | 152 +
.../core/fn/FnApiControlClientPoolService.java | 72 +
.../beam/runners/core/fn/FnDataReceiver.java | 37 +
.../apache/beam/runners/core/fn/FnDataService.java | 85 +
.../beam/runners/core/fn/SdkHarnessClient.java | 176 +
.../beam/runners/core/fn/SdkHarnessDoFnRunner.java | 108 +
.../apache/beam/runners/core/fn/package-info.java | 22 +
.../beam/runners/core/metrics/CounterCell.java | 10 +-
.../beam/runners/core/metrics/DirtyState.java | 4 +-
.../runners/core/metrics/DistributionCell.java | 10 +-
.../beam/runners/core/metrics/GaugeCell.java | 11 +-
.../beam/runners/core/metrics/MetricUpdates.java | 1 +
.../runners/core/metrics/MetricsContainerImpl.java | 45 +-
.../core/metrics/MetricsContainerStepMap.java | 2 +
.../core/triggers/AfterAllStateMachine.java | 25 +-
.../AfterDelayFromFirstElementStateMachine.java | 6 +-
.../core/triggers/AfterFirstStateMachine.java | 20 +-
.../core/triggers/AfterPaneStateMachine.java | 6 +-
.../core/triggers/AfterWatermarkStateMachine.java | 15 +-
.../triggers/ExecutableTriggerStateMachine.java | 23 +-
.../runners/core/triggers/NeverStateMachine.java | 5 +-
.../runners/core/triggers/TriggerStateMachine.java | 27 -
.../core/triggers/TriggerStateMachineRunner.java | 3 +-
.../core/triggers/TriggerStateMachines.java | 2 +-
.../core/ElementAndRestrictionCoderTest.java | 127 -
.../runners/core/InMemoryStateInternalsTest.java | 569 +-
...BoundedSplittableProcessElementInvokerTest.java | 47 +-
.../beam/runners/core/ReduceFnRunnerTest.java | 457 +-
.../apache/beam/runners/core/ReduceFnTester.java | 80 +-
.../beam/runners/core/SimpleDoFnRunnerTest.java | 4 +-
.../runners/core/SplittableParDoProcessFnTest.java | 594 ++
.../beam/runners/core/SplittableParDoTest.java | 606 --
.../beam/runners/core/StateInternalsTest.java | 672 ++
.../beam/runners/core/StatefulDoFnRunnerTest.java | 4 +-
.../apache/beam/runners/core/WindowMatchers.java | 15 +
.../core/fn/FnApiControlClientPoolServiceTest.java | 65 +
.../runners/core/fn/FnApiControlClientTest.java | 139 +
.../beam/runners/core/fn/SdkHarnessClientTest.java | 96 +
.../runners/core/fn/SdkHarnessDoFnRunnerTest.java | 73 +
.../core/metrics/MetricsContainerImplTest.java | 10 +
.../core/triggers/AfterFirstStateMachineTest.java | 5 +-
.../triggers/AfterWatermarkStateMachineTest.java | 36 +-
.../core/triggers/StubTriggerStateMachine.java | 7 +-
.../core/triggers/TriggerStateMachineTester.java | 13 +-
.../core/triggers/TriggerStateMachinesTest.java | 2 +-
runners/direct-java/pom.xml | 190 +-
.../direct/BoundedReadEvaluatorFactory.java | 14 +-
.../beam/runners/direct/CommittedResult.java | 12 +-
.../direct/CopyOnAccessInMemoryStateInternals.java | 10 +-
.../runners/direct/DirectExecutionContext.java | 35 +-
...DirectGBKIntoKeyedWorkItemsOverrideFactory.java | 9 +-
.../apache/beam/runners/direct/DirectGraph.java | 62 +-
.../beam/runners/direct/DirectGraphVisitor.java | 64 +-
.../beam/runners/direct/DirectGroupByKey.java | 50 +-
.../direct/DirectGroupByKeyOverrideFactory.java | 14 +-
.../apache/beam/runners/direct/DirectMetrics.java | 4 +-
.../beam/runners/direct/DirectRegistrar.java | 2 +-
.../apache/beam/runners/direct/DirectRunner.java | 119 +-
.../beam/runners/direct/DirectTestOptions.java | 42 +
.../beam/runners/direct/DirectTimerInternals.java | 9 +
.../beam/runners/direct/EmptyInputProvider.java | 8 +-
.../beam/runners/direct/EvaluationContext.java | 72 +-
.../direct/ExecutorServiceParallelExecutor.java | 27 +-
.../direct/GroupAlsoByWindowEvaluatorFactory.java | 28 +-
.../runners/direct/KeyedPValueTrackingVisitor.java | 11 +-
.../beam/runners/direct/MultiStepCombine.java | 439 +
.../apache/beam/runners/direct/ParDoEvaluator.java | 18 +-
.../beam/runners/direct/ParDoEvaluatorFactory.java | 55 +-
.../runners/direct/ParDoMultiOverrideFactory.java | 129 +-
.../beam/runners/direct/ReadEvaluatorFactory.java | 97 +
.../beam/runners/direct/RootInputProvider.java | 7 +-
.../beam/runners/direct/RootProviderRegistry.java | 28 +-
.../apache/beam/runners/direct/SourceShard.java | 33 +
.../SplittableProcessElementsEvaluatorFactory.java | 55 +-
.../direct/StatefulParDoEvaluatorFactory.java | 29 +-
.../runners/direct/TestStreamEvaluatorFactory.java | 46 +-
.../runners/direct/TransformEvaluatorRegistry.java | 135 +-
.../direct/UnboundedReadEvaluatorFactory.java | 31 +-
.../beam/runners/direct/ViewEvaluatorFactory.java | 8 +-
.../beam/runners/direct/ViewOverrideFactory.java | 71 +-
.../beam/runners/direct/WatermarkManager.java | 37 +-
.../runners/direct/WindowEvaluatorFactory.java | 5 +-
.../runners/direct/WriteWithShardingFactory.java | 51 +-
.../direct/BoundedReadEvaluatorFactoryTest.java | 6 +-
.../beam/runners/direct/CommittedResultTest.java | 43 +-
.../CopyOnAccessInMemoryStateInternalsTest.java | 74 +-
.../runners/direct/DirectGraphVisitorTest.java | 13 +-
.../apache/beam/runners/direct/DirectGraphs.java | 17 +-
.../beam/runners/direct/DirectMetricsTest.java | 2 +-
.../beam/runners/direct/DirectRegistrarTest.java | 2 +-
.../beam/runners/direct/DirectRunnerTest.java | 4 +-
.../beam/runners/direct/EvaluationContextTest.java | 31 +-
.../direct/ImmutabilityEnforcementFactoryTest.java | 4 +-
.../beam/runners/direct/MultiStepCombineTest.java | 228 +
.../beam/runners/direct/ParDoEvaluatorTest.java | 11 +-
.../direct/StatefulParDoEvaluatorFactoryTest.java | 65 +-
.../beam/runners/direct/TransformExecutorTest.java | 12 +-
.../direct/UnboundedReadEvaluatorFactoryTest.java | 5 +-
.../runners/direct/ViewEvaluatorFactoryTest.java | 8 +-
.../runners/direct/ViewOverrideFactoryTest.java | 70 +-
.../direct/WatermarkCallbackExecutorTest.java | 1 +
.../beam/runners/direct/WatermarkManagerTest.java | 16 +-
.../runners/direct/WindowEvaluatorFactoryTest.java | 10 +
.../direct/WriteWithShardingFactoryTest.java | 75 +-
runners/flink/pom.xml | 28 +-
.../runners/flink/CreateStreamingFlinkView.java | 156 +
.../flink/FlinkBatchPipelineTranslator.java | 5 +-
.../flink/FlinkBatchTranslationContext.java | 3 +-
.../flink/FlinkPipelineExecutionEnvironment.java | 6 +-
.../beam/runners/flink/FlinkPipelineOptions.java | 22 +
.../org/apache/beam/runners/flink/FlinkRunner.java | 10 +-
.../flink/FlinkStreamingPipelineTranslator.java | 87 +-
.../flink/FlinkStreamingTransformTranslators.java | 412 +-
.../flink/FlinkStreamingTranslationContext.java | 3 +-
.../runners/flink/FlinkStreamingViewOverrides.java | 372 -
.../runners/flink/FlinkTransformOverrides.java | 53 +
.../translation/functions/FlinkAssignContext.java | 17 +-
.../translation/functions/FlinkDoFnFunction.java | 24 +-
.../FlinkMergingNonShuffleReduceFunction.java | 8 +-
.../functions/FlinkNoOpStepContext.java | 36 +-
.../functions/FlinkPartialReduceFunction.java | 8 +-
.../translation/functions/FlinkReduceFunction.java | 8 +-
.../functions/FlinkStatefulDoFnFunction.java | 20 +-
.../translation/types/CoderTypeSerializer.java | 78 +
.../translation/types/EncodedValueSerializer.java | 18 +-
.../utils/SerializedPipelineOptions.java | 77 -
.../translation/wrappers/SourceInputFormat.java | 10 +-
.../wrappers/streaming/DoFnOperator.java | 570 +-
.../wrappers/streaming/SplittableDoFnOperator.java | 38 +-
.../wrappers/streaming/WindowDoFnOperator.java | 8 +-
.../streaming/io/BoundedSourceWrapper.java | 10 +-
.../streaming/io/UnboundedSocketSource.java | 2 +-
.../streaming/io/UnboundedSourceWrapper.java | 19 +-
.../state/FlinkBroadcastStateInternals.java | 29 +-
.../streaming/state/FlinkSplitStateInternals.java | 8 +-
.../streaming/state/FlinkStateInternals.java | 503 +-
.../beam/runners/flink/PipelineOptionsTest.java | 191 +-
.../runners/flink/streaming/DoFnOperatorTest.java | 332 +-
.../FlinkBroadcastStateInternalsTest.java | 242 +-
.../streaming/FlinkKeyGroupStateInternalsTest.java | 359 +-
.../streaming/FlinkSplitStateInternalsTest.java | 132 +-
.../flink/streaming/FlinkStateInternalsTest.java | 343 +-
.../flink/streaming/TestCountingSource.java | 53 +-
.../streaming/UnboundedSourceWrapperTest.java | 330 +-
.../translation/types/CoderTypeSerializerTest.java | 79 +
runners/gcp/gcemd/Dockerfile | 30 +
runners/gcp/gcemd/main.go | 85 +
runners/gcp/gcemd/pom.xml | 154 +
runners/gcp/gcsproxy/Dockerfile | 30 +
runners/gcp/gcsproxy/main.go | 91 +
runners/gcp/gcsproxy/pom.xml | 154 +
runners/gcp/pom.xml | 38 +
runners/gearpump/README.md | 61 +
runners/gearpump/pom.xml | 277 +
.../runners/gearpump/GearpumpPipelineOptions.java | 67 +
.../runners/gearpump/GearpumpPipelineResult.java | 110 +
.../beam/runners/gearpump/GearpumpRunner.java | 119 +
.../runners/gearpump/GearpumpRunnerRegistrar.java | 64 +
.../beam/runners/gearpump/TestGearpumpRunner.java | 63 +
.../apache/beam/runners/gearpump/package-info.java | 22 +
.../CreateGearpumpPCollectionViewTranslator.java | 45 +
.../translators/CreateStreamingGearpumpView.java | 156 +
.../translators/FlattenPCollectionsTranslator.java | 83 +
.../translators/GearpumpPipelineTranslator.java | 143 +
.../gearpump/translators/GroupByKeyTranslator.java | 258 +
.../translators/ParDoMultiOutputTranslator.java | 103 +
.../translators/ReadBoundedTranslator.java | 46 +
.../translators/ReadUnboundedTranslator.java | 48 +
.../gearpump/translators/TransformTranslator.java | 30 +
.../gearpump/translators/TranslationContext.java | 105 +
.../translators/WindowAssignTranslator.java | 103 +
.../translators/functions/DoFnFunction.java | 193 +
.../translators/functions/package-info.java | 22 +
.../translators/io/BoundedSourceWrapper.java | 45 +
.../gearpump/translators/io/GearpumpSource.java | 114 +
.../translators/io/UnboundedSourceWrapper.java | 46 +
.../gearpump/translators/io/ValuesSource.java | 165 +
.../gearpump/translators/io/package-info.java | 22 +
.../runners/gearpump/translators/package-info.java | 22 +
.../translators/utils/DoFnRunnerFactory.java | 83 +
.../translators/utils/NoOpStepContext.java | 41 +
.../translators/utils/TranslatorUtils.java | 198 +
.../gearpump/translators/utils/package-info.java | 22 +
.../gearpump/GearpumpRunnerRegistrarTest.java | 55 +
.../beam/runners/gearpump/PipelineOptionsTest.java | 73 +
...reateGearpumpPCollectionViewTranslatorTest.java | 56 +
.../FlattenPCollectionsTranslatorTest.java | 155 +
.../translators/GroupByKeyTranslatorTest.java | 152 +
.../translators/ReadBoundedTranslatorTest.java | 70 +
.../translators/ReadUnboundedTranslatorTest.java | 70 +
.../translators/WindowAssignTranslatorTest.java | 110 +
.../translators/io/GearpumpSourceTest.java | 99 +
.../gearpump/translators/io/ValueSoureTest.java | 79 +
.../translators/utils/TranslatorUtilsTest.java | 73 +
runners/google-cloud-dataflow-java/pom.xml | 174 +-
.../beam/runners/dataflow/AssignWindows.java | 90 -
.../dataflow/BatchStatefulParDoOverrides.java | 75 +-
.../beam/runners/dataflow/BatchViewOverrides.java | 201 +-
.../beam/runners/dataflow/CreateDataflowView.java | 7 +-
.../beam/runners/dataflow/DataflowMetrics.java | 306 +-
.../beam/runners/dataflow/DataflowPipelineJob.java | 18 +-
.../dataflow/DataflowPipelineTranslator.java | 143 +-
.../beam/runners/dataflow/DataflowRunner.java | 489 +-
.../beam/runners/dataflow/DataflowRunnerInfo.java | 91 +-
.../dataflow/PrimitiveParDoSingleFactory.java | 14 +-
.../beam/runners/dataflow/ReadTranslator.java | 7 +-
.../runners/dataflow/SplittableParDoOverrides.java | 76 +
.../runners/dataflow/StreamingViewOverrides.java | 10 +-
.../beam/runners/dataflow/TransformTranslator.java | 30 +-
.../dataflow/options/CloudDebuggerOptions.java | 3 +-
.../options/DataflowPipelineDebugOptions.java | 23 +-
.../dataflow/options/DataflowPipelineOptions.java | 8 +
.../options/DataflowPipelineWorkerPoolOptions.java | 3 +
.../dataflow/options/DataflowProfilingOptions.java | 3 +-
.../options/DataflowWorkerLoggingOptions.java | 6 +
.../runners/dataflow/util/CloudObjectKinds.java | 1 +
.../dataflow/util/CloudObjectTranslators.java | 5 +-
...DefaultCoderCloudObjectTranslatorRegistrar.java | 2 +
.../beam/runners/dataflow/util/DoFnInfo.java | 129 -
.../beam/runners/dataflow/util/GcsStager.java | 53 +-
.../beam/runners/dataflow/util/MonitoringUtil.java | 18 +-
.../beam/runners/dataflow/util/PackageUtil.java | 581 +-
.../beam/runners/dataflow/util/PropertyNames.java | 47 +-
.../runners/dataflow/util/RandomAccessData.java | 2 +-
.../apache/beam/runners/dataflow/util/Stager.java | 32 +-
.../beam/runners/dataflow/util/TimeUtil.java | 24 +-
.../dataflow/BatchStatefulParDoOverridesTest.java | 46 +-
.../beam/runners/dataflow/DataflowMetricsTest.java | 215 +-
.../dataflow/DataflowPipelineTranslatorTest.java | 214 +-
.../runners/dataflow/DataflowRunnerInfoTest.java | 9 +
.../beam/runners/dataflow/DataflowRunnerTest.java | 334 +-
.../transforms/DataflowGroupByKeyTest.java | 12 +-
.../dataflow/transforms/DataflowViewTest.java | 14 +-
.../runners/dataflow/util/MonitoringUtilTest.java | 4 +-
.../runners/dataflow/util/PackageUtilTest.java | 124 +-
.../beam/runners/dataflow/util/TimeUtilTest.java | 6 +
runners/java-fn-execution/pom.xml | 135 +
.../beam/runners/fnexecution/ServerFactory.java | 104 +
.../fnexecution/control/FnApiControlClient.java | 148 +
.../control/FnApiControlClientPoolService.java | 66 +
.../fnexecution/control/SdkHarnessClient.java | 173 +
.../runners/fnexecution/control/package-info.java | 23 +
.../runners/fnexecution/data/FnDataReceiver.java | 27 +
.../runners/fnexecution/data/package-info.java | 23 +
.../beam/runners/fnexecution/package-info.java | 23 +
.../runners/fnexecution/ServerFactoryTest.java | 153 +
.../control/FnApiControlClientPoolServiceTest.java | 65 +
.../control/FnApiControlClientTest.java | 139 +
.../fnexecution/control/SdkHarnessClientTest.java | 96 +
runners/jstorm/pom.xml | 2 +-
.../jstorm/translation/DefaultStepContext.java | 37 +-
.../jstorm/translation/JStormMetricResults.java | 4 +-
.../translation/JStormPipelineTranslator.java | 4 +-
.../jstorm/translation/JStormTimerInternals.java | 3 +
.../jstorm/translation/MetricsReporter.java | 2 +-
runners/local-artifact-service-java/pom.xml | 116 +
.../LocalFileSystemArtifactStagerService.java | 279 +
.../apache/beam/artifact/local/package-info.java | 22 +
.../LocalFileSystemArtifactStagerServiceTest.java | 301 +
runners/pom.xml | 15 +-
runners/reference/job-server/pom.xml | 82 +
.../reference/job/ReferenceRunnerJobServer.java | 77 +
.../reference/job/ReferenceRunnerJobService.java | 79 +
.../beam/runners/reference/job/package-info.java | 23 +
.../job/ReferenceRunnerJobServiceTest.java | 34 +
runners/reference/pom.xml | 39 +
runners/spark/pom.xml | 94 +-
.../runners/spark/SparkNativePipelineVisitor.java | 3 +-
.../org/apache/beam/runners/spark/SparkRunner.java | 14 +-
.../apache/beam/runners/spark/TestSparkRunner.java | 2 +-
.../spark/aggregators/NamedAggregators.java | 93 -
.../apache/beam/runners/spark/io/CreateStream.java | 115 +-
.../beam/runners/spark/io/MicrobatchSource.java | 4 +-
.../beam/runners/spark/io/SourceDStream.java | 20 +-
.../apache/beam/runners/spark/io/SourceRDD.java | 22 +-
.../runners/spark/io/SparkUnboundedSource.java | 8 +-
.../SparkGroupAlsoByWindowViaWindowSet.java | 821 +-
.../spark/stateful/SparkTimerInternals.java | 39 +-
.../runners/spark/stateful/StateSpecFunctions.java | 10 +-
.../runners/spark/translation/BoundedDataset.java | 17 +-
.../beam/runners/spark/translation/Dataset.java | 3 +-
.../spark/translation/EvaluationContext.java | 38 +-
.../spark/translation/MultiDoFnFunction.java | 126 +-
.../spark/translation/SparkAbstractCombineFn.java | 9 +-
.../spark/translation/SparkContextFactory.java | 2 -
.../spark/translation/SparkGlobalCombineFn.java | 5 +-
.../SparkGroupAlsoByWindowViaOutputBufferFn.java | 13 +-
.../spark/translation/SparkKeyedCombineFn.java | 5 +-
.../spark/translation/SparkProcessContext.java | 50 +-
.../spark/translation/SparkRuntimeContext.java | 98 -
.../spark/translation/StorageLevelPTransform.java | 43 -
.../spark/translation/TransformTranslator.java | 209 +-
.../spark/translation/TranslationUtils.java | 78 +
.../streaming/StreamingTransformTranslator.java | 110 +-
.../translation/streaming/UnboundedDataset.java | 27 +-
.../streaming/WatermarkSyncedDStream.java | 149 +
.../runners/spark/util/GlobalWatermarkHolder.java | 307 +-
.../util/SinglePrimitiveOutputPTransform.java | 51 -
.../runners/spark/GlobalWatermarkHolderTest.java | 18 +-
.../beam/runners/spark/SparkPipelineStateTest.java | 4 +-
.../runners/spark/SparkRunnerDebuggerTest.java | 31 +-
.../spark/stateful/SparkStateInternalsTest.java | 66 +
.../spark/translation/SparkRuntimeContextTest.java | 122 -
.../spark/translation/StorageLevelTest.java | 75 -
.../translation/streaming/CreateStreamTest.java | 33 +-
.../streaming/TrackStreamingSourcesTest.java | 14 +-
runners/spark/src/test/resources/log4j.properties | 11 +-
sdks/CONTAINERS.md | 162 +
sdks/common/fn-api/pom.xml | 104 -
.../common/fn-api/src/main/proto/beam_fn_api.proto | 761 --
sdks/common/pom.xml | 39 -
sdks/common/runner-api/pom.xml | 86 -
.../src/main/proto/beam_runner_api.proto | 711 --
sdks/go/BUILD.md | 63 +
sdks/go/cmd/beamctl/cmd/artifact.go | 98 +
sdks/go/cmd/beamctl/cmd/root.go | 56 +
sdks/go/cmd/beamctl/main.go | 31 +
sdks/go/descriptor.xml | 29 +
sdks/go/pkg/beam/artifact/gcsproxy/retrieval.go | 155 +
sdks/go/pkg/beam/artifact/gcsproxy/staging.go | 200 +
sdks/go/pkg/beam/artifact/materialize.go | 240 +
sdks/go/pkg/beam/artifact/materialize_test.go | 238 +
sdks/go/pkg/beam/artifact/server_test.go | 213 +
sdks/go/pkg/beam/artifact/stage.go | 238 +
sdks/go/pkg/beam/artifact/stage_test.go | 98 +
.../beam/model/fnexecution_v1/beam_fn_api.pb.go | 2729 +++++
.../model/fnexecution_v1/beam_provision_api.pb.go | 306 +
sdks/go/pkg/beam/model/gen.go | 22 +
.../model/jobmanagement_v1/beam_artifact_api.pb.go | 690 ++
.../beam/model/jobmanagement_v1/beam_job_api.pb.go | 903 ++
.../beam/model/pipeline_v1/beam_runner_api.pb.go | 3491 +++++++
sdks/go/pkg/beam/model/pipeline_v1/endpoints.pb.go | 160 +
.../model/pipeline_v1/standard_window_fns.pb.go | 120 +
sdks/go/pkg/beam/provision/provision_test.go | 54 +
sdks/go/pkg/beam/provision/provison.go | 80 +
sdks/go/pkg/beam/util/errorx/guarded.go | 47 +
sdks/go/pkg/beam/util/execx/exec.go | 33 +
sdks/go/pkg/beam/util/gcsx/gcs.go | 88 +
sdks/go/pkg/beam/util/grpcx/dial.go | 37 +
sdks/go/pkg/beam/util/grpcx/metadata.go | 55 +
sdks/go/pkg/beam/util/syscallx/syscall.go | 27 +
sdks/go/pkg/beam/util/syscallx/syscall_default.go | 28 +
sdks/go/pkg/beam/util/syscallx/syscall_linux.go | 38 +
sdks/go/pom.xml | 163 +
sdks/java/build-tools/pom.xml | 2 +-
.../src/main/resources/beam/checkstyle.xml | 8 +
.../src/main/resources/beam/findbugs-filter.xml | 74 +-
.../main/resources/docker/file/openjdk7/Dockerfile | 49 +
.../docker/file/openjdk7/docker-entrypoint.sh | 24 +
.../main/resources/docker/file/openjdk8/Dockerfile | 49 +
.../docker/file/openjdk8/docker-entrypoint.sh | 24 +
.../main/resources/docker/git/openjdk8/Dockerfile | 53 +
.../docker/git/openjdk8/docker-entrypoint.sh | 22 +
.../resources/docker/release/python2/Dockerfile | 21 +
sdks/java/container/Dockerfile | 28 +
sdks/java/container/boot.go | 134 +
sdks/java/container/pom.xml | 184 +
sdks/java/core/pom.xml | 186 +-
.../main/java/org/apache/beam/sdk/Pipeline.java | 73 +-
.../java/org/apache/beam/sdk/PipelineResult.java | 14 +-
.../apache/beam/sdk/annotations/Experimental.java | 13 +-
.../apache/beam/sdk/annotations/package-info.java | 4 +
.../apache/beam/sdk/coders/BeamRecordCoder.java | 111 +
.../org/apache/beam/sdk/coders/BooleanCoder.java | 59 +
.../java/org/apache/beam/sdk/coders/Coder.java | 12 +-
.../org/apache/beam/sdk/coders/CoderRegistry.java | 29 +
.../org/apache/beam/sdk/coders/DefaultCoder.java | 19 +-
.../org/apache/beam/sdk/coders/InstantCoder.java | 81 +-
.../apache/beam/sdk/coders/LengthPrefixCoder.java | 3 +-
.../apache/beam/sdk/coders/SerializableCoder.java | 7 +-
.../apache/beam/sdk/coders/ShardedKeyCoder.java | 66 +
.../apache/beam/sdk/coders/StructuredCoder.java | 42 +-
.../java/org/apache/beam/sdk/coders/VoidCoder.java | 12 +-
.../org/apache/beam/sdk/coders/package-info.java | 4 +
.../main/java/org/apache/beam/sdk/io/AvroIO.java | 1216 ++-
.../main/java/org/apache/beam/sdk/io/AvroSink.java | 95 +-
.../java/org/apache/beam/sdk/io/AvroSource.java | 393 +-
.../java/org/apache/beam/sdk/io/AvroUtils.java | 40 +
.../org/apache/beam/sdk/io/BlockBasedSource.java | 31 +-
.../sdk/io/BoundedReadFromUnboundedSource.java | 12 +-
.../org/apache/beam/sdk/io/CompressedSource.java | 310 +-
.../java/org/apache/beam/sdk/io/Compression.java | 228 +
.../beam/sdk/io/ConstantAvroDestination.java | 130 +
.../org/apache/beam/sdk/io/CountingSource.java | 49 +-
.../apache/beam/sdk/io/DefaultFilenamePolicy.java | 378 +-
.../beam/sdk/io/DynamicAvroDestinations.java | 46 +
.../beam/sdk/io/DynamicFileDestinations.java | 150 +
.../java/org/apache/beam/sdk/io/FileBasedSink.java | 833 +-
.../org/apache/beam/sdk/io/FileBasedSource.java | 81 +-
.../main/java/org/apache/beam/sdk/io/FileIO.java | 450 +
.../java/org/apache/beam/sdk/io/FileSystems.java | 69 +-
.../org/apache/beam/sdk/io/GenerateSequence.java | 3 +-
.../org/apache/beam/sdk/io/LocalFileSystem.java | 45 +-
.../org/apache/beam/sdk/io/OffsetBasedSource.java | 22 +-
.../src/main/java/org/apache/beam/sdk/io/Read.java | 27 +-
.../beam/sdk/io/ReadAllViaFileBasedSource.java | 114 +
.../org/apache/beam/sdk/io/ReadableFileCoder.java | 50 +
.../main/java/org/apache/beam/sdk/io/Source.java | 28 +-
.../java/org/apache/beam/sdk/io/TFRecordIO.java | 228 +-
.../main/java/org/apache/beam/sdk/io/TextIO.java | 1054 +-
.../main/java/org/apache/beam/sdk/io/TextSink.java | 22 +-
.../java/org/apache/beam/sdk/io/TextSource.java | 120 +-
.../java/org/apache/beam/sdk/io/WriteFiles.java | 782 +-
.../org/apache/beam/sdk/io/WriteFilesResult.java | 81 +
.../apache/beam/sdk/io/fs/EmptyMatchTreatment.java | 46 +
.../org/apache/beam/sdk/io/fs/MatchResult.java | 5 +-
.../org/apache/beam/sdk/io/fs/MetadataCoder.java | 63 +
.../org/apache/beam/sdk/io/fs/ResourceIdCoder.java | 56 +
.../java/org/apache/beam/sdk/io/package-info.java | 2 +-
.../beam/sdk/io/range/ByteKeyRangeTracker.java | 27 +-
.../org/apache/beam/sdk/io/range/OffsetRange.java | 101 +
.../beam/sdk/io/range/OffsetRangeTracker.java | 25 +-
.../org/apache/beam/sdk/metrics/MetricName.java | 7 +
.../apache/beam/sdk/metrics/MetricsContainer.java | 3 +-
.../org/apache/beam/sdk/metrics/package-info.java | 4 +
.../options/DefaultPipelineOptionsRegistrar.java | 2 +
.../beam/sdk/options/ExperimentalOptions.java | 38 +
.../apache/beam/sdk/options/PipelineOptions.java | 43 +-
.../beam/sdk/options/PipelineOptionsFactory.java | 63 +-
.../beam/sdk/options/PipelineOptionsValidator.java | 34 +-
.../beam/sdk/options/ProxyInvocationHandler.java | 19 +-
.../apache/beam/sdk/options/SdkHarnessOptions.java | 173 +
.../org/apache/beam/sdk/options/ValueProvider.java | 34 +-
.../apache/beam/sdk/options/ValueProviders.java | 25 +-
.../java/org/apache/beam/sdk/package-info.java | 4 +
.../beam/sdk/runners/TransformHierarchy.java | 247 +-
.../org/apache/beam/sdk/runners/package-info.java | 4 +
.../java/org/apache/beam/sdk/state/BagState.java | 6 +
.../org/apache/beam/sdk/state/CombiningState.java | 5 +
.../org/apache/beam/sdk/state/GroupingState.java | 12 +-
.../java/org/apache/beam/sdk/state/MapState.java | 20 +-
.../org/apache/beam/sdk/state/ReadableState.java | 6 +
.../org/apache/beam/sdk/state/ReadableStates.java | 4 +-
.../java/org/apache/beam/sdk/state/SetState.java | 10 +-
.../java/org/apache/beam/sdk/state/StateSpec.java | 53 +
.../java/org/apache/beam/sdk/state/StateSpecs.java | 57 +-
.../org/apache/beam/sdk/state/package-info.java | 4 +
.../apache/beam/sdk/testing/CombineFnTester.java | 147 +
.../beam/sdk/testing/FileChecksumMatcher.java | 41 +-
.../org/apache/beam/sdk/testing/LargeKeys.java | 48 +
.../java/org/apache/beam/sdk/testing/PAssert.java | 79 +-
.../apache/beam/sdk/testing/PaneExtractors.java | 25 +-
.../apache/beam/sdk/testing/SourceTestUtils.java | 23 +-
.../org/apache/beam/sdk/testing/StaticWindows.java | 17 +
.../org/apache/beam/sdk/testing/StreamingIT.java | 4 +
.../apache/beam/sdk/testing/SuccessOrFailure.java | 24 +-
.../org/apache/beam/sdk/testing/TestPipeline.java | 121 +-
.../org/apache/beam/sdk/testing/TestStream.java | 77 +-
.../beam/sdk/testing/UsesCustomWindowMerging.java | 23 +
.../apache/beam/sdk/testing/WindowFnTestUtils.java | 141 +-
.../apache/beam/sdk/testing/WindowSupplier.java | 4 +-
.../org/apache/beam/sdk/testing/package-info.java | 5 +
.../beam/sdk/transforms/ApproximateQuantiles.java | 14 +-
.../beam/sdk/transforms/ApproximateUnique.java | 4 +-
.../org/apache/beam/sdk/transforms/Combine.java | 63 +-
.../org/apache/beam/sdk/transforms/CombineFns.java | 2 +
.../org/apache/beam/sdk/transforms/Contextful.java | 127 +
.../java/org/apache/beam/sdk/transforms/Count.java | 10 +
.../org/apache/beam/sdk/transforms/Create.java | 119 +-
.../org/apache/beam/sdk/transforms/Distinct.java | 82 +-
.../java/org/apache/beam/sdk/transforms/DoFn.java | 66 +-
.../org/apache/beam/sdk/transforms/DoFnTester.java | 48 +-
.../org/apache/beam/sdk/transforms/Filter.java | 26 +-
.../beam/sdk/transforms/FlatMapElements.java | 150 +-
.../org/apache/beam/sdk/transforms/Flatten.java | 22 +-
.../org/apache/beam/sdk/transforms/GroupByKey.java | 12 +-
.../beam/sdk/transforms/GroupIntoBatches.java | 12 +-
.../org/apache/beam/sdk/transforms/Latest.java | 2 +-
.../apache/beam/sdk/transforms/MapElements.java | 77 +-
.../java/org/apache/beam/sdk/transforms/Max.java | 9 +-
.../java/org/apache/beam/sdk/transforms/Min.java | 12 +-
.../org/apache/beam/sdk/transforms/PTransform.java | 28 +-
.../java/org/apache/beam/sdk/transforms/ParDo.java | 110 +-
.../apache/beam/sdk/transforms/Requirements.java | 61 +
.../org/apache/beam/sdk/transforms/Reshuffle.java | 47 +
.../beam/sdk/transforms/SerializableFunctions.java | 50 +
.../java/org/apache/beam/sdk/transforms/Sum.java | 30 +
.../java/org/apache/beam/sdk/transforms/Top.java | 41 +-
.../java/org/apache/beam/sdk/transforms/View.java | 55 +-
.../java/org/apache/beam/sdk/transforms/Watch.java | 1066 ++
.../org/apache/beam/sdk/transforms/WithKeys.java | 12 +-
.../beam/sdk/transforms/display/DisplayData.java | 25 +-
.../beam/sdk/transforms/display/package-info.java | 4 +
.../beam/sdk/transforms/join/CoGbkResult.java | 15 +-
.../sdk/transforms/join/KeyedPCollectionTuple.java | 10 +-
.../beam/sdk/transforms/join/RawUnionValue.java | 8 +-
.../beam/sdk/transforms/join/package-info.java | 4 +
.../apache/beam/sdk/transforms/package-info.java | 4 +
.../reflect/ByteBuddyDoFnInvokerFactory.java | 30 +-
.../reflect/ByteBuddyOnTimerInvokerFactory.java | 73 +-
.../beam/sdk/transforms/reflect/DoFnInvoker.java | 65 +-
.../beam/sdk/transforms/reflect/DoFnInvokers.java | 9 -
.../beam/sdk/transforms/reflect/DoFnSignature.java | 33 +-
.../sdk/transforms/reflect/DoFnSignatures.java | 45 +-
.../transforms/reflect/OnTimerMethodSpecifier.java | 37 +
.../beam/sdk/transforms/reflect/package-info.java | 3 +
.../sdk/transforms/splittabledofn/OffsetRange.java | 77 -
.../splittabledofn/OffsetRangeTracker.java | 16 +-
.../splittabledofn/RestrictionTracker.java | 11 +-
.../transforms/splittabledofn/package-info.java | 4 +
.../sdk/transforms/windowing/BoundedWindow.java | 20 +-
.../sdk/transforms/windowing/CalendarWindows.java | 36 +
.../sdk/transforms/windowing/FixedWindows.java | 11 +
.../sdk/transforms/windowing/GlobalWindows.java | 22 +-
.../windowing/IncompatibleWindowException.java | 38 +
.../sdk/transforms/windowing/InvalidWindows.java | 11 +
.../windowing/MergeOverlappingIntervalWindows.java | 4 +-
.../transforms/windowing/PartitioningWindowFn.java | 5 +
.../beam/sdk/transforms/windowing/Sessions.java | 11 +
.../sdk/transforms/windowing/SlidingWindows.java | 16 +
.../beam/sdk/transforms/windowing/Trigger.java | 20 +-
.../beam/sdk/transforms/windowing/Window.java | 47 +-
.../beam/sdk/transforms/windowing/WindowFn.java | 32 +
.../sdk/transforms/windowing/package-info.java | 4 +
.../java/org/apache/beam/sdk/util/ApiSurface.java | 2 +
.../java/org/apache/beam/sdk/util/ClassPath.java | 8 +-
.../java/org/apache/beam/sdk/util/CoderUtils.java | 2 +-
.../java/org/apache/beam/sdk/util/DoFnInfo.java | 104 +
.../org/apache/beam/sdk/util/IdentityWindowFn.java | 17 +-
.../apache/beam/sdk/util/MutationDetectors.java | 79 +-
.../beam/sdk/util/SerializableThrowable.java | 49 +
.../apache/beam/sdk/util/SerializableUtils.java | 69 +-
.../org/apache/beam/sdk/values/BeamRecord.java | 319 +
.../org/apache/beam/sdk/values/BeamRecordType.java | 96 +
.../org/apache/beam/sdk/values/PCollection.java | 21 +-
.../apache/beam/sdk/values/PCollectionTuple.java | 10 +-
.../apache/beam/sdk/values/PCollectionViews.java | 37 +
.../java/org/apache/beam/sdk/values/PValue.java | 4 +-
.../org/apache/beam/sdk/values/PValueBase.java | 12 -
.../org/apache/beam/sdk/values/ShardedKey.java | 65 +
.../org/apache/beam/sdk/values/TypeDescriptor.java | 64 +-
.../apache/beam/sdk/values/TypeDescriptors.java | 127 +-
.../apache/beam/sdk/values/WindowingStrategy.java | 46 +-
.../java/org/apache/beam/sdk/PipelineTest.java | 51 +-
.../test/java/org/apache/beam/sdk/TestUtils.java | 88 -
.../org/apache/beam/sdk/coders/AvroCoderTest.java | 39 +-
.../apache/beam/sdk/coders/CoderRegistryTest.java | 69 +
.../apache/beam/sdk/coders/CommonCoderTest.java | 4 +-
.../apache/beam/sdk/coders/DefaultCoderTest.java | 3 +-
.../org/apache/beam/sdk/coders/VoidCoderTest.java | 11 +-
.../java/org/apache/beam/sdk/io/AvroIOTest.java | 917 +-
.../apache/beam/sdk/io/AvroIOTransformTest.java | 324 -
.../org/apache/beam/sdk/io/AvroSourceTest.java | 59 +-
.../apache/beam/sdk/io/CompressedSourceTest.java | 138 +-
.../beam/sdk/io/DefaultFilenamePolicyTest.java | 108 +-
.../sdk/io/DrunkWritableByteChannelFactory.java | 2 +-
.../org/apache/beam/sdk/io/FileBasedSinkTest.java | 202 +-
.../apache/beam/sdk/io/FileBasedSourceTest.java | 56 +-
.../java/org/apache/beam/sdk/io/FileIOTest.java | 313 +
.../org/apache/beam/sdk/io/FileSystemsTest.java | 15 +-
.../apache/beam/sdk/io/LocalFileSystemTest.java | 49 +
.../apache/beam/sdk/io/OffsetBasedSourceTest.java | 5 +-
.../test/java/org/apache/beam/sdk/io/ReadTest.java | 10 +-
.../java/org/apache/beam/sdk/io/SimpleSink.java | 77 +-
.../org/apache/beam/sdk/io/TFRecordIOTest.java | 35 +-
.../org/apache/beam/sdk/io/TextIOReadTest.java | 860 ++
.../java/org/apache/beam/sdk/io/TextIOTest.java | 1104 +-
.../org/apache/beam/sdk/io/TextIOWriteTest.java | 643 ++
.../org/apache/beam/sdk/io/WriteFilesTest.java | 479 +-
.../beam/sdk/io/range/ByteKeyRangeTrackerTest.java | 23 +
.../beam/sdk/metrics/MetricResultsMatchers.java | 2 +-
.../org/apache/beam/sdk/metrics/MetricsTest.java | 28 +
.../sdk/options/PipelineOptionsFactoryTest.java | 39 +
.../beam/sdk/options/PipelineOptionsTest.java | 11 +
.../sdk/options/PipelineOptionsValidatorTest.java | 44 +
.../sdk/options/ProxyInvocationHandlerTest.java | 23 +-
.../beam/sdk/options/SdkHarnessOptionsTest.java | 76 +
.../apache/beam/sdk/options/ValueProviderTest.java | 38 +-
.../beam/sdk/runners/TransformHierarchyTest.java | 216 +-
.../apache/beam/sdk/runners/TransformTreeTest.java | 19 +-
.../sdk/runners/dataflow/TestCountingSource.java | 5 +-
.../beam/sdk/testing/CombineFnTesterTest.java | 276 +
.../sdk/testing/InterceptingUrlClassLoader.java | 57 +
.../org/apache/beam/sdk/testing/PAssertTest.java | 41 +-
.../beam/sdk/testing/PCollectionViewTesting.java | 8 +
.../beam/sdk/testing/PaneExtractorsTest.java | 7 +-
.../apache/beam/sdk/testing/StaticWindowsTest.java | 12 +
.../apache/beam/sdk/testing/TestPipelineTest.java | 75 +-
.../sdk/transforms/ApproximateQuantilesTest.java | 530 +-
.../apache/beam/sdk/transforms/CombineTest.java | 377 +-
.../org/apache/beam/sdk/transforms/CreateTest.java | 70 +-
.../apache/beam/sdk/transforms/DistinctTest.java | 130 +-
.../apache/beam/sdk/transforms/DoFnTesterTest.java | 32 +
.../beam/sdk/transforms/FlatMapElementsTest.java | 35 +-
.../apache/beam/sdk/transforms/FlattenTest.java | 37 +-
.../apache/beam/sdk/transforms/GroupByKeyTest.java | 280 +-
.../beam/sdk/transforms/GroupIntoBatchesTest.java | 4 +-
.../beam/sdk/transforms/MapElementsTest.java | 42 +-
.../org/apache/beam/sdk/transforms/MaxTest.java | 10 +-
.../org/apache/beam/sdk/transforms/MeanTest.java | 4 +-
.../org/apache/beam/sdk/transforms/MinTest.java | 10 +-
.../org/apache/beam/sdk/transforms/ParDoTest.java | 322 +-
.../beam/sdk/transforms/SplittableDoFnTest.java | 155 +-
.../org/apache/beam/sdk/transforms/SumTest.java | 8 +-
.../org/apache/beam/sdk/transforms/TopTest.java | 10 +-
.../org/apache/beam/sdk/transforms/ViewTest.java | 11 +-
.../org/apache/beam/sdk/transforms/WatchTest.java | 797 ++
.../apache/beam/sdk/transforms/WithKeysTest.java | 30 +-
.../sdk/transforms/reflect/DoFnInvokersTest.java | 103 +-
.../reflect/DoFnSignaturesProcessElementTest.java | 2 +-
.../reflect/DoFnSignaturesSplittableDoFnTest.java | 83 +-
.../sdk/transforms/reflect/DoFnSignaturesTest.java | 14 +
.../splittabledofn/OffsetRangeTrackerTest.java | 1 +
.../transforms/windowing/CalendarWindowsTest.java | 24 +
.../sdk/transforms/windowing/FixedWindowsTest.java | 7 +
.../sdk/transforms/windowing/SessionsTest.java | 14 +
.../transforms/windowing/SlidingWindowsTest.java | 41 +-
.../beam/sdk/transforms/windowing/WindowTest.java | 225 +
.../beam/sdk/util/IdentitySideInputWindowFn.java | 4 +
.../beam/sdk/util/MutationDetectorsTest.java | 56 +
.../beam/sdk/util/SerializableUtilsTest.java | 60 +
.../beam/sdk/values/PCollectionTupleTest.java | 7 +-
.../beam/sdk/values/TypeDescriptorsTest.java | 60 +
.../extensions/google-cloud-platform-core/pom.xml | 2 +-
.../sdk/extensions/gcp/storage/GcsFileSystem.java | 5 +-
.../java/org/apache/beam/sdk/util/GcsUtil.java | 16 +-
.../beam/sdk/util/RetryHttpRequestInitializer.java | 147 +-
.../sdk/extensions/gcp/GcpCoreApiSurfaceTest.java | 48 +-
.../java/org/apache/beam/sdk/util/GcsUtilTest.java | 61 +-
.../sdk/util/RetryHttpRequestInitializerTest.java | 31 +-
sdks/java/extensions/jackson/pom.xml | 2 +-
sdks/java/extensions/join-library/pom.xml | 2 +-
.../beam/sdk/extensions/joinlibrary/Join.java | 65 +-
.../extensions/joinlibrary/OuterFullJoinTest.java | 179 +
sdks/java/extensions/pom.xml | 4 +-
sdks/java/extensions/protobuf/pom.xml | 2 +-
sdks/java/extensions/sketching/pom.xml | 104 +
.../extensions/sketching/ApproximateDistinct.java | 573 ++
.../sdk/extensions/sketching/package-info.java | 22 +
.../sketching/ApproximateDistinctTest.java | 209 +
sdks/java/extensions/sorter/pom.xml | 8 +-
.../beam/sdk/extensions/sorter/SortValues.java | 20 +-
sdks/java/extensions/sql/NOTICE | 45 +
sdks/java/extensions/sql/pom.xml | 425 +
.../extensions/sql/src/main/codegen/config.fmpp | 23 +
.../sql/src/main/codegen/data/Parser.tdd | 75 +
.../sql/src/main/codegen/includes/license.ftl | 17 +
.../sql/src/main/codegen/includes/parserImpls.ftl | 89 +
.../beam/sdk/extensions/sql/BeamRecordSqlType.java | 186 +
.../apache/beam/sdk/extensions/sql/BeamSql.java | 250 +
.../apache/beam/sdk/extensions/sql/BeamSqlCli.java | 115 +
.../sdk/extensions/sql/BeamSqlRecordHelper.java | 217 +
.../beam/sdk/extensions/sql/BeamSqlTable.java | 54 +
.../apache/beam/sdk/extensions/sql/BeamSqlUdf.java | 43 +
.../sdk/extensions/sql/example/BeamSqlExample.java | 104 +
.../sdk/extensions/sql/example/package-info.java | 23 +
.../beam/sdk/extensions/sql/impl/BeamSqlEnv.java | 136 +
.../interpreter/BeamSqlExpressionExecutor.java | 44 +
.../sql/impl/interpreter/BeamSqlFnExecutor.java | 463 +
.../operator/BeamSqlCaseExpression.java | 64 +
.../operator/BeamSqlCastExpression.java | 132 +
.../interpreter/operator/BeamSqlExpression.java | 79 +
.../operator/BeamSqlInputRefExpression.java | 48 +
.../interpreter/operator/BeamSqlPrimitive.java | 157 +
.../interpreter/operator/BeamSqlUdfExpression.java | 92 +
.../operator/BeamSqlWindowEndExpression.java | 48 +
.../operator/BeamSqlWindowExpression.java | 51 +
.../operator/BeamSqlWindowStartExpression.java | 49 +
.../sql/impl/interpreter/operator/UdafImpl.java | 87 +
.../arithmetic/BeamSqlArithmeticExpression.java | 124 +
.../arithmetic/BeamSqlDivideExpression.java | 37 +
.../arithmetic/BeamSqlMinusExpression.java | 36 +
.../operator/arithmetic/BeamSqlModExpression.java | 36 +
.../arithmetic/BeamSqlMultiplyExpression.java | 36 +
.../operator/arithmetic/BeamSqlPlusExpression.java | 36 +
.../operator/arithmetic/package-info.java | 22 +
.../comparison/BeamSqlCompareExpression.java | 97 +
.../comparison/BeamSqlEqualsExpression.java | 49 +
.../comparison/BeamSqlGreaterThanExpression.java | 49 +
.../BeamSqlGreaterThanOrEqualsExpression.java | 49 +
.../comparison/BeamSqlIsNotNullExpression.java | 54 +
.../comparison/BeamSqlIsNullExpression.java | 54 +
.../comparison/BeamSqlLessThanExpression.java | 49 +
.../BeamSqlLessThanOrEqualsExpression.java | 49 +
.../comparison/BeamSqlNotEqualsExpression.java | 49 +
.../operator/comparison/package-info.java | 22 +
.../date/BeamSqlCurrentDateExpression.java | 45 +
.../date/BeamSqlCurrentTimeExpression.java | 53 +
.../date/BeamSqlCurrentTimestampExpression.java | 49 +
.../operator/date/BeamSqlDateCeilExpression.java | 55 +
.../operator/date/BeamSqlDateFloorExpression.java | 55 +
.../date/BeamSqlDatetimeMinusExpression.java | 107 +
.../date/BeamSqlDatetimePlusExpression.java | 129 +
.../operator/date/BeamSqlExtractExpression.java | 102 +
.../date/BeamSqlIntervalMultiplyExpression.java | 103 +
.../BeamSqlTimestampMinusIntervalExpression.java | 83 +
.../BeamSqlTimestampMinusTimestampExpression.java | 97 +
.../interpreter/operator/date/TimeUnitUtils.java | 54 +
.../interpreter/operator/date/package-info.java | 22 +
.../operator/logical/BeamSqlAndExpression.java | 48 +
.../operator/logical/BeamSqlLogicalExpression.java | 46 +
.../operator/logical/BeamSqlNotExpression.java | 54 +
.../operator/logical/BeamSqlOrExpression.java | 48 +
.../interpreter/operator/logical/package-info.java | 22 +
.../operator/math/BeamSqlAbsExpression.java | 74 +
.../operator/math/BeamSqlAcosExpression.java | 40 +
.../operator/math/BeamSqlAsinExpression.java | 40 +
.../operator/math/BeamSqlAtan2Expression.java | 42 +
.../operator/math/BeamSqlAtanExpression.java | 40 +
.../operator/math/BeamSqlCeilExpression.java | 45 +
.../operator/math/BeamSqlCosExpression.java | 40 +
.../operator/math/BeamSqlCotExpression.java | 40 +
.../operator/math/BeamSqlDegreesExpression.java | 40 +
.../operator/math/BeamSqlExpExpression.java | 40 +
.../operator/math/BeamSqlFloorExpression.java | 45 +
.../operator/math/BeamSqlLnExpression.java | 40 +
.../operator/math/BeamSqlLogExpression.java | 40 +
.../operator/math/BeamSqlMathBinaryExpression.java | 65 +
.../operator/math/BeamSqlMathUnaryExpression.java | 60 +
.../operator/math/BeamSqlPiExpression.java | 43 +
.../operator/math/BeamSqlPowerExpression.java | 44 +
.../operator/math/BeamSqlRadiansExpression.java | 40 +
.../operator/math/BeamSqlRandExpression.java | 55 +
.../math/BeamSqlRandIntegerExpression.java | 59 +
.../operator/math/BeamSqlRoundExpression.java | 107 +
.../operator/math/BeamSqlSignExpression.java | 72 +
.../operator/math/BeamSqlSinExpression.java | 40 +
.../operator/math/BeamSqlTanExpression.java | 40 +
.../operator/math/BeamSqlTruncateExpression.java | 75 +
.../interpreter/operator/math/package-info.java | 22 +
.../impl/interpreter/operator/package-info.java | 22 +
.../reinterpret/BeamSqlReinterpretExpression.java | 58 +
.../DatetimeReinterpretConversions.java | 59 +
.../reinterpret/IntegerReinterpretConversions.java | 44 +
.../reinterpret/ReinterpretConversion.java | 114 +
.../operator/reinterpret/Reinterpreter.java | 101 +
.../operator/reinterpret/package-info.java | 22 +
.../string/BeamSqlCharLengthExpression.java | 40 +
.../operator/string/BeamSqlConcatExpression.java | 63 +
.../operator/string/BeamSqlInitCapExpression.java | 56 +
.../operator/string/BeamSqlLowerExpression.java | 40 +
.../operator/string/BeamSqlOverlayExpression.java | 77 +
.../operator/string/BeamSqlPositionExpression.java | 73 +
.../string/BeamSqlStringUnaryExpression.java | 44 +
.../string/BeamSqlSubstringExpression.java | 83 +
.../operator/string/BeamSqlTrimExpression.java | 102 +
.../operator/string/BeamSqlUpperExpression.java | 40 +
.../interpreter/operator/string/package-info.java | 22 +
.../sql/impl/interpreter/package-info.java | 22 +
.../beam/sdk/extensions/sql/impl/package-info.java | 22 +
.../extensions/sql/impl/parser/BeamSqlParser.java | 50 +
.../sql/impl/parser/ColumnConstraint.java | 42 +
.../sql/impl/parser/ColumnDefinition.java | 56 +
.../extensions/sql/impl/parser/ParserUtils.java | 64 +
.../extensions/sql/impl/parser/SqlCreateTable.java | 141 +
.../extensions/sql/impl/parser/SqlDDLKeywords.java | 27 +
.../extensions/sql/impl/parser/UnparseUtil.java | 59 +
.../extensions/sql/impl/parser/package-info.java | 22 +
.../sql/impl/planner/BeamQueryPlanner.java | 168 +
.../sql/impl/planner/BeamRelDataTypeSystem.java | 40 +
.../extensions/sql/impl/planner/BeamRuleSets.java | 75 +
.../extensions/sql/impl/planner/package-info.java | 24 +
.../sql/impl/rel/BeamAggregationRel.java | 182 +
.../sdk/extensions/sql/impl/rel/BeamFilterRel.java | 69 +
.../sdk/extensions/sql/impl/rel/BeamIOSinkRel.java | 75 +
.../extensions/sql/impl/rel/BeamIOSourceRel.java | 62 +
.../extensions/sql/impl/rel/BeamIntersectRel.java | 58 +
.../sdk/extensions/sql/impl/rel/BeamJoinRel.java | 298 +
.../sql/impl/rel/BeamLogicalConvention.java | 72 +
.../sdk/extensions/sql/impl/rel/BeamMinusRel.java | 56 +
.../extensions/sql/impl/rel/BeamProjectRel.java | 80 +
.../sdk/extensions/sql/impl/rel/BeamRelNode.java | 39 +
.../sql/impl/rel/BeamSetOperatorRelBase.java | 98 +
.../sdk/extensions/sql/impl/rel/BeamSortRel.java | 235 +
.../extensions/sql/impl/rel/BeamSqlRelUtils.java | 72 +
.../sdk/extensions/sql/impl/rel/BeamUnionRel.java | 88 +
.../sdk/extensions/sql/impl/rel/BeamValuesRel.java | 79 +
.../sdk/extensions/sql/impl/rel/package-info.java | 23 +
.../sql/impl/rule/BeamAggregationRule.java | 162 +
.../extensions/sql/impl/rule/BeamFilterRule.java | 49 +
.../extensions/sql/impl/rule/BeamIOSinkRule.java | 81 +
.../extensions/sql/impl/rule/BeamIOSourceRule.java | 49 +
.../sql/impl/rule/BeamIntersectRule.java | 50 +
.../sdk/extensions/sql/impl/rule/BeamJoinRule.java | 53 +
.../extensions/sql/impl/rule/BeamMinusRule.java | 50 +
.../extensions/sql/impl/rule/BeamProjectRule.java | 50 +
.../sdk/extensions/sql/impl/rule/BeamSortRule.java | 51 +
.../extensions/sql/impl/rule/BeamUnionRule.java | 50 +
.../extensions/sql/impl/rule/BeamValuesRule.java | 48 +
.../sdk/extensions/sql/impl/rule/package-info.java | 23 +
.../extensions/sql/impl/schema/BaseBeamTable.java | 36 +
.../sdk/extensions/sql/impl/schema/BeamIOType.java | 28 +
.../sql/impl/schema/BeamPCollectionTable.java | 63 +
.../extensions/sql/impl/schema/BeamTableUtils.java | 118 +
.../extensions/sql/impl/schema/package-info.java | 22 +
.../impl/transform/BeamAggregationTransforms.java | 311 +
.../impl/transform/BeamBuiltinAggregations.java | 557 ++
.../sql/impl/transform/BeamJoinTransforms.java | 161 +
.../impl/transform/BeamSetOperatorsTransforms.java | 111 +
.../sql/impl/transform/BeamSqlFilterFn.java | 63 +
.../impl/transform/BeamSqlOutputToConsoleFn.java | 41 +
.../sql/impl/transform/BeamSqlProjectFn.java | 72 +
.../sql/impl/transform/package-info.java | 22 +
.../extensions/sql/impl/utils/CalciteUtils.java | 113 +
.../extensions/sql/impl/utils/SqlTypeUtils.java | 63 +
.../extensions/sql/impl/utils/package-info.java | 22 +
.../beam/sdk/extensions/sql/meta/Column.java | 51 +
.../apache/beam/sdk/extensions/sql/meta/Table.java | 69 +
.../beam/sdk/extensions/sql/meta/package-info.java | 22 +
.../extensions/sql/meta/provider/MetaUtils.java | 40 +
.../sql/meta/provider/TableProvider.java | 61 +
.../sql/meta/provider/kafka/BeamKafkaCSVTable.java | 111 +
.../sql/meta/provider/kafka/BeamKafkaTable.java | 115 +
.../meta/provider/kafka/KafkaTableProvider.java | 82 +
.../sql/meta/provider/kafka/package-info.java | 22 +
.../extensions/sql/meta/provider/package-info.java | 22 +
.../sql/meta/provider/text/BeamTextCSVTable.java | 80 +
.../provider/text/BeamTextCSVTableIOReader.java | 59 +
.../provider/text/BeamTextCSVTableIOWriter.java | 59 +
.../sql/meta/provider/text/BeamTextTable.java | 41 +
.../sql/meta/provider/text/TextTableProvider.java | 83 +
.../sql/meta/provider/text/package-info.java | 22 +
.../sql/meta/store/InMemoryMetaStore.java | 113 +
.../sdk/extensions/sql/meta/store/MetaStore.java | 56 +
.../extensions/sql/meta/store/package-info.java | 22 +
.../beam/sdk/extensions/sql/package-info.java | 22 +
.../sql/src/main/resources/log4j.properties | 23 +
...ckaged.org.codehaus.commons.compiler.properties | 18 +
.../sdk/extensions/sql/BeamSqlApiSurfaceTest.java | 62 +
.../beam/sdk/extensions/sql/BeamSqlCliTest.java | 75 +
.../extensions/sql/BeamSqlDslAggregationTest.java | 400 +
.../beam/sdk/extensions/sql/BeamSqlDslBase.java | 136 +
.../sdk/extensions/sql/BeamSqlDslFilterTest.java | 155 +
.../sdk/extensions/sql/BeamSqlDslJoinTest.java | 188 +
.../sdk/extensions/sql/BeamSqlDslProjectTest.java | 227 +
.../sdk/extensions/sql/BeamSqlDslUdfUdafTest.java | 139 +
.../apache/beam/sdk/extensions/sql/TestUtils.java | 190 +
.../impl/interpreter/BeamSqlFnExecutorTest.java | 460 +
.../interpreter/BeamSqlFnExecutorTestBase.java | 88 +
.../operator/BeamNullExperssionTest.java | 55 +
.../operator/BeamSqlAndOrExpressionTest.java | 61 +
.../operator/BeamSqlCaseExpressionTest.java | 93 +
.../operator/BeamSqlCastExpressionTest.java | 129 +
.../operator/BeamSqlCompareExpressionTest.java | 115 +
.../operator/BeamSqlInputRefExpressionTest.java | 57 +
.../interpreter/operator/BeamSqlPrimitiveTest.java | 59 +
.../operator/BeamSqlReinterpretExpressionTest.java | 133 +
.../operator/BeamSqlUdfExpressionTest.java | 51 +
.../BeamSqlArithmeticExpressionTest.java | 237 +
.../date/BeamSqlCurrentDateExpressionTest.java | 38 +
.../date/BeamSqlCurrentTimeExpressionTest.java | 39 +
.../BeamSqlCurrentTimestampExpressionTest.java | 39 +
.../date/BeamSqlDateCeilExpressionTest.java | 50 +
.../date/BeamSqlDateExpressionTestBase.java | 52 +
.../date/BeamSqlDateFloorExpressionTest.java | 49 +
.../date/BeamSqlDatetimeMinusExpressionTest.java | 133 +
.../date/BeamSqlDatetimePlusExpressionTest.java | 155 +
.../date/BeamSqlExtractExpressionTest.java | 103 +
.../BeamSqlIntervalMultiplyExpressionTest.java | 107 +
...eamSqlTimestampMinusIntervalExpressionTest.java | 163 +
...amSqlTimestampMinusTimestampExpressionTest.java | 233 +
.../operator/date/TimeUnitUtilsTest.java | 54 +
.../operator/logical/BeamSqlNotExpressionTest.java | 47 +
.../math/BeamSqlMathBinaryExpressionTest.java | 215 +
.../math/BeamSqlMathUnaryExpressionTest.java | 312 +
.../DatetimeReinterpretConversionsTest.java | 73 +
.../IntegerReinterpretConversionsTest.java | 81 +
.../reinterpret/ReinterpretConversionTest.java | 116 +
.../operator/reinterpret/ReinterpreterTest.java | 155 +
.../string/BeamSqlCharLengthExpressionTest.java | 44 +
.../string/BeamSqlConcatExpressionTest.java | 66 +
.../string/BeamSqlInitCapExpressionTest.java | 54 +
.../string/BeamSqlLowerExpressionTest.java | 44 +
.../string/BeamSqlOverlayExpressionTest.java | 87 +
.../string/BeamSqlPositionExpressionTest.java | 84 +
.../string/BeamSqlStringUnaryExpressionTest.java | 52 +
.../string/BeamSqlSubstringExpressionTest.java | 101 +
.../operator/string/BeamSqlTrimExpressionTest.java | 103 +
.../string/BeamSqlUpperExpressionTest.java | 44 +
.../sql/impl/parser/BeamSqlParserTest.java | 167 +
.../sdk/extensions/sql/impl/rel/BaseRelTest.java | 34 +
.../sql/impl/rel/BeamIntersectRelTest.java | 118 +
.../impl/rel/BeamJoinRelBoundedVsBoundedTest.java | 203 +
.../rel/BeamJoinRelUnboundedVsBoundedTest.java | 240 +
.../rel/BeamJoinRelUnboundedVsUnboundedTest.java | 218 +
.../extensions/sql/impl/rel/BeamMinusRelTest.java | 117 +
.../sql/impl/rel/BeamSetOperatorRelBaseTest.java | 105 +
.../extensions/sql/impl/rel/BeamSortRelTest.java | 257 +
.../extensions/sql/impl/rel/BeamUnionRelTest.java | 103 +
.../extensions/sql/impl/rel/BeamValuesRelTest.java | 104 +
.../sdk/extensions/sql/impl/rel/CheckSize.java | 41 +
.../sql/impl/schema/BeamSqlRowCoderTest.java | 77 +
.../transform/BeamAggregationTransformTest.java | 453 +
.../schema/transform/BeamTransformBaseTest.java | 97 +
.../sql/impl/utils/SqlTypeUtilsTest.java | 76 +
.../BeamSqlArithmeticOperatorsIntegrationTest.java | 165 +
...BeamSqlBuiltinFunctionsIntegrationTestBase.java | 168 +
.../BeamSqlComparisonOperatorsIntegrationTest.java | 329 +
...BeamSqlConditionalFunctionsIntegrationTest.java | 60 +
.../BeamSqlDateFunctionsIntegrationTest.java | 195 +
.../BeamSqlLogicalFunctionsIntegrationTest.java | 43 +
.../BeamSqlMathFunctionsIntegrationTest.java | 351 +
.../BeamSqlStringFunctionsIntegrationTest.java | 51 +
.../meta/provider/kafka/BeamKafkaCSVTableTest.java | 107 +
.../provider/kafka/KafkaTableProviderTest.java | 76 +
.../meta/provider/text/BeamTextCSVTableTest.java | 176 +
.../meta/provider/text/TextTableProviderTest.java | 87 +
.../sql/meta/store/InMemoryMetaStoreTest.java | 185 +
.../extensions/sql/mock/MockedBoundedTable.java | 134 +
.../beam/sdk/extensions/sql/mock/MockedTable.java | 42 +
.../extensions/sql/mock/MockedUnboundedTable.java | 110 +
sdks/java/fn-execution/pom.xml | 115 +
.../harness/channel/ManagedChannelFactory.java | 82 +
.../beam/harness/channel/SocketAddressFactory.java | 64 +
.../apache/beam/harness/channel/package-info.java | 22 +
.../harness/channel/ManagedChannelFactoryTest.java | 71 +
.../harness/channel/SocketAddressFactoryTest.java | 56 +
.../org/apache/beam/harness/test/Consumer.java | 26 +
.../org/apache/beam/harness/test/Supplier.java | 26 +
.../apache/beam/harness/test/TestExecutors.java | 93 +
.../beam/harness/test/TestExecutorsTest.java | 175 +
.../org/apache/beam/harness/test/TestStreams.java | 185 +
.../apache/beam/harness/test/TestStreamsTest.java | 109 +
sdks/java/harness/pom.xml | 133 +-
.../beam/fn/harness/BeamFnDataReadRunner.java | 171 +
.../beam/fn/harness/BeamFnDataWriteRunner.java | 156 +
.../beam/fn/harness/BoundedSourceRunner.java | 165 +
.../apache/beam/fn/harness/FnApiDoFnRunner.java | 918 ++
.../java/org/apache/beam/fn/harness/FnHarness.java | 64 +-
.../org/apache/beam/fn/harness/IdGenerator.java | 33 +
.../beam/fn/harness/PTransformRunnerFactory.java | 84 +
.../fn/harness/channel/ManagedChannelFactory.java | 80 -
.../fn/harness/channel/SocketAddressFactory.java | 64 -
.../beam/fn/harness/channel/package-info.java | 22 -
.../fn/harness/control/BeamFnControlClient.java | 18 +-
.../fn/harness/control/ProcessBundleHandler.java | 455 +-
.../beam/fn/harness/control/RegisterHandler.java | 18 +-
.../data/BeamFnDataBufferingOutboundObserver.java | 8 +-
.../beam/fn/harness/data/BeamFnDataClient.java | 7 +-
.../beam/fn/harness/data/BeamFnDataGrpcClient.java | 24 +-
.../fn/harness/data/BeamFnDataGrpcMultiplexer.java | 33 +-
.../fn/harness/data/BeamFnDataInboundObserver.java | 4 +-
.../beam/fn/harness/fake/FakeStepContext.java | 70 -
.../apache/beam/fn/harness/fake/package-info.java | 22 -
.../beam/fn/harness/fn/ThrowingBiConsumer.java | 33 +
.../fn/harness/logging/BeamFnLoggingClient.java | 238 +-
.../apache/beam/fn/harness/state/BagUserState.java | 121 +
.../beam/fn/harness/state/BeamFnStateClient.java | 39 +
.../harness/state/BeamFnStateGrpcClientCache.java | 173 +
.../state/LazyCachingIteratorToIterable.java | 72 +
.../fn/harness/state/StateFetchingIterators.java | 126 +
.../apache/beam/fn/harness/state/package-info.java | 22 +
.../fn/harness/stream/BufferingStreamObserver.java | 16 +-
.../apache/beam/fn/harness/stream/DataStreams.java | 229 +
.../fn/harness/stream/StreamObserverFactory.java | 4 +-
.../beam/runners/core/BeamFnDataReadRunner.java | 113 -
.../beam/runners/core/BeamFnDataWriteRunner.java | 96 -
.../beam/runners/core/BoundedSourceRunner.java | 105 -
.../org/apache/beam/runners/core/package-info.java | 22 -
.../beam/fn/harness/BeamFnDataReadRunnerTest.java | 280 +
.../beam/fn/harness/BeamFnDataWriteRunnerTest.java | 264 +
.../beam/fn/harness/BoundedSourceRunnerTest.java | 184 +
.../beam/fn/harness/FnApiDoFnRunnerTest.java | 408 +
.../org/apache/beam/fn/harness/FnHarnessTest.java | 25 +-
.../apache/beam/fn/harness/IdGeneratorTest.java | 40 +
.../harness/channel/ManagedChannelFactoryTest.java | 74 -
.../harness/channel/SocketAddressFactoryTest.java | 56 -
.../harness/control/BeamFnControlClientTest.java | 13 +-
.../harness/control/ProcessBundleHandlerTest.java | 777 +-
.../fn/harness/control/RegisterHandlerTest.java | 34 +-
.../BeamFnDataBufferingOutboundObserverTest.java | 4 +-
.../fn/harness/data/BeamFnDataGrpcClientTest.java | 30 +-
.../data/BeamFnDataGrpcMultiplexerTest.java | 9 +-
.../data/BeamFnDataInboundObserverTest.java | 2 +-
.../harness/logging/BeamFnLoggingClientTest.java | 136 +-
.../beam/fn/harness/state/BagUserStateTest.java | 106 +
.../state/BeamFnStateGrpcClientCacheTest.java | 234 +
.../fn/harness/state/FakeBeamFnStateClient.java | 110 +
.../state/LazyCachingIteratorToIterableTest.java | 76 +
.../harness/state/StateFetchingIteratorsTest.java | 99 +
.../stream/BufferingStreamObserverTest.java | 12 +-
.../beam/fn/harness/stream/DataStreamsTest.java | 167 +
.../harness/stream/DirectStreamObserverTest.java | 8 +-
.../apache/beam/fn/harness/test/TestExecutors.java | 85 -
.../beam/fn/harness/test/TestExecutorsTest.java | 160 -
.../apache/beam/fn/harness/test/TestStreams.java | 162 -
.../beam/fn/harness/test/TestStreamsTest.java | 84 -
.../runners/core/BeamFnDataReadRunnerTest.java | 189 -
.../runners/core/BeamFnDataWriteRunnerTest.java | 157 -
.../beam/runners/core/BoundedSourceRunnerTest.java | 113 -
sdks/java/io/amqp/pom.xml | 123 +
.../java/org/apache/beam/sdk/io/amqp/AmqpIO.java | 380 +
.../apache/beam/sdk/io/amqp/AmqpMessageCoder.java | 79 +
.../io/amqp/AmqpMessageCoderProviderRegistrar.java | 44 +
.../org/apache/beam/sdk/io/amqp/package-info.java | 22 +
.../org/apache/beam/sdk/io/amqp/AmqpIOTest.java | 124 +
.../beam/sdk/io/amqp/AmqpMessageCoderTest.java | 89 +
sdks/java/io/cassandra/pom.xml | 113 +
.../apache/beam/sdk/io/cassandra/CassandraIO.java | 486 +
.../beam/sdk/io/cassandra/CassandraService.java | 66 +
.../sdk/io/cassandra/CassandraServiceImpl.java | 398 +
.../apache/beam/sdk/io/cassandra/package-info.java | 22 +
.../beam/sdk/io/cassandra/CassandraIOIT.java | 254 +
.../beam/sdk/io/cassandra/CassandraIOTest.java | 279 +
.../sdk/io/cassandra/CassandraServiceImplTest.java | 138 +
.../sdk/io/cassandra/CassandraTestDataSet.java | 153 +
sdks/java/io/common/pom.xml | 12 +-
.../beam/sdk/io/common/IOTestPipelineOptions.java | 20 +-
.../org/apache/beam/sdk/io/common/TestRow.java | 114 +
.../elasticsearch-tests-2/pom.xml | 60 +
.../src/test/contrib/create_elk_container.sh | 0
.../sdk/io/elasticsearch/ElasticsearchIOIT.java | 123 +
.../sdk/io/elasticsearch/ElasticsearchIOTest.java | 185 +
.../elasticsearch-tests-5/pom.xml | 124 +
.../src/test/contrib/create_elk_container.sh | 0
.../sdk/io/elasticsearch/ElasticsearchIOIT.java | 122 +
.../sdk/io/elasticsearch/ElasticsearchIOTest.java | 185 +
.../java/org/elasticsearch/bootstrap/JarHell.java | 39 +
.../elasticsearch-tests-common/pom.xml | 77 +
.../io/elasticsearch/ElasticSearchIOTestUtils.java | 141 +
.../io/elasticsearch/ElasticsearchIOITCommon.java | 92 +
.../elasticsearch/ElasticsearchIOTestCommon.java | 306 +
sdks/java/io/elasticsearch-tests/pom.xml | 144 +
sdks/java/io/elasticsearch/pom.xml | 224 +-
.../beam/sdk/io/elasticsearch/ElasticsearchIO.java | 479 +-
.../beam/sdk/io/elasticsearch/package-info.java | 1 -
.../io/elasticsearch/ElasticSearchIOTestUtils.java | 129 -
.../sdk/io/elasticsearch/ElasticsearchIOIT.java | 155 -
.../sdk/io/elasticsearch/ElasticsearchIOTest.java | 353 -
.../io/elasticsearch/ElasticsearchTestDataSet.java | 111 -
sdks/java/io/google-cloud-platform/pom.xml | 188 +-
.../beam/sdk/io/gcp/bigquery/BatchLoads.java | 480 +-
.../beam/sdk/io/gcp/bigquery/BigQueryHelpers.java | 28 +-
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 903 +-
.../sdk/io/gcp/bigquery/BigQueryQuerySource.java | 49 +-
.../beam/sdk/io/gcp/bigquery/BigQueryServices.java | 61 +-
.../sdk/io/gcp/bigquery/BigQueryServicesImpl.java | 121 +-
.../sdk/io/gcp/bigquery/BigQuerySourceBase.java | 153 +-
.../io/gcp/bigquery/BigQueryTableRowIterator.java | 501 -
.../sdk/io/gcp/bigquery/BigQueryTableSource.java | 40 +-
.../beam/sdk/io/gcp/bigquery/CalculateSchemas.java | 78 -
.../beam/sdk/io/gcp/bigquery/CreateTables.java | 23 +-
.../sdk/io/gcp/bigquery/DynamicDestinations.java | 34 +-
.../gcp/bigquery/DynamicDestinationsHelpers.java | 27 +-
.../sdk/io/gcp/bigquery/GenerateShardedTable.java | 1 +
.../sdk/io/gcp/bigquery/InsertRetryPolicy.java | 86 +
.../io/gcp/bigquery/PassThroughThenCleanup.java | 48 +-
.../beam/sdk/io/gcp/bigquery/ReifyAsIterable.java | 51 +
.../beam/sdk/io/gcp/bigquery/SchemaAndRecord.java | 43 +
.../beam/sdk/io/gcp/bigquery/ShardedKey.java | 67 -
.../beam/sdk/io/gcp/bigquery/ShardedKeyCoder.java | 74 -
.../beam/sdk/io/gcp/bigquery/StreamingInserts.java | 39 +-
.../beam/sdk/io/gcp/bigquery/StreamingWriteFn.java | 51 +-
.../sdk/io/gcp/bigquery/StreamingWriteTables.java | 38 +-
.../beam/sdk/io/gcp/bigquery/TableDestination.java | 43 +-
.../sdk/io/gcp/bigquery/TableDestinationCoder.java | 2 +
.../io/gcp/bigquery/TableDestinationCoderV2.java | 59 +
.../beam/sdk/io/gcp/bigquery/TagWithUniqueIds.java | 1 +
.../sdk/io/gcp/bigquery/TransformingSource.java | 136 -
.../sdk/io/gcp/bigquery/WriteBundlesToFiles.java | 57 +-
.../gcp/bigquery/WriteGroupedRecordsToFiles.java | 8 +-
.../beam/sdk/io/gcp/bigquery/WritePartition.java | 21 +-
.../beam/sdk/io/gcp/bigquery/WriteRename.java | 112 +-
.../beam/sdk/io/gcp/bigquery/WriteResult.java | 27 +-
.../beam/sdk/io/gcp/bigquery/WriteTables.java | 190 +-
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 171 +-
.../sdk/io/gcp/bigtable/BigtableServiceImpl.java | 25 +-
.../sdk/io/gcp/datastore/AdaptiveThrottler.java | 103 +
.../beam/sdk/io/gcp/datastore/DatastoreV1.java | 339 +-
.../beam/sdk/io/gcp/datastore/MovingAverage.java | 50 +
.../apache/beam/sdk/io/gcp/pubsub/PubsubIO.java | 53 +-
.../sdk/io/gcp/pubsub/PubsubUnboundedSink.java | 6 +-
.../sdk/io/gcp/pubsub/PubsubUnboundedSource.java | 43 +-
.../sdk/io/gcp/spanner/CreateTransactionFn.java | 61 +
.../beam/sdk/io/gcp/spanner/MutationGroup.java | 67 +
.../sdk/io/gcp/spanner/MutationGroupEncoder.java | 660 ++
.../sdk/io/gcp/spanner/MutationSizeEstimator.java | 155 +
.../sdk/io/gcp/spanner/NaiveSpannerReadFn.java | 85 +
.../beam/sdk/io/gcp/spanner/OrderedCode.java | 764 ++
.../beam/sdk/io/gcp/spanner/ReadOperation.java | 96 +
.../beam/sdk/io/gcp/spanner/ReadSpannerSchema.java | 94 +
.../sdk/io/gcp/spanner/SerializedMutation.java | 35 +
.../io/gcp/spanner/SerializedMutationCoder.java | 60 +
.../beam/sdk/io/gcp/spanner/SpannerAccessor.java | 43 +
.../beam/sdk/io/gcp/spanner/SpannerConfig.java | 161 +
.../apache/beam/sdk/io/gcp/spanner/SpannerIO.java | 991 ++
.../beam/sdk/io/gcp/spanner/SpannerSchema.java | 144 +
.../beam/sdk/io/gcp/spanner/Transaction.java | 33 +
.../beam/sdk/io/gcp/spanner/package-info.java | 23 +
.../apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java | 18 +-
.../beam/sdk/io/gcp/bigquery/BigQueryIOTest.java | 1023 +-
.../io/gcp/bigquery/BigQueryServicesImplTest.java | 88 +-
.../gcp/bigquery/BigQueryTableRowIteratorTest.java | 358 -
.../beam/sdk/io/gcp/bigquery/BigQueryUtilTest.java | 198 +-
.../sdk/io/gcp/bigquery/FakeBigQueryServices.java | 78 -
.../sdk/io/gcp/bigquery/FakeDatasetService.java | 101 +-
.../beam/sdk/io/gcp/bigquery/FakeJobService.java | 46 +-
.../sdk/io/gcp/bigquery/InsertRetryPolicyTest.java | 79 +
.../beam/sdk/io/gcp/bigquery/TableContainer.java | 2 +
.../beam/sdk/io/gcp/bigtable/BigtableIOTest.java | 30 +-
.../beam/sdk/io/gcp/bigtable/BigtableReadIT.java | 5 +-
.../sdk/io/gcp/bigtable/BigtableTestOptions.java | 5 -
.../beam/sdk/io/gcp/bigtable/BigtableWriteIT.java | 4 +-
.../io/gcp/datastore/AdaptiveThrottlerTest.java | 111 +
.../beam/sdk/io/gcp/datastore/DatastoreV1Test.java | 326 +-
.../beam/sdk/io/gcp/datastore/SplitQueryFnIT.java | 5 +-
.../apache/beam/sdk/io/gcp/datastore/V1ReadIT.java | 2 +-
.../beam/sdk/io/gcp/datastore/V1TestUtil.java | 17 +-
.../beam/sdk/io/gcp/datastore/V1WriteIT.java | 36 +-
.../beam/sdk/io/gcp/pubsub/PubsubIOTest.java | 55 +
.../sdk/io/gcp/spanner/FakeServiceFactory.java | 82 +
.../io/gcp/spanner/MutationGroupEncoderTest.java | 636 ++
.../io/gcp/spanner/MutationSizeEstimatorTest.java | 150 +
.../beam/sdk/io/gcp/spanner/OrderedCodeTest.java | 890 ++
.../beam/sdk/io/gcp/spanner/RandomUtils.java | 41 +
.../sdk/io/gcp/spanner/ReadSpannerSchemaTest.java | 134 +
.../beam/sdk/io/gcp/spanner/SpannerIOReadTest.java | 263 +
.../sdk/io/gcp/spanner/SpannerIOWriteTest.java | 497 +
.../beam/sdk/io/gcp/spanner/SpannerReadIT.java | 166 +
.../beam/sdk/io/gcp/spanner/SpannerSchemaTest.java | 61 +
.../beam/sdk/io/gcp/spanner/SpannerWriteIT.java | 170 +
sdks/java/io/hadoop-common/pom.xml | 2 +-
.../sdk/io/hadoop/SerializableConfiguration.java | 18 +-
sdks/java/io/hadoop-file-system/pom.xml | 47 +-
.../apache/beam/sdk/io/hdfs/HadoopFileSystem.java | 27 +-
.../beam/sdk/io/hdfs/HadoopFileSystemTest.java | 95 +-
sdks/java/io/hadoop/input-format/pom.xml | 7 +-
.../io/hadoop/inputformat/HadoopInputFormatIO.java | 105 +-
.../inputformat/HadoopInputFormatIOTest.java | 94 +-
sdks/java/io/hadoop/jdk1.8-tests/pom.xml | 5 +-
.../hadoop/inputformat/HIFIOWithElasticTest.java | 21 +-
sdks/java/io/hadoop/pom.xml | 2 +-
sdks/java/io/hbase/pom.xml | 26 +-
.../sdk/io/hbase/HBaseCoderProviderRegistrar.java | 38 +
.../java/org/apache/beam/sdk/io/hbase/HBaseIO.java | 1020 +-
.../beam/sdk/io/hbase/HBaseMutationCoder.java | 41 +
.../apache/beam/sdk/io/hbase/HBaseResultCoder.java | 6 +-
.../apache/beam/sdk/io/hbase/SerializableScan.java | 37 +-
.../io/hbase/HBaseCoderProviderRegistrarTest.java | 43 +
.../org/apache/beam/sdk/io/hbase/HBaseIOTest.java | 740 +-
.../beam/sdk/io/hbase/HBaseMutationCoderTest.java | 4 +-
.../beam/sdk/io/hbase/HBaseResultCoderTest.java | 4 +-
.../beam/sdk/io/hbase/SerializableScanTest.java | 6 +-
sdks/java/io/hcatalog/pom.xml | 179 +
.../apache/beam/sdk/io/hcatalog/HCatalogIO.java | 479 +
.../apache/beam/sdk/io/hcatalog/package-info.java | 22 +
.../sdk/io/hcatalog/EmbeddedMetastoreService.java | 87 +
.../beam/sdk/io/hcatalog/HCatalogIOTest.java | 277 +
.../beam/sdk/io/hcatalog/HCatalogIOTestUtils.java | 108 +
.../io/hcatalog/src/test/resources/hive-site.xml | 301 +
sdks/java/io/jdbc/pom.xml | 196 +-
.../java/org/apache/beam/sdk/io/jdbc/JdbcIO.java | 322 +-
.../java/org/apache/beam/sdk/io/jdbc/JdbcIOIT.java | 203 +-
.../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java | 121 +-
.../apache/beam/sdk/io/jdbc/JdbcTestDataSet.java | 130 -
.../apache/beam/sdk/io/jdbc/JdbcTestHelper.java | 81 +
sdks/java/io/jms/pom.xml | 2 +-
.../java/org/apache/beam/sdk/io/jms/JmsIO.java | 80 +-
.../java/org/apache/beam/sdk/io/jms/JmsIOTest.java | 78 +
sdks/java/io/kafka/README.md | 36 +
sdks/java/io/kafka/pom.xml | 31 +-
.../org/apache/beam/sdk/io/kafka/ConsumerSpEL.java | 56 +
.../java/org/apache/beam/sdk/io/kafka/KafkaIO.java | 956 +-
.../org/apache/beam/sdk/io/kafka/ProducerSpEL.java | 135 +
.../org/apache/beam/sdk/io/kafka/KafkaIOTest.java | 385 +-
sdks/java/io/kinesis/pom.xml | 18 +-
.../beam/sdk/io/kinesis/AWSClientsProvider.java | 36 +
.../beam/sdk/io/kinesis/CheckpointGenerator.java | 6 +-
.../apache/beam/sdk/io/kinesis/CustomOptional.java | 111 +-
.../sdk/io/kinesis/DynamicCheckpointGenerator.java | 52 +-
.../sdk/io/kinesis/GetKinesisRecordsResult.java | 55 +-
.../beam/sdk/io/kinesis/KinesisClientProvider.java | 31 -
.../org/apache/beam/sdk/io/kinesis/KinesisIO.java | 348 +-
.../apache/beam/sdk/io/kinesis/KinesisReader.java | 274 +-
.../sdk/io/kinesis/KinesisReaderCheckpoint.java | 83 +-
.../apache/beam/sdk/io/kinesis/KinesisRecord.java | 177 +-
.../beam/sdk/io/kinesis/KinesisRecordCoder.java | 68 +-
.../apache/beam/sdk/io/kinesis/KinesisSource.java | 161 +-
.../apache/beam/sdk/io/kinesis/RecordFilter.java | 18 +-
.../org/apache/beam/sdk/io/kinesis/RoundRobin.java | 53 -
.../beam/sdk/io/kinesis/ShardCheckpoint.java | 235 +-
.../beam/sdk/io/kinesis/ShardReadersPool.java | 162 +
.../beam/sdk/io/kinesis/ShardRecordsIterator.java | 120 +-
.../sdk/io/kinesis/SimplifiedKinesisClient.java | 291 +-
.../apache/beam/sdk/io/kinesis/StartingPoint.java | 84 +-
.../sdk/io/kinesis/StaticCheckpointGenerator.java | 27 +-
.../sdk/io/kinesis/TransientKinesisException.java | 9 +-
.../beam/sdk/io/kinesis/AmazonKinesisMock.java | 532 +-
.../beam/sdk/io/kinesis/CustomOptionalTest.java | 27 +-
.../io/kinesis/DynamicCheckpointGeneratorTest.java | 33 +-
.../beam/sdk/io/kinesis/KinesisMockReadTest.java | 98 +-
.../io/kinesis/KinesisReaderCheckpointTest.java | 52 +-
.../beam/sdk/io/kinesis/KinesisReaderIT.java | 128 +-
.../beam/sdk/io/kinesis/KinesisReaderTest.java | 254 +-
.../sdk/io/kinesis/KinesisRecordCoderTest.java | 34 +-
.../beam/sdk/io/kinesis/KinesisTestOptions.java | 43 +-
.../beam/sdk/io/kinesis/KinesisUploader.java | 70 +-
.../beam/sdk/io/kinesis/RecordFilterTest.java | 52 +-
.../apache/beam/sdk/io/kinesis/RoundRobinTest.java | 57 -
.../beam/sdk/io/kinesis/ShardCheckpointTest.java | 203 +-
.../beam/sdk/io/kinesis/ShardReadersPoolTest.java | 185 +
.../sdk/io/kinesis/ShardRecordsIteratorTest.java | 221 +-
.../io/kinesis/SimplifiedKinesisClientTest.java | 420 +-
sdks/java/io/mongodb/pom.xml | 2 +-
.../beam/sdk/io/mongodb/MongoDbGridFSIO.java | 8 +-
.../org/apache/beam/sdk/io/mongodb/MongoDbIO.java | 321 +-
.../beam/sdk/io/mongodb/MongoDBGridFSIOTest.java | 2 +-
.../apache/beam/sdk/io/mongodb/MongoDbIOTest.java | 48 +
sdks/java/io/mqtt/pom.xml | 2 +-
.../java/org/apache/beam/sdk/io/mqtt/MqttIO.java | 53 +-
sdks/java/io/pom.xml | 66 +-
sdks/java/io/redis/pom.xml | 90 +
.../sdk/io/redis/RedisConnectionConfiguration.java | 122 +
.../java/org/apache/beam/sdk/io/redis/RedisIO.java | 451 +
.../org/apache/beam/sdk/io/redis/package-info.java | 22 +
.../org/apache/beam/sdk/io/redis/RedisIOTest.java | 109 +
sdks/java/io/solr/pom.xml | 145 +
.../beam/sdk/io/solr/AuthorizedSolrClient.java | 91 +
.../apache/beam/sdk/io/solr/JavaBinCodecCoder.java | 98 +
.../java/org/apache/beam/sdk/io/solr/SolrIO.java | 705 ++
.../org/apache/beam/sdk/io/solr/package-info.java | 20 +
.../beam/sdk/io/solr/JavaBinCodecCoderTest.java | 81 +
.../org/apache/beam/sdk/io/solr/SolrIOTest.java | 269 +
.../apache/beam/sdk/io/solr/SolrIOTestUtils.java | 132 +
.../test/resources/cloud-minimal/conf/schema.xml | 29 +
.../resources/cloud-minimal/conf/solrconfig.xml | 48 +
sdks/java/io/tika/pom.xml | 113 +
.../org/apache/beam/sdk/io/tika/ParseResult.java | 144 +
.../java/org/apache/beam/sdk/io/tika/TikaIO.java | 284 +
.../org/apache/beam/sdk/io/tika/package-info.java | 22 +
.../apache/beam/sdk/io/tika/ParseResultTest.java | 83 +
.../org/apache/beam/sdk/io/tika/TikaIOTest.java | 149 +
sdks/java/io/tika/src/test/resources/damaged.pdf | 2 +
.../test/resources/valid/apache-beam-tika-pdf.zip | Bin 0 -> 11685 bytes
.../src/test/resources/valid/apache-beam-tika.odt | Bin 0 -> 12540 bytes
sdks/java/io/xml/pom.xml | 2 +-
.../java/org/apache/beam/sdk/io/xml/XmlIO.java | 375 +-
.../java/org/apache/beam/sdk/io/xml/XmlSink.java | 33 +-
.../java/org/apache/beam/sdk/io/xml/XmlSource.java | 65 +-
.../org/apache/beam/sdk/io/xml/XmlSinkTest.java | 16 +-
.../org/apache/beam/sdk/io/xml/XmlSourceTest.java | 74 +-
sdks/java/java8tests/pom.xml | 2 +-
sdks/java/javadoc/ant.xml | 1 +
sdks/java/javadoc/pom.xml | 44 +-
.../examples-java8/generate-sources.sh | 2 +-
sdks/java/maven-archetypes/examples-java8/pom.xml | 2 +-
.../src/main/resources/archetype-resources/pom.xml | 65 +-
.../maven-archetypes/examples/generate-sources.sh | 2 +-
sdks/java/maven-archetypes/examples/pom.xml | 2 +-
.../src/main/resources/archetype-resources/pom.xml | 49 +-
sdks/java/maven-archetypes/pom.xml | 2 +-
sdks/java/maven-archetypes/starter/pom.xml | 2 +-
.../src/main/resources/archetype-resources/pom.xml | 12 +-
.../resources/projects/basic/reference/pom.xml | 12 +-
sdks/java/nexmark/pom.xml | 265 +
.../java/org/apache/beam/sdk/nexmark/Main.java | 303 +
.../java/org/apache/beam/sdk/nexmark/Monitor.java | 78 +
.../beam/sdk/nexmark/NexmarkConfiguration.java | 721 ++
.../apache/beam/sdk/nexmark/NexmarkLauncher.java | 1157 +++
.../apache/beam/sdk/nexmark/NexmarkOptions.java | 403 +
.../org/apache/beam/sdk/nexmark/NexmarkPerf.java | 207 +
.../org/apache/beam/sdk/nexmark/NexmarkSuite.java | 112 +
.../org/apache/beam/sdk/nexmark/NexmarkUtils.java | 674 ++
.../org/apache/beam/sdk/nexmark/model/Auction.java | 187 +
.../apache/beam/sdk/nexmark/model/AuctionBid.java | 85 +
.../beam/sdk/nexmark/model/AuctionCount.java | 84 +
.../beam/sdk/nexmark/model/AuctionPrice.java | 88 +
.../org/apache/beam/sdk/nexmark/model/Bid.java | 177 +
.../beam/sdk/nexmark/model/BidsPerSession.java | 87 +
.../beam/sdk/nexmark/model/CategoryPrice.java | 97 +
.../org/apache/beam/sdk/nexmark/model/Done.java | 80 +
.../org/apache/beam/sdk/nexmark/model/Event.java | 171 +
.../beam/sdk/nexmark/model/IdNameReserve.java | 98 +
.../apache/beam/sdk/nexmark/model/KnownSize.java | 26 +
.../beam/sdk/nexmark/model/NameCityStateId.java | 103 +
.../org/apache/beam/sdk/nexmark/model/Person.java | 163 +
.../apache/beam/sdk/nexmark/model/SellerPrice.java | 89 +
.../beam/sdk/nexmark/model/package-info.java | 22 +
.../org/apache/beam/sdk/nexmark/package-info.java | 21 +
.../sdk/nexmark/queries/AbstractSimulator.java | 211 +
.../beam/sdk/nexmark/queries/NexmarkQuery.java | 270 +
.../sdk/nexmark/queries/NexmarkQueryModel.java | 117 +
.../apache/beam/sdk/nexmark/queries/Query0.java | 70 +
.../beam/sdk/nexmark/queries/Query0Model.java | 64 +
.../apache/beam/sdk/nexmark/queries/Query1.java | 67 +
.../apache/beam/sdk/nexmark/queries/Query10.java | 367 +
.../apache/beam/sdk/nexmark/queries/Query11.java | 79 +
.../apache/beam/sdk/nexmark/queries/Query12.java | 80 +
.../beam/sdk/nexmark/queries/Query1Model.java | 76 +
.../apache/beam/sdk/nexmark/queries/Query2.java | 79 +
.../beam/sdk/nexmark/queries/Query2Model.java | 80 +
.../apache/beam/sdk/nexmark/queries/Query3.java | 301 +
.../beam/sdk/nexmark/queries/Query3Model.java | 124 +
.../apache/beam/sdk/nexmark/queries/Query4.java | 116 +
.../beam/sdk/nexmark/queries/Query4Model.java | 186 +
.../apache/beam/sdk/nexmark/queries/Query5.java | 138 +
.../beam/sdk/nexmark/queries/Query5Model.java | 176 +
.../apache/beam/sdk/nexmark/queries/Query6.java | 155 +
.../beam/sdk/nexmark/queries/Query6Model.java | 133 +
.../apache/beam/sdk/nexmark/queries/Query7.java | 90 +
.../beam/sdk/nexmark/queries/Query7Model.java | 130 +
.../apache/beam/sdk/nexmark/queries/Query8.java | 98 +
.../beam/sdk/nexmark/queries/Query8Model.java | 148 +
.../apache/beam/sdk/nexmark/queries/Query9.java | 44 +
.../beam/sdk/nexmark/queries/Query9Model.java | 44 +
.../beam/sdk/nexmark/queries/WinningBids.java | 418 +
.../sdk/nexmark/queries/WinningBidsSimulator.java | 206 +
.../beam/sdk/nexmark/queries/package-info.java | 22 +
.../sdk/nexmark/sources/BoundedEventSource.java | 190 +
.../apache/beam/sdk/nexmark/sources/Generator.java | 609 ++
.../beam/sdk/nexmark/sources/GeneratorConfig.java | 298 +
.../sdk/nexmark/sources/UnboundedEventSource.java | 329 +
.../beam/sdk/nexmark/sources/package-info.java | 22 +
.../nexmark/src/main/resources/log4j.properties | 55 +
.../apache/beam/sdk/nexmark/queries/QueryTest.java | 185 +
.../nexmark/sources/BoundedEventSourceTest.java | 70 +
.../beam/sdk/nexmark/sources/GeneratorTest.java | 110 +
.../nexmark/sources/UnboundedEventSourceTest.java | 105 +
sdks/java/pom.xml | 5 +-
sdks/pom.xml | 6 +-
sdks/python/MANIFEST.in | 1 +
sdks/python/apache_beam/__init__.py | 47 +-
sdks/python/apache_beam/coders/coder_impl.py | 26 +-
sdks/python/apache_beam/coders/coders.py | 134 +-
sdks/python/apache_beam/coders/coders_test.py | 2 +-
.../apache_beam/coders/coders_test_common.py | 21 +-
sdks/python/apache_beam/coders/observable_test.py | 1 -
.../coders/proto2_coder_test_messages_pb2.py | 6 +-
.../apache_beam/coders/standard_coders_test.py | 11 +-
sdks/python/apache_beam/coders/stream.pxd | 2 +-
sdks/python/apache_beam/coders/stream.pyx | 2 +-
sdks/python/apache_beam/coders/stream_test.py | 1 -
sdks/python/apache_beam/coders/typecoders.py | 1 -
.../apache_beam/examples/complete/autocomplete.py | 21 +-
.../examples/complete/autocomplete_test.py | 31 +-
.../apache_beam/examples/complete/estimate_pi.py | 16 +-
.../examples/complete/estimate_pi_test.py | 14 +-
.../examples/complete/game/game_stats.py | 393 +
.../examples/complete/game/game_stats_test.py | 81 +
.../examples/complete/game/hourly_team_score.py | 256 +-
.../examples/complete/game/leader_board.py | 349 +
.../examples/complete/game/leader_board_test.py | 69 +
.../examples/complete/game/user_score.py | 195 +-
.../complete/juliaset/juliaset/juliaset.py | 48 +-
.../complete/juliaset/juliaset/juliaset_test.py | 6 +-
.../examples/complete/juliaset/juliaset_main.py | 2 -
.../examples/complete/juliaset/setup.py | 7 +-
sdks/python/apache_beam/examples/complete/tfidf.py | 38 +-
.../apache_beam/examples/complete/tfidf_test.py | 38 +-
.../examples/complete/top_wikipedia_sessions.py | 17 +-
.../complete/top_wikipedia_sessions_test.py | 10 +-
.../examples/cookbook/bigquery_schema.py | 159 +-
.../examples/cookbook/bigquery_side_input.py | 56 +-
.../examples/cookbook/bigquery_side_input_test.py | 39 +-
.../examples/cookbook/bigquery_tornadoes.py | 28 +-
.../cookbook/bigquery_tornadoes_it_test.py | 14 +-
.../examples/cookbook/bigquery_tornadoes_test.py | 19 +-
.../python/apache_beam/examples/cookbook/coders.py | 16 +-
.../apache_beam/examples/cookbook/coders_test.py | 14 +-
.../examples/cookbook/custom_ptransform.py | 29 +-
.../examples/cookbook/custom_ptransform_test.py | 11 +-
.../examples/cookbook/datastore_wordcount.py | 57 +-
.../apache_beam/examples/cookbook/filters.py | 21 +-
.../examples/cookbook/group_with_coder.py | 47 +-
.../examples/cookbook/group_with_coder_test.py | 10 +-
.../apache_beam/examples/cookbook/mergecontacts.py | 121 +-
.../examples/cookbook/mergecontacts_test.py | 6 +-
.../examples/cookbook/multiple_output_pardo.py | 84 +-
.../cookbook/multiple_output_pardo_test.py | 13 +-
.../apache_beam/examples/snippets/snippets.py | 576 +-
.../apache_beam/examples/snippets/snippets_test.py | 470 +-
.../apache_beam/examples/streaming_wordcap.py | 22 +-
.../apache_beam/examples/streaming_wordcount.py | 66 +-
.../apache_beam/examples/windowed_wordcount.py | 96 +
sdks/python/apache_beam/examples/wordcount.py | 13 +-
.../apache_beam/examples/wordcount_debugging.py | 65 +-
.../examples/wordcount_debugging_test.py | 3 +-
.../python/apache_beam/examples/wordcount_fnapi.py | 151 +
.../apache_beam/examples/wordcount_it_test.py | 16 +
.../apache_beam/examples/wordcount_minimal.py | 35 +-
.../apache_beam/examples/wordcount_minimal_test.py | 3 +-
sdks/python/apache_beam/examples/wordcount_test.py | 3 +-
sdks/python/apache_beam/internal/gcp/auth.py | 1 -
sdks/python/apache_beam/internal/gcp/json_value.py | 46 +-
.../apache_beam/internal/gcp/json_value_test.py | 3 +-
sdks/python/apache_beam/internal/pickler.py | 2 +-
sdks/python/apache_beam/internal/util.py | 6 +-
sdks/python/apache_beam/io/avroio.py | 164 +-
sdks/python/apache_beam/io/avroio_test.py | 54 +-
sdks/python/apache_beam/io/concat_source_test.py | 1 -
sdks/python/apache_beam/io/filebasedsink.py | 22 +-
sdks/python/apache_beam/io/filebasedsink_test.py | 19 +-
sdks/python/apache_beam/io/filebasedsource.py | 234 +-
sdks/python/apache_beam/io/filebasedsource_test.py | 8 +-
sdks/python/apache_beam/io/filesystem.py | 78 +-
sdks/python/apache_beam/io/filesystem_test.py | 5 +-
sdks/python/apache_beam/io/filesystems.py | 1 -
sdks/python/apache_beam/io/filesystems_test.py | 4 +-
sdks/python/apache_beam/io/gcp/bigquery.py | 458 +-
sdks/python/apache_beam/io/gcp/bigquery_test.py | 266 +-
.../io/gcp/datastore/v1/adaptive_throttler.py | 94 +
.../io/gcp/datastore/v1/adaptive_throttler_test.py | 95 +
.../apache_beam/io/gcp/datastore/v1/datastoreio.py | 136 +-
.../io/gcp/datastore/v1/datastoreio_test.py | 78 +-
.../apache_beam/io/gcp/datastore/v1/helper.py | 64 +-
.../apache_beam/io/gcp/datastore/v1/helper_test.py | 27 +-
.../python/apache_beam/io/gcp/datastore/v1/util.py | 95 +
.../apache_beam/io/gcp/datastore/v1/util_test.py | 67 +
.../apache_beam/io/gcp/gcsfilesystem_test.py | 1 +
sdks/python/apache_beam/io/gcp/gcsio.py | 29 +-
sdks/python/apache_beam/io/gcp/gcsio_test.py | 22 +-
.../clients/bigquery/bigquery_v2_messages.py | 3 -
.../clients/storage/storage_v1_messages.py | 3 -
sdks/python/apache_beam/io/gcp/pubsub.py | 167 +-
sdks/python/apache_beam/io/gcp/pubsub_test.py | 97 +-
.../apache_beam/io/gcp/tests/bigquery_matcher.py | 6 +-
.../io/gcp/tests/bigquery_matcher_test.py | 5 +-
sdks/python/apache_beam/io/gcp/tests/utils.py | 63 +
sdks/python/apache_beam/io/gcp/tests/utils_test.py | 110 +
sdks/python/apache_beam/io/iobase.py | 110 +-
sdks/python/apache_beam/io/localfilesystem_test.py | 4 +-
sdks/python/apache_beam/io/range_trackers.py | 184 +-
sdks/python/apache_beam/io/range_trackers_test.py | 223 +-
sdks/python/apache_beam/io/source_test_utils.py | 95 +-
.../apache_beam/io/source_test_utils_test.py | 2 +-
sdks/python/apache_beam/io/sources_test.py | 1 -
sdks/python/apache_beam/io/textio.py | 245 +-
sdks/python/apache_beam/io/textio_test.py | 499 +-
sdks/python/apache_beam/io/tfrecordio.py | 5 +-
sdks/python/apache_beam/io/tfrecordio_test.py | 8 +-
sdks/python/apache_beam/io/vcfio.py | 436 +
sdks/python/apache_beam/io/vcfio_test.py | 519 +
sdks/python/apache_beam/metrics/cells_test.py | 2 +-
sdks/python/apache_beam/metrics/execution.py | 5 +-
sdks/python/apache_beam/metrics/execution_test.py | 8 +-
sdks/python/apache_beam/metrics/metric.py | 3 +-
sdks/python/apache_beam/metrics/metric_test.py | 18 +-
sdks/python/apache_beam/metrics/metricbase.py | 4 +
.../python/apache_beam/options/pipeline_options.py | 66 +-
.../apache_beam/options/pipeline_options_test.py | 46 +-
.../options/pipeline_options_validator_test.py | 11 +-
sdks/python/apache_beam/options/value_provider.py | 2 -
.../apache_beam/options/value_provider_test.py | 93 +-
sdks/python/apache_beam/pipeline.py | 385 +-
sdks/python/apache_beam/pipeline_test.py | 143 +-
sdks/python/apache_beam/portability/__init__.py | 18 +
.../python/apache_beam/portability/api/__init__.py | 21 +
sdks/python/apache_beam/pvalue.py | 111 +-
sdks/python/apache_beam/pvalue_test.py | 8 +
sdks/python/apache_beam/runners/api/__init__.py | 32 -
.../apache_beam/runners/api/beam_fn_api_pb2.py | 2742 -----
.../runners/api/beam_fn_api_pb2_grpc.py | 205 -
.../apache_beam/runners/api/beam_runner_api_pb2.py | 2872 ------
sdks/python/apache_beam/runners/common.py | 6 +-
sdks/python/apache_beam/runners/common_test.py | 2 +-
.../runners/dataflow/dataflow_metrics.py | 2 +-
.../runners/dataflow/dataflow_runner.py | 320 +-
.../runners/dataflow/dataflow_runner_test.py | 142 +-
.../runners/dataflow/internal/apiclient.py | 148 +-
.../runners/dataflow/internal/apiclient_test.py | 135 +-
.../clients/dataflow/dataflow_v1b3_client.py | 259 +-
.../clients/dataflow/dataflow_v1b3_messages.py | 92 +-
.../internal/clients/dataflow/message_matchers.py | 1 -
.../clients/dataflow/message_matchers_test.py | 3 +-
.../runners/dataflow/internal/dependency.py | 127 +-
.../runners/dataflow/internal/dependency_test.py | 22 +-
.../apache_beam/runners/dataflow/internal/names.py | 2 +
.../runners/dataflow/native_io/iobase.py | 6 +-
.../runners/dataflow/native_io/iobase_test.py | 59 +-
.../runners/dataflow/native_io/streaming_create.py | 72 +
.../runners/dataflow/ptransform_overrides.py | 52 +
.../runners/dataflow/template_runner_test.py | 5 +-
.../runners/dataflow/test_dataflow_runner.py | 10 +-
.../apache_beam/runners/direct/bundle_factory.py | 2 +-
.../consumer_tracking_pipeline_visitor_test.py | 2 +-
.../apache_beam/runners/direct/direct_metrics.py | 2 +-
.../runners/direct/direct_metrics_test.py | 8 +-
.../apache_beam/runners/direct/direct_runner.py | 146 +-
.../runners/direct/direct_runner_test.py | 41 +
.../runners/direct/evaluation_context.py | 90 +-
sdks/python/apache_beam/runners/direct/executor.py | 242 +-
.../runners/direct/helper_transforms.py | 2 +-
.../runners/direct/transform_evaluator.py | 462 +-
.../apache_beam/runners/direct/transform_result.py | 41 -
sdks/python/apache_beam/runners/direct/util.py | 70 +
.../runners/direct/watermark_manager.py | 100 +-
.../apache_beam/runners/experimental/__init__.py | 16 +
.../experimental/python_rpc_direct/__init__.py | 22 +
.../python_rpc_direct/python_rpc_direct_runner.py | 110 +
.../experimental/python_rpc_direct/server.py | 111 +
sdks/python/apache_beam/runners/job/__init__.py | 16 +
sdks/python/apache_beam/runners/job/manager.py | 52 +
sdks/python/apache_beam/runners/job/utils.py | 32 +
.../python/apache_beam/runners/pipeline_context.py | 31 +-
.../runners/portability/fn_api_runner.py | 1096 +-
.../runners/portability/fn_api_runner_test.py | 134 +-
.../runners/portability/maptask_executor_runner.py | 19 +-
.../portability/maptask_executor_runner_test.py | 12 +-
.../runners/portability/universal_local_runner.py | 409 +
.../portability/universal_local_runner_main.py | 44 +
.../portability/universal_local_runner_test.py | 86 +
sdks/python/apache_beam/runners/runner.py | 61 +-
sdks/python/apache_beam/runners/runner_test.py | 2 +-
.../apache_beam/runners/worker/bundle_processor.py | 590 ++
.../apache_beam/runners/worker/data_plane.py | 71 +-
.../apache_beam/runners/worker/data_plane_test.py | 9 +-
.../apache_beam/runners/worker/log_handler.py | 17 +-
.../apache_beam/runners/worker/log_handler_test.py | 22 +-
.../apache_beam/runners/worker/opcounters.py | 10 +-
.../apache_beam/runners/worker/opcounters_test.py | 1 -
.../apache_beam/runners/worker/operation_specs.py | 20 +-
.../apache_beam/runners/worker/operations.pxd | 4 +-
.../apache_beam/runners/worker/operations.py | 87 +-
.../apache_beam/runners/worker/sdk_worker.py | 560 +-
.../apache_beam/runners/worker/sdk_worker_main.py | 30 +-
.../apache_beam/runners/worker/sdk_worker_test.py | 106 +-
.../apache_beam/runners/worker/statesampler.pyx | 68 +-
.../runners/worker/statesampler_fake.py | 17 +-
.../runners/worker/statesampler_test.py | 5 +-
.../apache_beam/testing/data/vcf/valid-4.0.vcf | 23 +
.../apache_beam/testing/data/vcf/valid-4.0.vcf.bz2 | Bin 0 -> 781 bytes
.../apache_beam/testing/data/vcf/valid-4.0.vcf.gz | Bin 0 -> 727 bytes
.../testing/data/vcf/valid-4.1-large.vcf | 10000 +++++++++++++++++++
.../testing/data/vcf/valid-4.1-large.vcf.gz | Bin 0 -> 156715 bytes
.../apache_beam/testing/data/vcf/valid-4.2.vcf | 42 +
.../apache_beam/testing/data/vcf/valid-4.2.vcf.gz | Bin 0 -> 1240 bytes
.../apache_beam/testing/pipeline_verifiers.py | 1 -
.../apache_beam/testing/pipeline_verifiers_test.py | 5 +-
sdks/python/apache_beam/testing/test_pipeline.py | 54 +-
.../apache_beam/testing/test_pipeline_test.py | 4 +-
sdks/python/apache_beam/testing/test_stream.py | 6 +-
.../python/apache_beam/testing/test_stream_test.py | 69 +
sdks/python/apache_beam/testing/test_utils.py | 63 +-
sdks/python/apache_beam/testing/test_utils_test.py | 86 +
sdks/python/apache_beam/testing/util.py | 21 +-
sdks/python/apache_beam/testing/util_test.py | 4 +-
sdks/python/apache_beam/transforms/combiners.py | 17 +-
.../apache_beam/transforms/combiners_test.py | 70 +-
sdks/python/apache_beam/transforms/core.py | 561 +-
sdks/python/apache_beam/transforms/create_test.py | 6 +-
sdks/python/apache_beam/transforms/display.py | 91 +-
sdks/python/apache_beam/transforms/display_test.py | 6 +-
sdks/python/apache_beam/transforms/ptransform.py | 202 +-
.../apache_beam/transforms/ptransform_test.py | 78 +-
.../apache_beam/transforms/sideinputs_test.py | 5 +-
sdks/python/apache_beam/transforms/timeutil.py | 1 -
sdks/python/apache_beam/transforms/trigger.py | 79 +-
sdks/python/apache_beam/transforms/trigger_test.py | 19 +-
sdks/python/apache_beam/transforms/util.py | 273 +-
sdks/python/apache_beam/transforms/util_test.py | 108 +
sdks/python/apache_beam/transforms/window.py | 68 +-
sdks/python/apache_beam/transforms/window_test.py | 158 +-
.../transforms/write_ptransform_test.py | 11 +-
sdks/python/apache_beam/typehints/decorators.py | 129 +-
.../typehints/native_type_compatibility.py | 166 +
.../typehints/native_type_compatibility_test.py | 92 +
sdks/python/apache_beam/typehints/opcodes.py | 19 +-
.../apache_beam/typehints/trivial_inference.py | 83 +-
.../typehints/trivial_inference_test.py | 16 +-
sdks/python/apache_beam/typehints/typecheck.py | 6 +-
.../apache_beam/typehints/typed_pipeline_test.py | 55 +-
sdks/python/apache_beam/typehints/typehints.py | 79 +-
.../python/apache_beam/typehints/typehints_test.py | 3 +-
sdks/python/apache_beam/utils/annotations_test.py | 1 +
sdks/python/apache_beam/utils/counters.py | 107 +-
sdks/python/apache_beam/utils/counters_test.py | 78 +
sdks/python/apache_beam/utils/plugin.py | 42 +
sdks/python/apache_beam/utils/processes_test.py | 1 -
sdks/python/apache_beam/utils/proto_utils.py | 17 +
sdks/python/apache_beam/utils/retry.py | 9 +-
sdks/python/apache_beam/utils/retry_test.py | 5 +-
sdks/python/apache_beam/utils/timestamp.py | 5 +
sdks/python/apache_beam/utils/urns.py | 49 +-
sdks/python/apache_beam/version.py | 2 +-
sdks/python/container/Dockerfile | 27 +
sdks/python/container/boot.go | 123 +
sdks/python/container/pom.xml | 154 +
sdks/python/gen_protos.py | 141 +
sdks/python/generate_pydoc.sh | 140 +-
sdks/python/pom.xml | 9 +-
sdks/python/run_postcommit.sh | 20 -
sdks/python/run_pylint.sh | 49 +-
sdks/python/run_validatesrunner.sh | 71 +
sdks/python/setup.py | 65 +-
sdks/python/tox.ini | 32 +-
1777 files changed, 160041 insertions(+), 41448 deletions(-)
--
To stop receiving notification emails like this one, please contact
"commits@beam.apache.org" <co...@beam.apache.org>.