You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@wayang.apache.org by rp...@apache.org on 2021/09/27 22:10:44 UTC
[incubator-wayang] 01/01: Merge pull request #30 from
apache/WAYANG-34
This is an automated email from the ASF dual-hosted git repository.
rpardomeza pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git
commit 945cc59c5bfbba466060135fc120bbfad59d3aa2
Merge: 7df205f 41dacad
Author: Rodrigo Pardo Meza <ro...@gmail.com>
AuthorDate: Mon Sep 27 19:10:35 2021 -0300
Merge pull request #30 from apache/WAYANG-34
[WAYANG-34] Addition of terasort,
.travis.yml | 7 +-
NOTICE.TXT | 31 +-
RELEASE_NOTES | 47 +++
bin/build.sh | 9 +-
bin/change-scala-version.sh | 2 +
pom.xml | 3 +
.../apache/wayang/api/python/PythonAPITest.java | 4 +-
.../scala/org/apache/wayang/api/DataQuanta.scala | 33 +-
.../org/apache/wayang/api/DataQuantaBuilder.scala | 1 +
.../org/apache/wayang/api/JavaPlanBuilder.scala | 1 +
.../scala/org/apache/wayang/api/PlanBuilder.scala | 11 +-
.../java/org/apache/wayang/api/JavaApiTest.java | 41 ++-
.../test/scala/org/apache/wayang/api/ApiTest.scala | 2 +-
wayang-benchmark/pom.xml | 10 +
.../java/org/apache/wayang/apps/sgd/SGDImpl.java | 9 +-
.../apache/wayang/apps/sgd/SGDImprovedImpl.java | 13 +-
.../org/apache/wayang/apps/terasort/Random16.java | 376 ++++++++++++++++++++
.../apache/wayang/apps/terasort/Unsigned16.java | 299 ++++++++++++++++
.../java/org/apache/wayang/apps/tpch/Main.java | 7 +-
.../org/apache/wayang/apps/wordcount/Main.java | 21 +-
.../apps/benchmark/OptimizerScalabilityTest.scala | 1 -
.../wayang/apps/benchmark/PlanGenerator.scala | 2 +-
.../org/apache/wayang/apps/crocopr/CrocoPR.scala | 3 +-
.../org/apache/wayang/apps/kmeans/Kmeans.scala | 5 +-
.../wayang/apps/kmeans/postgres/Kmeans.scala | 5 +-
.../scala/org/apache/wayang/apps/sgd/SGD.scala | 1 -
.../wayang/apps/simwords/ScrubFunction.scala | 3 +-
.../simwords/SelectNearestCentroidFunction.scala | 4 +-
.../org/apache/wayang/apps/simwords/SimWords.scala | 3 +-
.../apache/wayang/apps/simwords/Word2NVec.scala | 3 +-
.../scala/org/apache/wayang/apps/sindy/Sindy.scala | 8 +-
.../org/apache/wayang/apps/terasort/TeraApp.scala | 152 ++++++++
.../org/apache/wayang/apps/terasort/TeraGen.scala | 129 +++++++
.../org/apache/wayang/apps/terasort/TeraSort.scala | 45 +++
.../apache/wayang/apps/terasort/TeraValidate.scala | 100 ++++++
.../scala/org/apache/wayang/apps/tpch/TpcH.scala | 16 +-
.../apache/wayang/apps/tpch/queries/Query1.scala | 2 +-
.../wayang/apps/tpch/queries/Query3Database.scala | 3 +-
.../wayang/apps/tpch/queries/Query3File.scala | 3 +-
.../wayang/apps/tpch/queries/Query3Hybrid.scala | 3 +-
.../org/apache/wayang/apps/util/Parameters.scala | 2 +-
.../apache/wayang/apps/util/ProfileDBHelper.scala | 3 -
.../wayang/apps/wordcount/WordCountScala.scala | 3 +-
.../wayang/apps/tpch/data/LineItemTupleTest.java | 5 +-
.../org/apache/wayang/apps/kmeans/KmeansTest.scala | 4 +-
.../apache/wayang/basic/channels/FileChannel.java | 15 +-
.../java/org/apache/wayang/basic/data/Record.java | 5 +-
.../basic/function/ProjectionDescriptor.java | 9 +-
.../wayang/basic/mapping/GlobalReduceMapping.java | 5 +-
.../org/apache/wayang/basic/mapping/Mappings.java | 3 +-
.../basic/mapping/MaterializedGroupByMapping.java | 5 +-
.../wayang/basic/mapping/PageRankMapping.java | 9 +-
.../wayang/basic/mapping/ReduceByMapping.java | 5 +-
.../apache/wayang/basic/mapping/RepeatMapping.java | 7 +-
.../wayang/basic/operators/CartesianOperator.java | 3 +-
.../wayang/basic/operators/CoGroupOperator.java | 3 +-
.../wayang/basic/operators/CollectionSource.java | 7 +-
.../wayang/basic/operators/CountOperator.java | 3 +-
.../wayang/basic/operators/DistinctOperator.java | 3 +-
.../wayang/basic/operators/DoWhileOperator.java | 9 +-
.../wayang/basic/operators/FilterOperator.java | 3 +-
.../wayang/basic/operators/FlatMapOperator.java | 3 +-
.../operators/GlobalMaterializedGroupOperator.java | 3 +-
.../basic/operators/GlobalReduceOperator.java | 3 +-
.../wayang/basic/operators/IntersectOperator.java | 3 +-
.../wayang/basic/operators/JoinOperator.java | 3 +-
.../wayang/basic/operators/LocalCallbackSink.java | 7 +-
.../wayang/basic/operators/LoopOperator.java | 9 +-
.../apache/wayang/basic/operators/MapOperator.java | 3 +-
.../basic/operators/MapPartitionsOperator.java | 3 +-
.../operators/MaterializedGroupByOperator.java | 3 +-
.../wayang/basic/operators/ObjectFileSink.java | 69 ++++
.../{TextFileSource.java => ObjectFileSource.java} | 125 ++++---
.../wayang/basic/operators/PageRankOperator.java | 3 +-
.../wayang/basic/operators/ReduceByOperator.java | 3 +-
.../wayang/basic/operators/ReduceOperator.java | 3 +-
.../wayang/basic/operators/RepeatOperator.java | 9 +-
.../wayang/basic/operators/SampleOperator.java | 11 +-
.../wayang/basic/operators/SortOperator.java | 3 +-
.../wayang/basic/operators/TextFileSink.java | 3 +-
.../wayang/basic/operators/TextFileSource.java | 17 +-
.../wayang/basic/operators/UnionAllOperator.java | 3 +-
.../wayang/basic/operators/ZipWithIdOperator.java | 3 +-
.../apache/wayang/basic/plugin/WayangBasic.java | 7 +-
.../wayang/basic/plugin/WayangBasicGraph.java | 5 +-
.../org/apache/wayang/basic/types/RecordType.java | 5 +-
.../basic/function/ProjectionDescriptorTest.java | 7 +-
.../wayang/basic/mapping/ReduceByMappingTest.java | 6 +-
.../operators/MaterializedGroupByOperatorTest.java | 5 +-
.../wayang/basic/operators/TextFileSourceTest.java | 30 +-
.../apache/wayang/basic/types/RecordTypeTest.java | 4 +-
.../org/apache/wayang/core/api/Configuration.java | 39 +--
.../main/java/org/apache/wayang/core/api/Job.java | 27 +-
.../org/apache/wayang/core/api/WayangContext.java | 4 +-
.../core/api/configuration/CollectionProvider.java | 5 +-
.../configuration/ExplicitCollectionProvider.java | 7 +-
.../FunctionalCollectionProvider.java | 3 +-
.../configuration/FunctionalKeyValueProvider.java | 3 +-
.../api/configuration/FunctionalValueProvider.java | 3 +-
.../core/api/configuration/KeyValueProvider.java | 7 +-
.../configuration/MapBasedKeyValueProvider.java | 5 +-
.../core/api/configuration/ValueProvider.java | 7 +-
.../core/function/AggregationDescriptor.java | 3 +-
.../wayang/core/function/ConsumerDescriptor.java | 3 +-
.../wayang/core/function/ExecutionContext.java | 3 +-
.../wayang/core/function/FlatMapDescriptor.java | 5 +-
.../wayang/core/function/FunctionDescriptor.java | 9 +-
.../core/function/MapPartitionsDescriptor.java | 5 +-
.../wayang/core/function/PredicateDescriptor.java | 3 +-
.../wayang/core/function/ReduceDescriptor.java | 3 +-
.../core/function/TransformationDescriptor.java | 3 +-
.../wayang/core/mapping/OperatorPattern.java | 7 +-
.../wayang/core/mapping/PlanTransformation.java | 13 +-
.../core/mapping/ReplacementSubplanFactory.java | 5 +-
.../apache/wayang/core/mapping/SubplanMatch.java | 5 +-
.../apache/wayang/core/mapping/SubplanPattern.java | 15 +-
.../wayang/core/monitor/DisabledMonitor.java | 3 +-
.../apache/wayang/core/monitor/FileMonitor.java | 7 +-
.../apache/wayang/core/monitor/HttpMonitor.java | 3 +-
.../org/apache/wayang/core/monitor/Monitor.java | 3 +-
.../apache/wayang/core/monitor/ZeroMQMonitor.java | 3 +-
.../optimizer/AggregateOptimizationContext.java | 5 +-
.../core/optimizer/DefaultOptimizationContext.java | 13 +-
.../wayang/core/optimizer/OptimizationContext.java | 25 +-
.../wayang/core/optimizer/OptimizationUtils.java | 11 +-
.../optimizer/ProbabilisticIntervalEstimate.java | 3 +-
.../wayang/core/optimizer/SanityChecker.java | 11 +-
.../AggregatingCardinalityEstimator.java | 5 +-
.../CardinalityEstimationTraversal.java | 21 +-
.../cardinality/CardinalityEstimatorManager.java | 9 +-
.../optimizer/cardinality/CardinalityPusher.java | 9 +-
.../cardinality/DefaultCardinalityEstimator.java | 5 +-
.../LoopHeadAlternativeCardinalityPusher.java | 7 +-
.../cardinality/LoopSubplanCardinalityPusher.java | 11 +-
.../OperatorAlternativeCardinalityPusher.java | 5 +-
.../cardinality/SubplanCardinalityPusher.java | 11 +-
.../SwitchForwardCardinalityEstimator.java | 2 +-
.../core/optimizer/channels/ChannelConversion.java | 5 +-
.../optimizer/channels/ChannelConversionGraph.java | 35 +-
.../channels/DefaultChannelConversion.java | 7 +-
.../core/optimizer/costs/DefaultLoadEstimator.java | 7 +-
.../core/optimizer/costs/EstimationContext.java | 5 +-
.../optimizer/costs/IntervalLoadEstimator.java | 7 +-
.../wayang/core/optimizer/costs/LoadEstimator.java | 3 +-
.../wayang/core/optimizer/costs/LoadProfile.java | 5 +-
.../core/optimizer/costs/LoadProfileEstimator.java | 3 +-
.../optimizer/costs/LoadProfileEstimators.java | 27 +-
.../costs/NestableLoadProfileEstimator.java | 7 +-
.../optimizer/costs/SimpleEstimationContext.java | 5 +-
.../optimizer/enumeration/ExecutionTaskFlow.java | 19 +-
.../enumeration/ExecutionTaskFlowCompiler.java | 23 +-
.../enumeration/LatentOperatorPruningStrategy.java | 13 +-
.../core/optimizer/enumeration/LoopEnumerator.java | 13 +-
.../optimizer/enumeration/LoopImplementation.java | 7 +-
.../optimizer/enumeration/PlanEnumeration.java | 29 +-
.../core/optimizer/enumeration/PlanEnumerator.java | 43 ++-
.../optimizer/enumeration/PlanImplementation.java | 39 +--
.../enumeration/RandomPruningStrategy.java | 3 +-
.../enumeration/StageAssignmentTraversal.java | 27 +-
.../optimizer/enumeration/TopKPruningStrategy.java | 3 +-
.../wayang/core/plan/executionplan/Channel.java | 29 +-
.../core/plan/executionplan/ExecutionPlan.java | 11 +-
.../core/plan/executionplan/ExecutionStage.java | 9 +-
.../plan/executionplan/ExecutionStageLoop.java | 5 +-
.../core/plan/executionplan/PlatformExecution.java | 5 +-
.../core/plan/wayangplan/ElementaryOperator.java | 3 +-
.../plan/wayangplan/EstimationContextProperty.java | 3 +-
.../core/plan/wayangplan/ExecutionOperator.java | 13 +-
.../wayang/core/plan/wayangplan/InputSlot.java | 3 +-
.../core/plan/wayangplan/LoopHeadAlternative.java | 7 +-
.../core/plan/wayangplan/LoopHeadOperator.java | 3 +-
.../wayang/core/plan/wayangplan/LoopIsolator.java | 7 +-
.../wayang/core/plan/wayangplan/LoopSubplan.java | 7 +-
.../wayang/core/plan/wayangplan/Operator.java | 15 +-
.../core/plan/wayangplan/OperatorAlternative.java | 11 +-
.../wayang/core/plan/wayangplan/OperatorBase.java | 15 +-
.../core/plan/wayangplan/OperatorContainer.java | 9 +-
.../core/plan/wayangplan/OperatorContainers.java | 3 +-
.../wayang/core/plan/wayangplan/OutputSlot.java | 3 +-
.../wayang/core/plan/wayangplan/PlanTraversal.java | 5 +-
.../apache/wayang/core/plan/wayangplan/Slot.java | 3 +-
.../wayang/core/plan/wayangplan/SlotMapping.java | 10 +-
.../wayang/core/plan/wayangplan/Subplan.java | 11 +-
.../wayang/core/plan/wayangplan/WayangPlan.java | 15 +-
.../traversal/AbstractTopologicalTraversal.java | 11 +-
.../core/platform/AbstractChannelInstance.java | 5 +-
.../wayang/core/platform/AtomicExecutionGroup.java | 3 +-
.../core/platform/CardinalityBreakpoint.java | 4 +-
.../wayang/core/platform/ChannelDescriptor.java | 5 +-
.../wayang/core/platform/ChannelInstance.java | 3 +-
.../core/platform/ConjunctiveBreakpoint.java | 5 +-
.../core/platform/CrossPlatformExecutor.java | 25 +-
.../wayang/core/platform/ExecutionState.java | 3 +-
.../wayang/core/platform/ExecutorTemplate.java | 17 +-
.../apache/wayang/core/platform/FixBreakpoint.java | 7 +-
.../org/apache/wayang/core/platform/Junction.java | 11 +-
.../wayang/core/platform/PartialExecution.java | 9 +-
.../wayang/core/platform/PushExecutorTemplate.java | 21 +-
.../platform/lineage/ExecutionLineageNode.java | 5 +-
.../platform/lineage/LazyExecutionLineageNode.java | 7 +-
.../apache/wayang/core/plugin/DynamicPlugin.java | 21 +-
.../java/org/apache/wayang/core/plugin/Plugin.java | 5 +-
.../core/profiling/CardinalityRepository.java | 19 +-
.../apache/wayang/core/profiling/ExecutionLog.java | 19 +-
.../core/profiling/ExecutionPlanMeasurement.java | 11 +-
.../profiling/FullInstrumentationStrategy.java | 3 +-
.../org/apache/wayang/core/types/DataSetType.java | 3 +-
.../wayang/core/types/DataUnitGroupType.java | 3 +-
.../apache/wayang/core/util/JsonSerializable.java | 3 +-
.../apache/wayang/core/util/JsonSerializables.java | 4 +-
.../org/apache/wayang/core/util/JuelUtils.java | 9 +-
.../java/org/apache/wayang/core/util/LruCache.java | 3 +-
.../apache/wayang/core/util/ReflectionUtils.java | 11 +-
.../apache/wayang/core/util/WayangCollections.java | 3 +-
.../apache/wayang/core/util/fs/FileSystems.java | 9 +-
.../org/apache/wayang/core/util/fs/FileUtils.java | 5 +-
.../wayang/core/util/fs/HadoopFileSystem.java | 15 +-
.../wayang/core/util/fs/LocalFileSystem.java | 7 +-
.../apache/wayang/core/util/mathex/Context.java | 3 +-
.../wayang/core/util/mathex/DefaultContext.java | 3 +-
.../wayang/core/util/mathex/ExpressionBuilder.java | 5 +-
.../core/util/mathex/model/CompiledFunction.java | 5 +-
.../core/util/mathex/model/NamedFunction.java | 7 +-
.../test/java/org/apache/wayang/core/SlotTest.java | 2 +-
.../wayang/core/mapping/OperatorPatternTest.java | 2 +-
.../core/mapping/PlanTransformationTest.java | 6 +-
.../wayang/core/mapping/SubplanPatternTest.java | 9 +-
.../wayang/core/mapping/test/TestSinkMapping.java | 5 +-
.../AggregatingCardinalityEstimatorTest.java | 13 +-
.../DefaultCardinalityEstimatorTest.java | 13 +-
.../LoopSubplanCardinalityPusherTest.java | 6 +-
.../cardinality/SubplanCardinalityPusherTest.java | 6 +-
.../channels/ChannelConversionGraphTest.java | 11 +-
.../costs/NestableLoadProfileEstimatorTest.java | 7 +-
.../enumeration/StageAssignmentTraversalTest.java | 7 +-
.../core/plan/wayangplan/LoopIsolatorTest.java | 9 +-
.../wayang/core/plan/wayangplan/OperatorTest.java | 7 +-
.../core/plan/wayangplan/SlotMappingTest.java | 11 +-
.../plan/wayangplan/test/TestFilterOperator.java | 3 +-
.../wayang/core/plan/wayangplan/test/TestJoin.java | 3 +-
.../core/plan/wayangplan/test/TestLoopHead.java | 7 +-
.../core/plan/wayangplan/test/TestMapOperator.java | 3 +-
.../wayang/core/plan/wayangplan/test/TestSink.java | 3 +-
.../core/plan/wayangplan/test/TestSource.java | 3 +-
.../wayang/core/platform/PartialExecutionTest.java | 15 +-
.../wayang/core/plugin/DynamicPluginTest.java | 15 +-
.../wayang/core/test/DummyExecutionOperator.java | 7 +-
.../org/apache/wayang/core/test/MockFactory.java | 11 +-
.../core/util/ConsumerIteratorAdapterTest.java | 9 +-
.../wayang/core/util/CrossProductIterableTest.java | 5 +-
.../wayang/core/util/LimitedInputStreamTest.java | 5 +-
.../wayang/core/util/ReflectionUtilsTest.java | 5 +-
.../wayang/core/util/WayangCollectionsTest.java | 5 +-
.../core/util/mathex/ExpressionBuilderTest.java | 7 +-
.../wayang/core/util/mathex/ExpressionTest.java | 9 +-
.../wayang/commons/util/profiledb/ProfileDB.java | 13 +-
.../profiledb/json/MeasurementDeserializer.java | 3 +-
.../util/profiledb/json/MeasurementSerializer.java | 3 +-
.../model/measurement/TimeMeasurement.java | 5 +-
.../util/profiledb/storage/FileStorage.java | 8 +-
.../util/profiledb/storage/JDBCStorage.java | 3 +-
.../commons/util/profiledb/storage/Storage.java | 16 +-
.../commons/util/profiledb/ProfileDBTest.java | 18 +-
.../measurement/TestMemoryMeasurement.java | 3 +-
.../profiledb/measurement/TestTimeMeasurement.java | 5 +-
wayang-docs/src/main/resources/Gemfile.lock | 3 +-
wayang-docs/src/main/resources/_data/menus.yml | 116 +++---
.../main/resources/how_contribute/bugs_report.md | 7 +-
.../pull_request.md | 24 +-
.../main/resources/how_contribute/helping_users.md | 2 +-
.../how_contribute/jira_issue_maintenance.md | 18 +-
.../resources/how_contribute/preparing_releases.md | 388 +++++++++++++++++++++
.../resources/how_contribute/slack_channels.md | 3 +-
.../wayang/flink/channels/ChannelConversions.java | 5 +-
.../wayang/flink/channels/DataSetChannel.java | 3 +-
.../wayang/flink/compiler/FunctionCompiler.java | 7 +-
.../wayang/flink/compiler/KeySelectorDistinct.java | 3 +-
.../wayang/flink/compiler/KeySelectorFunction.java | 5 +-
.../flink/compiler/OutputFormatConsumer.java | 5 +-
.../flink/compiler/WayangFileOutputFormat.java | 17 +-
.../flink/compiler/criterion/WayangAggregator.java | 5 +-
.../criterion/WayangConvergenceCriterion.java | 5 +-
.../flink/compiler/criterion/WayangListValue.java | 3 +-
.../flink/compiler/criterion/WayangValue.java | 7 +-
.../flink/execution/FlinkExecutionContext.java | 5 +-
.../wayang/flink/execution/FlinkExecutor.java | 7 +-
.../wayang/flink/mapping/CartesianMapping.java | 5 +-
.../wayang/flink/mapping/CoGroupMapping.java | 5 +-
.../flink/mapping/CollectionSourceMapping.java | 5 +-
.../apache/wayang/flink/mapping/CountMapping.java | 5 +-
.../wayang/flink/mapping/DistinctMapping.java | 5 +-
.../wayang/flink/mapping/DoWhileMapping.java | 5 +-
.../apache/wayang/flink/mapping/FilterMapping.java | 5 +-
.../wayang/flink/mapping/FlatMapMapping.java | 5 +-
.../mapping/GlobalMaterializedGroupMapping.java | 5 +-
.../wayang/flink/mapping/GlobalReduceMapping.java | 5 +-
.../wayang/flink/mapping/GroupByMapping.java | 5 +-
.../wayang/flink/mapping/IntersectMapping.java | 5 +-
.../apache/wayang/flink/mapping/JoinMapping.java | 5 +-
.../flink/mapping/LocalCallbackSinkMapping.java | 5 +-
.../apache/wayang/flink/mapping/LoopMapping.java | 5 +-
.../apache/wayang/flink/mapping/MapMapping.java | 5 +-
.../wayang/flink/mapping/MapPartitionsMapping.java | 5 +-
.../org/apache/wayang/flink/mapping/Mappings.java | 5 +-
.../flink/mapping/MaterializedGroupByMapping.java | 5 +-
...sectMapping.java => ObjectFileSinkMapping.java} | 25 +-
...ceMapping.java => ObjectFileSourceMapping.java} | 25 +-
.../wayang/flink/mapping/PageRankMapping.java | 5 +-
.../wayang/flink/mapping/ReduceByMapping.java | 5 +-
.../apache/wayang/flink/mapping/RepeatMapping.java | 5 +-
.../apache/wayang/flink/mapping/SampleMapping.java | 5 +-
.../apache/wayang/flink/mapping/SortMapping.java | 5 +-
.../wayang/flink/mapping/TextFileSinkMapping.java | 5 +-
.../flink/mapping/TextFileSourceMapping.java | 5 +-
.../wayang/flink/mapping/UnionAllMapping.java | 5 +-
.../wayang/flink/mapping/ZipWithIdMapping.java | 5 +-
.../flink/operators/FlinkCartesianOperator.java | 9 +-
.../flink/operators/FlinkCoGroupOperator.java | 11 +-
.../flink/operators/FlinkCollectionSink.java | 9 +-
.../flink/operators/FlinkCollectionSource.java | 7 +-
.../wayang/flink/operators/FlinkCountOperator.java | 9 +-
.../flink/operators/FlinkDistinctOperator.java | 9 +-
.../flink/operators/FlinkDoWhileOperator.java | 9 +-
.../flink/operators/FlinkExecutionOperator.java | 5 +-
.../flink/operators/FlinkFilterOperator.java | 11 +-
.../flink/operators/FlinkFlatMapOperator.java | 11 +-
.../FlinkGlobalMaterializedGroupOperator.java | 11 +-
.../flink/operators/FlinkGlobalReduceOperator.java | 11 +-
.../flink/operators/FlinkGroupByOperator.java | 13 +-
.../flink/operators/FlinkIntersectOperator.java | 7 +-
.../wayang/flink/operators/FlinkJoinOperator.java | 11 +-
.../flink/operators/FlinkLocalCallbackSink.java | 9 +-
.../wayang/flink/operators/FlinkLoopOperator.java | 7 +-
.../wayang/flink/operators/FlinkMapOperator.java | 9 +-
.../operators/FlinkMapPartitionsOperator.java | 11 +-
.../FlinkMaterializedGroupByOperator.java | 11 +-
.../flink/operators/FlinkObjectFileSink.java | 35 +-
.../flink/operators/FlinkObjectFileSource.java | 28 +-
.../flink/operators/FlinkPageRankOperator.java | 15 +-
.../flink/operators/FlinkReduceByOperator.java | 11 +-
.../operators/FlinkRepeatExpandedOperator.java | 7 +-
.../flink/operators/FlinkRepeatOperator.java | 7 +-
.../flink/operators/FlinkSampleOperator.java | 15 +-
.../wayang/flink/operators/FlinkSortOperator.java | 9 +-
.../wayang/flink/operators/FlinkTextFileSink.java | 9 +-
.../flink/operators/FlinkTextFileSource.java | 7 +-
.../wayang/flink/operators/FlinkTsvFileSink.java | 9 +-
.../flink/operators/FlinkUnionAllOperator.java | 7 +-
.../flink/operators/FlinkZipWithIdOperator.java | 9 +-
.../wayang/flink/platform/FlinkPlatform.java | 13 +-
.../wayang/flink/plugin/FlinkBasicPlugin.java | 5 +-
.../wayang/flink/plugin/FlinkConversionPlugin.java | 7 +-
.../wayang/giraph/Algorithm/PageRankAlgorithm.java | 5 +-
.../giraph/Algorithm/PageRankParameters.java | 5 +-
.../wayang/giraph/channels/ChannelConversions.java | 4 +-
.../wayang/giraph/execution/GiraphExecutor.java | 14 +-
.../apache/wayang/giraph/mappings/Mappings.java | 3 +-
.../wayang/giraph/mappings/PageRankMapping.java | 13 +-
.../giraph/operators/GiraphExecutionOperator.java | 3 +-
.../giraph/operators/GiraphPageRankOperator.java | 16 +-
.../wayang/giraph/platform/GiraphPlatform.java | 1 -
.../apache/wayang/giraph/plugin/GiraphPlugin.java | 7 +-
.../operators/GiraphPagaRankOperatorTest.java | 12 +-
.../wayang/java/channels/ChannelConversions.java | 5 +-
.../wayang/java/channels/CollectionChannel.java | 5 +-
.../wayang/java/channels/JavaChannelInstance.java | 3 +-
.../apache/wayang/java/channels/StreamChannel.java | 7 +-
.../wayang/java/compiler/FunctionCompiler.java | 7 +-
.../java/execution/JavaExecutionContext.java | 3 +-
.../apache/wayang/java/execution/JavaExecutor.java | 7 +-
.../wayang/java/mapping/CartesianMapping.java | 5 +-
.../apache/wayang/java/mapping/CoGroupMapping.java | 5 +-
.../java/mapping/CollectionSourceMapping.java | 5 +-
.../apache/wayang/java/mapping/CountMapping.java | 5 +-
.../wayang/java/mapping/DistinctMapping.java | 5 +-
.../apache/wayang/java/mapping/DoWhileMapping.java | 5 +-
.../apache/wayang/java/mapping/FilterMapping.java | 5 +-
.../apache/wayang/java/mapping/FlatMapMapping.java | 5 +-
.../mapping/GlobalMaterializedGroupMapping.java | 5 +-
.../wayang/java/mapping/GlobalReduceMapping.java | 5 +-
.../wayang/java/mapping/IntersectMapping.java | 5 +-
.../apache/wayang/java/mapping/JoinMapping.java | 5 +-
.../java/mapping/LocalCallbackSinkMapping.java | 5 +-
.../apache/wayang/java/mapping/LoopMapping.java | 5 +-
.../org/apache/wayang/java/mapping/MapMapping.java | 5 +-
.../wayang/java/mapping/MapPartitionsMapping.java | 5 +-
.../org/apache/wayang/java/mapping/Mappings.java | 7 +-
.../java/mapping/MaterializedGroupByMapping.java | 5 +-
...SinkMapping.java => ObjectFileSinkMapping.java} | 19 +-
...llMapping.java => ObjectFileSourceMapping.java} | 24 +-
.../wayang/java/mapping/ReduceByMapping.java | 5 +-
.../apache/wayang/java/mapping/RepeatMapping.java | 5 +-
.../apache/wayang/java/mapping/SampleMapping.java | 5 +-
.../apache/wayang/java/mapping/SortMapping.java | 5 +-
.../wayang/java/mapping/TextFileSinkMapping.java | 5 +-
.../wayang/java/mapping/TextFileSourceMapping.java | 5 +-
.../wayang/java/mapping/UnionAllMapping.java | 5 +-
.../wayang/java/mapping/ZipWithIdMapping.java | 5 +-
.../wayang/java/mapping/graph/PageRankMapping.java | 5 +-
.../java/operators/JavaCartesianOperator.java | 15 +-
.../wayang/java/operators/JavaCoGroupOperator.java | 21 +-
.../wayang/java/operators/JavaCollectOperator.java | 11 +-
.../java/operators/JavaCollectionSource.java | 7 +-
.../wayang/java/operators/JavaCountOperator.java | 9 +-
.../java/operators/JavaDistinctOperator.java | 9 +-
.../wayang/java/operators/JavaDoWhileOperator.java | 13 +-
.../java/operators/JavaExecutionOperator.java | 5 +-
.../wayang/java/operators/JavaFilterOperator.java | 13 +-
.../wayang/java/operators/JavaFlatMapOperator.java | 19 +-
.../JavaGlobalMaterializedGroupOperator.java | 9 +-
.../java/operators/JavaGlobalReduceOperator.java | 13 +-
.../java/operators/JavaIntersectOperator.java | 17 +-
.../wayang/java/operators/JavaJoinOperator.java | 21 +-
.../java/operators/JavaLocalCallbackSink.java | 11 +-
.../wayang/java/operators/JavaLoopOperator.java | 13 +-
.../wayang/java/operators/JavaMapOperator.java | 13 +-
.../java/operators/JavaMapPartitionsOperator.java | 19 +-
.../operators/JavaMaterializedGroupByOperator.java | 17 +-
.../wayang/java/operators/JavaObjectFileSink.java | 49 +--
.../java/operators/JavaObjectFileSource.java | 47 ++-
.../java/operators/JavaRandomSampleOperator.java | 19 +-
.../java/operators/JavaReduceByOperator.java | 29 +-
.../wayang/java/operators/JavaRepeatOperator.java | 9 +-
.../operators/JavaReservoirSampleOperator.java | 21 +-
.../wayang/java/operators/JavaSortOperator.java | 11 +-
.../wayang/java/operators/JavaTextFileSink.java | 19 +-
.../wayang/java/operators/JavaTextFileSource.java | 19 +-
.../wayang/java/operators/JavaTsvFileSink.java | 17 +-
.../wayang/java/operators/JavaTsvFileSource.java | 20 +-
.../java/operators/JavaUnionAllOperator.java | 11 +-
.../java/operators/graph/JavaPageRankOperator.java | 12 +-
.../apache/wayang/java/plugin/JavaBasicPlugin.java | 5 +-
.../java/plugin/JavaChannelConversionPlugin.java | 5 +-
.../apache/wayang/java/plugin/JavaGraphPlugin.java | 5 +-
.../wayang/java/execution/JavaExecutorTest.java | 11 +-
.../java/operators/JavaCartesianOperatorTest.java | 11 +-
.../java/operators/JavaCoGroupOperatorTest.java | 17 +-
.../java/operators/JavaCollectionSourceTest.java | 9 +-
.../java/operators/JavaCountOperatorTest.java | 9 +-
.../java/operators/JavaDistinctOperatorTest.java | 9 +-
.../operators/JavaExecutionOperatorTestBase.java | 13 +-
.../java/operators/JavaFilterOperatorTest.java | 11 +-
.../JavaGlobalMaterializedGroupOperatorTest.java | 11 +-
.../operators/JavaGlobalReduceOperatorTest.java | 13 +-
.../java/operators/JavaJoinOperatorTest.java | 15 +-
.../java/operators/JavaLocalCallbackSinkTest.java | 9 +-
.../JavaMaterializedGroupByOperatorTest.java | 15 +-
.../java/operators/JavaObjectFileSinkTest.java | 11 +-
.../java/operators/JavaObjectFileSourceTest.java | 13 +-
.../operators/JavaRandomSampleOperatorTest.java | 9 +-
.../java/operators/JavaReduceByOperatorTest.java | 13 +-
.../operators/JavaReservoirSampleOperatorTest.java | 9 +-
.../java/operators/JavaSortOperatorTest.java | 11 +-
.../java/operators/JavaTextFileSinkTest.java | 35 +-
.../java/operators/JavaUnionAllOperatorTest.java | 9 +-
.../apache/wayang/java/test/ChannelFactory.java | 11 +-
.../wayang/jdbc/channels/SqlQueryChannel.java | 3 +-
.../wayang/jdbc/execution/DatabaseDescriptor.java | 3 +-
.../apache/wayang/jdbc/execution/JdbcExecutor.java | 27 +-
.../jdbc/operators/JdbcExecutionOperator.java | 7 +-
.../wayang/jdbc/operators/JdbcFilterOperator.java | 5 +-
.../jdbc/operators/JdbcProjectionOperator.java | 5 +-
.../wayang/jdbc/operators/JdbcTableSource.java | 9 +-
.../wayang/jdbc/operators/SqlToStreamOperator.java | 29 +-
.../wayang/jdbc/platform/JdbcPlatformTemplate.java | 3 +-
.../wayang/jdbc/execution/JdbcExecutorTest.java | 15 +-
.../wayang/jdbc/operators/JdbcTableSourceTest.java | 17 +-
.../wayang/jdbc/operators/OperatorTestBase.java | 8 +-
.../jdbc/operators/SqlToStreamOperatorTest.java | 23 +-
.../wayang/jdbc/test/HsqldbFilterOperator.java | 3 +-
.../wayang/jdbc/test/HsqldbProjectionOperator.java | 3 +-
.../apache/wayang/jdbc/test/HsqldbTableSource.java | 3 +-
.../postgres/channels/ChannelConversions.java | 5 +-
.../wayang/postgres/mapping/FilterMapping.java | 5 +-
.../apache/wayang/postgres/mapping/Mappings.java | 3 +-
.../wayang/postgres/mapping/ProjectionMapping.java | 5 +-
.../postgres/operators/PostgresTableSource.java | 3 +-
.../postgres/plugin/PostgresConversionsPlugin.java | 7 +-
.../wayang/postgres/plugin/PostgresPlugin.java | 5 +-
.../wayang/spark/channels/ChannelConversions.java | 5 +-
.../apache/wayang/spark/channels/RddChannel.java | 4 +-
.../spark/compiler/BinaryOperatorAdapter.java | 3 +-
.../compiler/ExtendedFlatMapFunctionAdapter.java | 3 +-
.../ExtendedMapPartitionsFunctionAdapter.java | 9 +-
.../spark/compiler/FlatMapFunctionAdapter.java | 3 +-
.../wayang/spark/compiler/FunctionCompiler.java | 7 +-
.../compiler/MapPartitionsFunctionAdapter.java | 5 +-
.../wayang/spark/compiler/PredicateAdapter.java | 3 +-
.../spark/execution/SparkExecutionContext.java | 9 +-
.../wayang/spark/execution/SparkExecutor.java | 7 +-
.../wayang/spark/mapping/CartesianMapping.java | 5 +-
.../wayang/spark/mapping/CoGroupMapping.java | 5 +-
.../spark/mapping/CollectionSourceMapping.java | 5 +-
.../apache/wayang/spark/mapping/CountMapping.java | 5 +-
.../wayang/spark/mapping/DistinctMapping.java | 5 +-
.../wayang/spark/mapping/DoWhileMapping.java | 5 +-
.../apache/wayang/spark/mapping/FilterMapping.java | 5 +-
.../wayang/spark/mapping/FlatMapMapping.java | 5 +-
.../mapping/GlobalMaterializedGroupMapping.java | 5 +-
.../wayang/spark/mapping/GlobalReduceMapping.java | 5 +-
.../wayang/spark/mapping/IntersectMapping.java | 5 +-
.../apache/wayang/spark/mapping/JoinMapping.java | 5 +-
.../spark/mapping/LocalCallbackSinkMapping.java | 5 +-
.../apache/wayang/spark/mapping/LoopMapping.java | 5 +-
.../apache/wayang/spark/mapping/MapMapping.java | 5 +-
.../wayang/spark/mapping/MapPartitionsMapping.java | 5 +-
.../org/apache/wayang/spark/mapping/Mappings.java | 7 +-
.../spark/mapping/MaterializedGroupByMapping.java | 5 +-
...sectMapping.java => ObjectFileSinkMapping.java} | 24 +-
...anMapping.java => ObjectFileSourceMapping.java} | 24 +-
.../wayang/spark/mapping/ReduceByMapping.java | 5 +-
.../apache/wayang/spark/mapping/RepeatMapping.java | 5 +-
.../apache/wayang/spark/mapping/SampleMapping.java | 5 +-
.../apache/wayang/spark/mapping/SortMapping.java | 5 +-
.../wayang/spark/mapping/TextFileSinkMapping.java | 5 +-
.../spark/mapping/TextFileSourceMapping.java | 5 +-
.../wayang/spark/mapping/UnionAllMapping.java | 5 +-
.../wayang/spark/mapping/ZipWithIdMapping.java | 5 +-
.../spark/mapping/graph/PageRankMapping.java | 7 +-
.../operators/SparkBernoulliSampleOperator.java | 13 +-
.../spark/operators/SparkBroadcastOperator.java | 7 +-
.../wayang/spark/operators/SparkCacheOperator.java | 9 +-
.../spark/operators/SparkCartesianOperator.java | 9 +-
.../spark/operators/SparkCoGroupOperator.java | 9 +-
.../spark/operators/SparkCollectOperator.java | 11 +-
.../spark/operators/SparkCollectionSource.java | 11 +-
.../wayang/spark/operators/SparkCountOperator.java | 9 +-
.../spark/operators/SparkDistinctOperator.java | 9 +-
.../spark/operators/SparkDoWhileOperator.java | 11 +-
.../spark/operators/SparkExecutionOperator.java | 3 +-
.../spark/operators/SparkFilterOperator.java | 11 +-
.../spark/operators/SparkFlatMapOperator.java | 11 +-
.../SparkGlobalMaterializedGroupOperator.java | 13 +-
.../spark/operators/SparkGlobalReduceOperator.java | 11 +-
.../spark/operators/SparkIntersectOperator.java | 9 +-
.../wayang/spark/operators/SparkJoinOperator.java | 11 +-
.../spark/operators/SparkLocalCallbackSink.java | 11 +-
.../wayang/spark/operators/SparkLoopOperator.java | 11 +-
.../wayang/spark/operators/SparkMapOperator.java | 11 +-
.../operators/SparkMapPartitionsOperator.java | 13 +-
.../SparkMaterializedGroupByOperator.java | 11 +-
.../spark/operators/SparkObjectFileSink.java | 32 +-
.../spark/operators/SparkObjectFileSource.java | 28 +-
.../SparkRandomPartitionSampleOperator.java | 31 +-
.../spark/operators/SparkReduceByOperator.java | 11 +-
.../spark/operators/SparkRepeatOperator.java | 9 +-
.../SparkShufflePartitionSampleOperator.java | 19 +-
.../wayang/spark/operators/SparkSortOperator.java | 9 +-
.../wayang/spark/operators/SparkTextFileSink.java | 9 +-
.../spark/operators/SparkTextFileSource.java | 9 +-
.../wayang/spark/operators/SparkTsvFileSink.java | 9 +-
.../wayang/spark/operators/SparkTsvFileSource.java | 7 +-
.../spark/operators/SparkUnionAllOperator.java | 9 +-
.../spark/operators/SparkZipWithIdOperator.java | 9 +-
.../wayang/spark/platform/SparkPlatform.java | 9 +-
.../wayang/spark/plugin/SparkBasicPlugin.java | 5 +-
.../wayang/spark/plugin/SparkConversionPlugin.java | 7 +-
.../wayang/spark/plugin/SparkGraphPlugin.java | 5 +-
.../SparkBernoulliSampleOperatorTest.java | 11 +-
.../operators/SparkCartesianOperatorTest.java | 9 +-
.../spark/operators/SparkCoGroupOperatorTest.java | 19 +-
.../spark/operators/SparkCollectionSourceTest.java | 11 +-
.../spark/operators/SparkCountOperatorTest.java | 9 +-
.../spark/operators/SparkDistinctOperatorTest.java | 9 +-
.../spark/operators/SparkFilterOperatorTest.java | 9 +-
.../spark/operators/SparkFlatMapOperatorTest.java | 9 +-
.../SparkGlobalMaterializedGroupOperatorTest.java | 11 +-
.../operators/SparkGlobalReduceOperatorTest.java | 13 +-
.../spark/operators/SparkJoinOperatorTest.java | 9 +-
.../operators/SparkMapPartitionsOperatorTest.java | 11 +-
.../SparkMaterializedGroupByOperatorTest.java | 17 +-
.../spark/operators/SparkObjectFileSinkTest.java | 13 +-
.../spark/operators/SparkObjectFileSourceTest.java | 15 +-
.../spark/operators/SparkOperatorTestBase.java | 11 +-
.../SparkRandomPartitionSampleOperatorTest.java | 9 +-
.../spark/operators/SparkReduceByOperatorTest.java | 15 +-
.../SparkShufflePartitionSampleOperatorTest.java | 9 +-
.../spark/operators/SparkSortOperatorTest.java | 9 +-
.../spark/operators/SparkTextFileSinkTest.java | 9 +-
.../spark/operators/SparkUnionAllOperatorTest.java | 9 +-
.../apache/wayang/spark/test/ChannelFactory.java | 9 +-
.../sqlite3/channels/ChannelConversions.java | 5 +-
.../wayang/sqlite3/mapping/FilterMapping.java | 5 +-
.../apache/wayang/sqlite3/mapping/Mappings.java | 3 +-
.../wayang/sqlite3/mapping/ProjectionMapping.java | 5 +-
.../sqlite3/operators/Sqlite3TableSource.java | 3 +-
.../sqlite3/plugin/Sqlite3ConversionPlugin.java | 7 +-
.../wayang/sqlite3/plugin/Sqlite3Plugin.java | 5 +-
.../main/java/org/apache/wayang/iejoin/IEJoin.java | 9 +-
.../org/apache/wayang/iejoin/mapping/Mappings.java | 5 +-
.../wayang/iejoin/mapping/java/IEJoinMapping.java | 5 +-
.../iejoin/mapping/java/IESelfJoinMapping.java | 5 +-
.../wayang/iejoin/mapping/spark/IEJoinMapping.java | 5 +-
.../iejoin/mapping/spark/IESelfJoinMapping.java | 5 +-
.../iejoin/operators/JavaIEJoinOperator.java | 17 +-
.../iejoin/operators/JavaIESelfJoinOperator.java | 17 +-
.../iejoin/operators/SparkIEJoinOperator.java | 13 +-
.../iejoin/operators/SparkIESelfJoinOperator.java | 13 +-
.../iejoin/operators/java_helpers/BitSetJoin.java | 5 +-
.../operators/java_helpers/DataComparator.java | 5 +-
.../iejoin/operators/java_helpers/extractData.java | 3 +-
.../iejoin/operators/java_helpers/myMergeSort.java | 5 +-
.../operators/java_helpers/revDataComparator.java | 5 +-
.../iejoin/operators/spark_helpers/BitSetJoin.java | 7 +-
.../spark_helpers/List2AttributesObjectSkinny.java | 3 +-
.../operators/spark_helpers/addUniqueID.java | 5 +-
.../operators/spark_helpers/build2ListObject.java | 7 +-
.../operators/spark_helpers/myMergeSort.java | 3 +-
.../operators/spark_helpers/revDataComparator.java | 3 +-
.../operators/JavaExecutionOperatorTestBase.java | 13 +-
.../iejoin/operators/JavaIEJoinOperatorTest.java | 13 +-
.../iejoin/operators/SparkIEJoinOperatorTest.java | 9 +-
.../iejoin/operators/SparkIEJoinOperatorTest2.java | 9 +-
.../iejoin/operators/SparkIEJoinOperatorTest3.java | 9 +-
.../iejoin/operators/SparkIEJoinOperatorTest4.java | 9 +-
.../operators/SparkIESelfJoinOperatorTest.java | 9 +-
.../iejoin/operators/SparkOperatorTestBase.java | 11 +-
.../apache/wayang/iejoin/test/ChannelFactory.java | 11 +-
.../wayang/profiler/data/DataGenerators.java | 3 +-
.../wayang/profiler/hardware/DiskProfiler.java | 11 +-
.../profiler/java/BinaryOperatorProfiler.java | 7 +-
.../java/JavaCollectionSourceProfiler.java | 5 +-
.../profiler/java/JavaTextFileSourceProfiler.java | 3 +-
.../wayang/profiler/java/OperatorProfiler.java | 17 +-
.../wayang/profiler/java/OperatorProfilers.java | 13 +-
.../org/apache/wayang/profiler/java/Profiler.java | 15 +-
.../apache/wayang/profiler/java/SinkProfiler.java | 7 +-
.../wayang/profiler/java/SourceProfiler.java | 6 +-
.../profiler/java/UnaryOperatorProfiler.java | 7 +-
.../profiler/log/DynamicEstimationContext.java | 3 +-
.../wayang/profiler/log/DynamicLoadEstimator.java | 13 +-
.../profiler/log/DynamicLoadProfileEstimator.java | 7 +-
.../profiler/log/DynamicLoadProfileEstimators.java | 9 +-
.../wayang/profiler/log/GeneticOptimizer.java | 15 +-
.../wayang/profiler/log/GeneticOptimizerApp.java | 39 +--
.../org/apache/wayang/profiler/log/Individual.java | 11 +-
.../apache/wayang/profiler/log/LogEvaluator.java | 23 +-
.../wayang/profiler/log/OptimizationSpace.java | 3 +-
.../profiler/spark/BinaryOperatorProfiler.java | 3 +-
.../org/apache/wayang/profiler/spark/Main.java | 15 +-
.../wayang/profiler/spark/OperatorProfilers.java | 7 +-
.../apache/wayang/profiler/spark/SinkProfiler.java | 3 +-
.../spark/SparkCollectionSourceProfiler.java | 5 +-
.../profiler/spark/SparkOperatorProfiler.java | 19 +-
.../wayang/profiler/spark/SparkSourceProfiler.java | 3 +-
.../spark/SparkTextFileSourceProfiler.java | 9 +-
.../profiler/spark/SparkUnaryOperatorProfiler.java | 3 +-
.../wayang/profiler/util/ProfilingUtils.java | 2 +-
.../apache/wayang/profiler/util/RrdAccessor.java | 9 +-
.../apache/wayang/tests/FlinkIntegrationIT.java | 25 +-
.../org/apache/wayang/tests/FullIntegrationIT.java | 41 ++-
.../apache/wayang/tests/GiraphIntegrationIT.java | 15 +-
.../org/apache/wayang/tests/JavaIntegrationIT.java | 33 +-
.../apache/wayang/tests/PostgresIntegrationIT.java | 11 +-
.../java/org/apache/wayang/tests/RegressionIT.java | 9 +-
.../apache/wayang/tests/SparkIntegrationIT.java | 33 +-
.../java/org/apache/wayang/tests/WayangPlans.java | 27 +-
.../apache/wayang/tests/WayangPlansOperators.java | 9 +-
.../java/org/apache/wayang/tests/WordCountIT.java | 25 +-
.../wayang/tests/platform/MyMadeUpPlatform.java | 9 +-
660 files changed, 4446 insertions(+), 3249 deletions(-)