You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gearpump.apache.org by hu...@apache.org on 2016/04/26 11:42:59 UTC
[49/49] incubator-gearpump git commit: GEARPUMP-11, fix code style
GEARPUMP-11, fix code style
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/c176e448
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/c176e448
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/c176e448
Branch: refs/heads/master
Commit: c176e4485d2689eb9953977d6513efae4666ff9c
Parents: 678a509
Author: Sean Zhong <cl...@gmail.com>
Authored: Thu Apr 7 00:31:11 2016 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Tue Apr 26 14:27:02 2016 +0800
----------------------------------------------------------------------
.travis.yml | 12 +-
CHANGELOG.md | 41 +--
CONTRIBUTING.md | 3 +-
LICENSE | 26 --
README.md | 8 +
ReleaseProcess.md | 1 -
conf/gear.conf | 6 -
conf/log4j.properties | 5 +-
.../netty/ITransportMessageSerializer.java | 2 +
.../gearpump/transport/netty/MessageBatch.java | 14 +-
.../transport/netty/MessageDecoder.java | 7 +-
.../transport/netty/MessageEncoder.java | 4 +-
.../netty/NettyRenameThreadFactory.java | 4 +-
.../gearpump/transport/netty/TaskMessage.java | 8 +-
.../transport/netty/WrappedChannelBuffer.java | 11 +-
.../main/java/io/gearpump/util/AkkaHelper.java | 7 +-
.../io/gearpump/util/HadoopFSLogAppender.java | 2 +-
.../util/RecreateRollingFileAppender.java | 59 ++--
core/src/main/resources/geardefault.conf | 41 +--
core/src/main/resources/log4j.properties | 5 +-
core/src/main/scala/io/gearpump/Message.scala | 8 +-
.../io/gearpump/cluster/AppDescription.scala | 122 ++++---
.../io/gearpump/cluster/ClusterConfig.scala | 27 +-
.../gearpump/cluster/ClusterConfigSource.scala | 3 +-
.../io/gearpump/cluster/ClusterMessage.scala | 162 +++++++--
.../scala/io/gearpump/cluster/UserConfig.scala | 40 ++-
.../appmaster/AppMasterRuntimeEnvironment.scala | 63 ++--
.../appmaster/AppMasterRuntimeInfo.scala | 23 +-
.../cluster/appmaster/ApplicationState.scala | 44 +--
.../cluster/appmaster/ExecutorSystem.scala | 11 +-
.../appmaster/ExecutorSystemLauncher.scala | 38 ++-
.../appmaster/ExecutorSystemScheduler.scala | 66 ++--
.../appmaster/MasterConnectionKeeper.scala | 30 +-
.../gearpump/cluster/client/ClientContext.scala | 67 ++--
.../gearpump/cluster/client/MasterClient.scala | 45 ++-
.../gearpump/cluster/main/ArgumentsParser.scala | 49 +--
.../cluster/master/AppMasterLauncher.scala | 56 +--
.../gearpump/cluster/master/MasterProxy.scala | 43 ++-
.../gearpump/cluster/master/MasterSummary.scala | 27 +-
.../gearpump/cluster/scheduler/Resource.scala | 54 +--
.../worker/ExecutorProcessLauncher.scala | 47 +--
.../io/gearpump/cluster/worker/WorkerId.scala | 63 ++++
.../gearpump/cluster/worker/WorkerSummary.scala | 39 +--
.../io/gearpump/jarstore/JarStoreService.scala | 19 +-
.../io/gearpump/metrics/AkkaReporter.scala | 17 +-
.../scala/io/gearpump/metrics/Counter.scala | 6 +-
.../scala/io/gearpump/metrics/Histogram.scala | 18 +-
.../io/gearpump/metrics/JvmMetricsSet.scala | 24 +-
.../main/scala/io/gearpump/metrics/Meter.scala | 7 +-
.../scala/io/gearpump/metrics/Metrics.scala | 53 +--
.../io/gearpump/metrics/MetricsAggregator.scala | 13 +-
.../metrics/MetricsReporterService.scala | 39 ++-
core/src/main/scala/io/gearpump/package.scala | 62 +---
.../partitioner/BroadcastPartitioner.scala | 6 +-
.../partitioner/CoLocationPartitioner.scala | 20 +-
.../gearpump/partitioner/HashPartitioner.scala | 4 +-
.../io/gearpump/partitioner/Partitioner.scala | 62 ++--
.../ShuffleGroupingPartitioner.scala | 4 +-
.../partitioner/ShufflePartitioner.scala | 11 +-
.../io/gearpump/security/Authenticator.scala | 25 +-
.../security/ConfigFileBasedAuthenticator.scala | 44 ++-
.../io/gearpump/security/PasswordUtil.scala | 24 +-
.../FastKryoSerializationFramework.scala | 11 +-
.../serializer/FastKryoSerializer.scala | 11 +-
.../serializer/GearpumpSerialization.scala | 18 +-
.../serializer/SerializationFramework.scala | 5 +-
.../io/gearpump/serializer/Serializer.scala | 7 +-
.../scala/io/gearpump/transport/Express.scala | 37 +-
.../scala/io/gearpump/transport/HostPort.scala | 4 +-
.../io/gearpump/transport/netty/Client.scala | 59 ++--
.../io/gearpump/transport/netty/Context.scala | 78 +++--
.../io/gearpump/transport/netty/IContext.scala | 13 +-
.../gearpump/transport/netty/NettyConfig.scala | 10 +-
.../io/gearpump/transport/netty/NettyUtil.scala | 21 +-
.../io/gearpump/transport/netty/Server.scala | 45 ++-
.../io/gearpump/util/ActorSystemBooter.scala | 68 ++--
.../main/scala/io/gearpump/util/ActorUtil.scala | 70 ++--
.../main/scala/io/gearpump/util/AkkaApp.scala | 25 +-
.../main/scala/io/gearpump/util/Constants.scala | 36 +-
.../main/scala/io/gearpump/util/FileUtils.scala | 7 +-
.../src/main/scala/io/gearpump/util/Graph.scala | 67 ++--
.../gearpump/util/HistoryMetricsService.scala | 108 +++---
.../main/scala/io/gearpump/util/LogUtil.scala | 25 +-
.../io/gearpump/util/ProcessLogRedirector.scala | 6 +-
.../scala/io/gearpump/util/ReferenceEqual.scala | 9 +-
.../scala/io/gearpump/util/RestartPolicy.scala | 17 +-
.../scala/io/gearpump/util/RichProcess.scala | 10 +-
.../io/gearpump/util/TimeOutScheduler.scala | 10 +-
core/src/main/scala/io/gearpump/util/Util.scala | 71 ++--
core/src/test/resources/log4j.properties | 4 +-
core/src/test/resources/test.conf | 3 -
.../test/scala/io/gearpump/TestProbeUtil.scala | 6 +-
.../io/gearpump/cluster/MasterHarness.scala | 43 +--
.../scala/io/gearpump/cluster/TestUtil.scala | 10 +-
.../AppMasterRuntimeEnvironmentSpec.scala | 41 ++-
.../appmaster/ExecutorSystemLauncherSpec.scala | 30 +-
.../appmaster/ExecutorSystemSchedulerSpec.scala | 36 +-
.../appmaster/MasterConnectionKeeperSpec.scala | 59 ++--
.../cluster/main/ArgumentParserSpec.scala | 4 +-
.../cluster/master/AppMasterLauncherSpec.scala | 35 +-
.../cluster/master/ApplicationStateSpec.scala | 6 +-
.../cluster/master/MasterProxySpec.scala | 3 +-
.../io/gearpump/cluster/master/MasterSpec.scala | 1 -
.../scala/io/gearpump/metrics/MetricsSpec.scala | 10 +-
.../gearpump/partitioner/PartitionerSpec.scala | 7 +-
.../ConfigFileBasedAuthenticatorSpec.scala | 20 +-
.../io/gearpump/security/PasswordUtilSpec.scala | 9 +-
.../io/gearpump/serializer/SerializerSpec.scala | 17 +-
.../transport/MockTransportSerializer.scala | 8 +-
.../scala/io/gearpump/transport/NettySpec.scala | 21 +-
.../gearpump/util/ActorSystemBooterSpec.scala | 44 +--
.../scala/io/gearpump/util/ActorUtilSpec.scala | 32 +-
.../scala/io/gearpump/util/ConfigsSpec.scala | 21 +-
.../scala/io/gearpump/util/FileUtilsSpec.scala | 5 +-
.../test/scala/io/gearpump/util/GraphSpec.scala | 17 +-
.../io/gearpump/util/TimeOutSchedulerSpec.scala | 14 +-
.../test/scala/io/gearpump/util/UtilSpec.scala | 9 +-
.../io.gearpump.jarstore.JarStoreService | 18 +
.../io/gearpump/cluster/DaemonMessage.scala | 21 +-
.../cluster/embedded/EmbeddedCluster.scala | 36 +-
.../io/gearpump/cluster/main/AppSubmitter.scala | 86 ++---
.../scala/io/gearpump/cluster/main/Gear.scala | 21 +-
.../scala/io/gearpump/cluster/main/Info.scala | 14 +-
.../scala/io/gearpump/cluster/main/Kill.scala | 23 +-
.../scala/io/gearpump/cluster/main/Local.scala | 34 +-
.../io/gearpump/cluster/main/MainRunner.scala | 10 +-
.../scala/io/gearpump/cluster/main/Master.scala | 90 ++---
.../scala/io/gearpump/cluster/main/Replay.scala | 21 +-
.../scala/io/gearpump/cluster/main/Worker.scala | 26 +-
.../io/gearpump/cluster/master/AppManager.scala | 100 +++---
.../cluster/master/InMemoryKVService.scala | 39 +--
.../io/gearpump/cluster/master/Master.scala | 113 ++++---
.../cluster/scheduler/PriorityScheduler.scala | 52 +--
.../gearpump/cluster/scheduler/Scheduler.scala | 35 +-
.../worker/DefaultExecutorProcessLauncher.scala | 11 +-
.../io/gearpump/cluster/worker/Worker.scala | 245 ++++++++------
.../jarstore/dfs/DFSJarStoreService.scala | 17 +-
.../gearpump/jarstore/local/LocalJarStore.scala | 18 +-
.../jarstore/local/LocalJarStoreService.scala | 39 ++-
.../scala/io/gearpump/util/FileDirective.scala | 30 +-
.../scala/io/gearpump/util/FileServer.scala | 42 +--
.../io.gearpump.jarstore.JarStoreService | 18 +
.../scala/io/gearpump/cluster/MiniCluster.scala | 26 +-
.../io/gearpump/cluster/main/MainSpec.scala | 102 +++---
.../cluster/main/MasterWatcherSpec.scala | 20 +-
.../cluster/master/AppManagerSpec.scala | 55 +--
.../cluster/master/InMemoryKVServiceSpec.scala | 26 +-
.../scheduler/PrioritySchedulerSpec.scala | 123 +++++--
.../io/gearpump/cluster/worker/WorkerSpec.scala | 55 +--
.../scala/io/gearpump/util/FileServerSpec.scala | 31 +-
docs/README.md | 31 +-
docs/_layouts/404.html | 310 ++++++++---------
docs/_layouts/global.html | 267 ++++++++-------
docs/build_doc.sh | 3 +-
docs/css/api-javadocs.css | 4 +-
docs/css/main.css | 29 +-
docs/deployment-configuration.md | 3 +-
docs/deployment-msg-delivery.md | 4 +-
docs/deployment-resource-isolation.md | 12 +-
docs/deployment-security.md | 4 +-
docs/deployment-standalone.md | 1 -
docs/deployment-ui-authentication.md | 7 +-
docs/dev-connectors.md | 5 +-
docs/dev-custom-serializer.md | 2 -
docs/dev-non-streaming-example.md | 1 -
docs/dev-rest-api.md | 5 -
docs/dev-storm.md | 11 +-
docs/dev-write-1st-app.md | 5 +-
docs/features.md | 3 +-
docs/gearpump-internals.md | 6 +-
docs/hardware-requirement.md | 1 -
docs/how-to-contribute.md | 2 -
docs/index.md | 3 +-
docs/js/api-docs.js | 4 +-
docs/js/api-javadocs.js | 18 +-
docs/js/main.js | 19 +-
docs/message-delivery.md | 11 +-
.../distributedshell/DistShellAppMaster.scala | 31 +-
.../distributedshell/DistributedShell.scala | 13 +-
.../DistributedShellClient.scala | 18 +-
.../distributedshell/ShellExecutor.scala | 18 +-
.../DistShellAppMasterSpec.scala | 38 ++-
.../DistributedShellClientSpec.scala | 31 +-
.../distributedshell/DistributedShellSpec.scala | 24 +-
.../ShellCommandResultAggregatorSpec.scala | 4 +-
.../distributedshell/ShellExecutorSpec.scala | 30 +-
examples/distributeservice/README.md | 2 +-
.../DistServiceAppMaster.scala | 40 ++-
.../distributeservice/DistServiceExecutor.scala | 38 ++-
.../distributeservice/DistributeService.scala | 15 +-
.../DistributeServiceClient.scala | 32 +-
.../distributeservice/FileServer.scala | 33 +-
.../DistServiceAppMasterSpec.scala | 50 +--
.../pagerank/PageRankApplication.scala | 17 +-
.../pagerank/PageRankController.scala | 20 +-
.../experiments/pagerank/PageRankWorker.scala | 29 +-
.../pagerank/example/PageRankExample.scala | 9 +-
.../streaming/examples/complexdag/Dag.scala | 77 ++---
.../streaming/examples/complexdag/Node.scala | 10 +-
.../streaming/examples/complexdag/Sink.scala | 9 +-
.../streaming/examples/complexdag/Source.scala | 6 +-
.../streaming/examples/complexdag/DagSpec.scala | 24 +-
.../examples/complexdag/NodeSpec.scala | 18 +-
.../examples/complexdag/SinkSpec.scala | 12 +-
.../examples/complexdag/SourceSpec.scala | 13 +-
.../streaming/examples/fsio/HadoopConfig.scala | 25 +-
.../examples/fsio/SeqFileStreamProcessor.scala | 43 +--
.../examples/fsio/SeqFileStreamProducer.scala | 30 +-
.../examples/fsio/SequenceFileIO.scala | 29 +-
.../examples/fsio/HadoopConfigSpec.scala | 7 +-
.../fsio/SeqFileStreamProcessorSpec.scala | 25 +-
.../fsio/SeqFileStreamProducerSpec.scala | 34 +-
.../examples/fsio/SequenceFileIOSpec.scala | 24 +-
.../examples/kafka/KafkaReadWrite.scala | 27 +-
.../kafka/wordcount/KafkaWordCount.scala | 30 +-
.../examples/kafka/wordcount/Split.scala | 16 +-
.../examples/kafka/wordcount/Sum.scala | 11 +-
.../kafka/wordcount/KafkaWordCountSpec.scala | 24 +-
.../examples/kafka/wordcount/SplitSpec.scala | 9 +-
.../examples/kafka/wordcount/SumSpec.scala | 17 +-
examples/streaming/sol/README.md | 2 +-
.../gearpump/streaming/examples/sol/SOL.scala | 21 +-
.../examples/sol/SOLStreamProcessor.scala | 33 +-
.../examples/sol/SOLStreamProducer.scala | 32 +-
.../streaming/examples/sol/SOLSpec.scala | 24 +-
.../examples/sol/SOLStreamProcessorSpec.scala | 11 +-
.../examples/sol/SOLStreamProducerSpec.scala | 11 +-
.../examples/state/MessageCountApp.scala | 41 ++-
.../examples/state/WindowAverageApp.scala | 35 +-
.../state/processor/CountProcessor.scala | 9 +-
.../processor/NumberGeneratorProcessor.scala | 7 +-
.../processor/WindowAverageProcessor.scala | 21 +-
.../examples/state/MessageCountAppSpec.scala | 25 +-
.../examples/state/WindowAverageAppSpec.scala | 22 +-
.../state/processor/CountProcessorSpec.scala | 41 ++-
.../NumberGeneratorProcessorSpec.scala | 25 +-
.../processor/WindowAverageProcessorSpec.scala | 45 ++-
.../src/main/resources/stock/css/custom.css | 95 +++---
.../src/main/resources/stock/js/stock.js | 108 +++---
.../src/main/resources/stock/stock.html | 119 ++++---
.../streaming/examples/stock/Analyzer.scala | 65 ++--
.../streaming/examples/stock/Crawler.scala | 17 +-
.../streaming/examples/stock/Data.scala | 28 +-
.../streaming/examples/stock/QueryServer.scala | 44 +--
.../streaming/examples/stock/StockMarket.scala | 43 +--
.../streaming/examples/stock/main/Stock.scala | 35 +-
examples/streaming/transport/README.md | 3 +
.../src/main/resources/transport/css/custom.css | 95 +++---
.../main/resources/transport/js/transport.js | 296 ++++++++--------
.../src/main/resources/transport/transport.html | 121 ++++---
.../streaming/examples/transport/Data.scala | 4 +-
.../examples/transport/DataSource.scala | 111 +++---
.../examples/transport/QueryServer.scala | 44 +--
.../examples/transport/Transport.scala | 131 +++----
.../examples/transport/VelocityInspector.scala | 237 ++++++-------
.../examples/transport/generator/MockCity.scala | 174 +++++-----
.../generator/PassRecordGenerator.scala | 137 ++++----
.../examples/transport/DataSourceSpec.scala | 15 +-
.../examples/transport/TransportSpec.scala | 29 +-
.../transport/generator/MockCitySpec.scala | 63 ++--
.../generator/PassRecordGeneratorSpec.scala | 13 +-
.../streaming/examples/wordcountjava/Split.java | 2 +-
.../streaming/examples/wordcountjava/Sum.java | 6 +-
.../examples/wordcountjava/WordCount.java | 5 +-
.../examples/wordcountjava/dsl/WordCount.java | 10 +-
.../examples/wordcountjava/WordCountSpec.scala | 22 +-
.../streaming/examples/wordcount/Split.scala | 15 +-
.../streaming/examples/wordcount/Sum.scala | 44 +--
.../examples/wordcount/WordCount.scala | 23 +-
.../examples/wordcount/dsl/WordCount.scala | 5 +-
.../examples/wordcount/SplitSpec.scala | 24 +-
.../streaming/examples/wordcount/SumSpec.scala | 13 +-
.../examples/wordcount/WordCountSpec.scala | 23 +-
experiments/akkastream/README.md | 4 +-
.../scala/akka/stream/BaseMaterializer.scala | 7 +-
.../main/scala/akka/stream/ModuleGraph.scala | 19 +-
.../akka/stream/gearpump/GearAttributes.scala | 8 +-
.../stream/gearpump/GearpumpMaterializer.scala | 25 +-
.../akka/stream/gearpump/example/Test.scala | 37 +-
.../akka/stream/gearpump/example/Test2.scala | 21 +-
.../akka/stream/gearpump/example/Test3.scala | 15 +-
.../akka/stream/gearpump/example/Test4.scala | 15 +-
.../akka/stream/gearpump/example/Test5.scala | 15 +-
.../akka/stream/gearpump/example/Test6.scala | 23 +-
.../stream/gearpump/example/WikipediaApp.scala | 41 ++-
.../stream/gearpump/graph/GraphCutter.scala | 32 +-
.../akka/stream/gearpump/graph/LocalGraph.scala | 13 +-
.../stream/gearpump/graph/RemoteGraph.scala | 18 +-
.../akka/stream/gearpump/graph/SubGraph.scala | 9 +-
.../materializer/LocalMaterializer.scala | 59 ++--
.../materializer/LocalMaterializerImpl.scala | 149 ++++----
.../materializer/RemoteMaterializerImpl.scala | 62 ++--
.../stream/gearpump/module/BridgeModule.scala | 57 ++--
.../stream/gearpump/module/DummyModule.scala | 6 +-
.../gearpump/module/GearpumpTaskModule.scala | 33 +-
.../stream/gearpump/module/GroupByModule.scala | 10 +-
.../stream/gearpump/module/ReduceModule.scala | 7 +-
.../akka/stream/gearpump/scaladsl/Api.scala | 50 ++-
.../akka/stream/gearpump/task/BalanceTask.scala | 6 +-
.../stream/gearpump/task/BroadcastTask.scala | 6 +-
.../akka/stream/gearpump/task/GraphTask.scala | 23 +-
.../stream/gearpump/task/SinkBridgeTask.scala | 22 +-
.../stream/gearpump/task/SourceBridgeTask.scala | 27 +-
.../akka/stream/gearpump/task/UnZip2Task.scala | 7 +-
.../gearpump/util/MaterializedValueOps.scala | 8 +-
.../akka/stream/gearpump/AttributesSpec.scala | 5 +-
experiments/cgroup/README.md | 1 +
.../gearpump/cluster/cgroup/CGroupResource.java | 9 +-
.../gearpump/cluster/cgroup/CgroupCenter.java | 10 +-
.../gearpump/cluster/cgroup/CgroupCommon.java | 9 +-
.../cluster/cgroup/CgroupCommonOperation.java | 9 +-
.../cluster/cgroup/CgroupCoreFactory.java | 12 +-
.../cluster/cgroup/CgroupOperation.java | 9 +-
.../io/gearpump/cluster/cgroup/CgroupUtils.java | 9 +-
.../io/gearpump/cluster/cgroup/Constants.java | 9 +-
.../io/gearpump/cluster/cgroup/Hierarchy.java | 8 +-
.../gearpump/cluster/cgroup/ResourceType.java | 8 +-
.../cluster/cgroup/core/CgroupCore.java | 9 +-
.../gearpump/cluster/cgroup/core/CpuCore.java | 9 +-
.../gearpump/cluster/utils/SystemOperation.java | 72 ++--
.../gearpump/cluster/worker/CGroupManager.scala | 14 +-
.../cluster/worker/CGroupProcessLauncher.scala | 34 +-
experiments/storm/README.md | 5 +-
.../storm/util/TimeCacheMapWrapper.java | 42 +++
.../experiments/storm/StormRunner.scala | 17 +-
.../experiments/storm/main/GearpumpNimbus.scala | 95 +++---
.../storm/main/GearpumpStormClient.scala | 13 +-
.../storm/partitioner/StormPartitioner.scala | 20 +-
.../processor/StormBoltOutputCollector.scala | 11 +-
.../storm/processor/StormProcessor.scala | 15 +-
.../storm/producer/StormProducer.scala | 16 +-
.../producer/StormSpoutOutputCollector.scala | 19 +-
.../storm/topology/GearpumpStormComponent.scala | 108 +++---
.../storm/topology/GearpumpStormTopology.scala | 66 ++--
.../storm/topology/GearpumpTuple.scala | 16 +-
.../experiments/storm/util/GraphBuilder.scala | 3 +-
.../experiments/storm/util/Grouper.scala | 36 +-
.../experiments/storm/util/StormConstants.scala | 4 +-
.../storm/util/StormOutputCollector.scala | 59 ++--
.../util/StormSerializationFramework.scala | 9 +-
.../experiments/storm/util/StormUtil.scala | 30 +-
.../partitioner/StormPartitionerSpec.scala | 36 +-
.../StormBoltOutputCollectorSpec.scala | 15 +-
.../storm/processor/StormProcessorSpec.scala | 12 +-
.../storm/producer/StormProducerSpec.scala | 9 +-
.../StormSpoutOutputCollectorSpec.scala | 14 +-
.../topology/GearpumpStormComponentSpec.scala | 51 +--
.../topology/GearpumpStormTopologySpec.scala | 28 +-
.../storm/topology/GearpumpTupleSpec.scala | 15 +-
.../storm/util/GraphBuilderSpec.scala | 11 +-
.../experiments/storm/util/GrouperSpec.scala | 25 +-
.../storm/util/StormOutputCollectorSpec.scala | 56 +--
.../storm/util/StormSerializerPoolSpec.scala | 14 +-
.../experiments/storm/util/StormUtilSpec.scala | 38 +--
.../experiments/storm/util/TopologyUtil.scala | 2 +-
.../gearpump/experiments/yarn/Constants.scala | 2 +-
.../experiments/yarn/appmaster/Command.scala | 39 ++-
.../experiments/yarn/appmaster/UIService.scala | 23 +-
.../yarn/appmaster/YarnAppMaster.scala | 87 +++--
.../yarn/client/AppMasterResolver.scala | 16 +-
.../experiments/yarn/client/Client.scala | 16 +-
.../experiments/yarn/client/LaunchCluster.scala | 76 +++--
.../experiments/yarn/client/ManageCluster.scala | 46 ++-
.../yarn/glue/ContainerLaunchContext.scala | 31 +-
.../experiments/yarn/glue/FileSystem.scala | 22 +-
.../experiments/yarn/glue/NMClient.scala | 35 +-
.../experiments/yarn/glue/RMClient.scala | 22 +-
.../experiments/yarn/glue/Records.scala | 44 ++-
.../experiments/yarn/glue/YarnClient.scala | 25 +-
.../experiments/yarn/glue/YarnConfig.scala | 4 +-
.../experiments/yarn/glue/package.scala | 26 ++
.../yarn/appmaster/CommandSpec.scala | 17 +-
.../yarn/appmaster/UIServiceSpec.scala | 28 +-
.../yarn/appmaster/YarnAppMasterSpec.scala | 77 +++--
.../yarn/client/LaunchClusterSpec.scala | 55 +--
.../yarn/client/ManageClusterSpec.scala | 48 +--
external/hadoopfs/README.md | 2 +-
.../hadoop/HadoopCheckpointStore.scala | 47 +--
.../hadoop/HadoopCheckpointStoreFactory.scala | 18 +-
.../streaming/hadoop/SequenceFileSink.scala | 45 +--
.../lib/HadoopCheckpointStoreReader.scala | 9 +-
.../lib/HadoopCheckpointStoreWriter.scala | 5 +-
.../streaming/hadoop/lib/HadoopUtil.scala | 16 +-
.../lib/format/DefaultSequenceFormatter.scala | 7 +-
.../hadoop/lib/format/OutputFormatter.scala | 7 +-
.../hadoop/lib/rotation/FileSizeRotation.scala | 7 +-
.../hadoop/lib/rotation/Rotation.scala | 4 +-
.../HadoopCheckpointStoreIntegrationSpec.scala | 25 +-
.../lib/rotation/FileSizeRotationSpec.scala | 5 +-
external/hbase/README.md | 5 +-
.../io/gearpump/external/hbase/HBaseSink.scala | 54 ++-
.../external/hbase/dsl/HBaseDSLSink.scala | 17 +-
.../gearpump/external/hbase/HBaseSinkSpec.scala | 32 +-
.../io/gearpump/streaming/kafka/KafkaSink.scala | 17 +-
.../gearpump/streaming/kafka/KafkaSource.scala | 67 ++--
.../gearpump/streaming/kafka/KafkaStorage.scala | 39 +--
.../streaming/kafka/dsl/KafkaDSLSink.scala | 7 +-
.../streaming/kafka/dsl/KafkaDSLUtil.scala | 25 +-
.../kafka/lib/DefaultMessageDecoder.scala | 9 +-
.../kafka/lib/KafkaOffsetManager.scala | 17 +-
.../streaming/kafka/lib/KafkaSourceConfig.scala | 77 +++--
.../streaming/kafka/lib/KafkaUtil.scala | 42 +--
.../consumer/ExponentialBackoffSleeper.scala | 15 +-
.../kafka/lib/consumer/FetchThread.scala | 29 +-
.../kafka/lib/consumer/KafkaConsumer.scala | 23 +-
.../kafka/lib/consumer/KafkaMessage.scala | 8 +-
.../kafka/lib/grouper/KafkaDefaultGrouper.scala | 8 +-
.../kafka/lib/grouper/KafkaGrouper.scala | 6 +-
.../streaming/kafka/KafkaSinkSpec.scala | 7 +-
.../streaming/kafka/KafkaSourceSpec.scala | 38 +--
.../kafka/lib/DefaultMessageDecoderSpec.scala | 4 +-
.../kafka/lib/KafkaOffsetManagerSpec.scala | 28 +-
.../streaming/kafka/lib/KafkaStorageSpec.scala | 329 +++++++++---------
.../streaming/kafka/lib/KafkaUtilSpec.scala | 40 ++-
.../ExponentialBackoffSleeperSpec.scala | 18 +-
.../kafka/lib/consumer/FetchThreadSpec.scala | 31 +-
.../kafka/lib/consumer/KafkaConsumerSpec.scala | 2 +-
.../lib/grouper/KafkaDefaultGrouperSpec.scala | 3 +-
.../kafka/util/KafkaServerHarness.scala | 15 +-
.../streaming/kafka/util/ZookeeperHarness.scala | 9 +-
.../streaming/monoid/AlgebirdMonoid.scala | 5 +-
.../streaming/serializer/ChillSerializer.scala | 7 +-
integrationtest/README.md | 1 -
.../core/src/it/resources/log4j.properties | 2 +-
.../integrationtest/MiniClusterProvider.scala | 9 +-
.../gearpump/integrationtest/TestSpecBase.scala | 11 +-
.../checklist/CommandLineSpec.scala | 3 +-
.../checklist/ConnectorKafkaSpec.scala | 19 +-
.../checklist/DynamicDagSpec.scala | 24 +-
.../integrationtest/checklist/ExampleSpec.scala | 28 +-
.../checklist/MessageDeliverySpec.scala | 50 ++-
.../checklist/RestServiceSpec.scala | 42 ++-
.../checklist/StabilitySpec.scala | 38 ++-
.../checklist/StormCompatibilitySpec.scala | 37 +-
.../suites/StandaloneModeSuite.scala | 8 +-
.../io/gearpump/integrationtest/Docker.scala | 28 +-
.../io/gearpump/integrationtest/ShellExec.scala | 33 +-
.../io/gearpump/integrationtest/Util.scala | 14 +-
.../integrationtest/hadoop/HadoopCluster.scala | 28 +-
.../integrationtest/kafka/KafkaCluster.scala | 18 +-
.../kafka/NumericalDataProducer.scala | 11 +-
.../integrationtest/kafka/ResultVerifier.scala | 3 +-
.../kafka/SimpleKafkaReader.scala | 13 +-
.../minicluster/BaseContainer.scala | 11 +-
.../minicluster/CommandLineClient.scala | 5 +-
.../minicluster/MiniCluster.scala | 25 +-
.../minicluster/RestClient.scala | 40 ++-
.../integrationtest/storm/StormClient.scala | 25 +-
.../integrationtest/storm/Adaptor.scala | 6 +-
.../storm/Storm010KafkaTopology.scala | 24 +-
.../integrationtest/storm/Adaptor.scala | 6 +-
.../storm/Storm09KafkaTopology.scala | 24 +-
jvm.sbt | 28 +-
notes/0.2.3.markdown | 2 +-
notes/0.3.5.markdown | 2 +-
notes/0.4.1.markdown | 1 -
notes/0.4.markdown | 2 +-
notes/0.8.0.markdown | 1 -
project/Build.scala | 337 +++++++++++--------
project/BuildExample.scala | 232 +++++++------
project/BuildIntegrationTest.scala | 87 +++--
project/Pack.scala | 164 ++++++---
project/PhantomJS2Env.scala | 28 +-
project/build.properties | 4 +-
project/plugins.sbt | 26 +-
scalastyle-config.xml | 240 +++++++++++++
services/dashboard/dashboard.js | 192 +++++------
services/dashboard/index.html | 3 +-
services/dashboard/login/login.html | 23 +-
services/dashboard/login/login.js | 10 +-
.../dashboard/services/health_check_service.js | 14 +-
services/dashboard/services/locator.js | 10 +-
services/dashboard/services/login_check.js | 28 +-
services/dashboard/services/models/dag.js | 44 +--
services/dashboard/services/models/metrics.js | 18 +-
.../services/models/metrics_provider.js | 48 +--
services/dashboard/services/models/models.js | 166 ++++-----
.../services/models/streamingapp_dag.js | 64 ++--
.../models/streamingapp_metrics_provider.js | 34 +-
services/dashboard/services/restapi.js | 92 ++---
services/dashboard/styles/dashboard.css | 16 +-
.../dashboard/views/apps/app/alerts_table.html | 14 +-
.../dashboard/views/apps/app/alerts_table.js | 8 +-
services/dashboard/views/apps/app/app.js | 8 +-
.../views/apps/app/executors_table.html | 8 +-
.../dashboard/views/apps/app/executors_table.js | 14 +-
services/dashboard/views/apps/app/overview.js | 10 +-
services/dashboard/views/apps/apps.html | 3 +-
services/dashboard/views/apps/apps.js | 29 +-
.../dashboard/views/apps/compose/compose.js | 61 ++--
.../views/apps/compose/popups/choose_edge.js | 8 +-
.../apps/compose/popups/choose_processor.js | 6 +-
.../dashboard/views/apps/streamingapp/dag.html | 4 +-
.../dashboard/views/apps/streamingapp/dag.js | 36 +-
.../views/apps/streamingapp/executor.js | 17 +-
.../views/apps/streamingapp/metric_selector.js | 8 +-
.../views/apps/streamingapp/metrics_charts.html | 4 +-
.../views/apps/streamingapp/metrics_charts.js | 16 +-
.../views/apps/streamingapp/metrics_table.html | 8 +-
.../views/apps/streamingapp/metrics_table.js | 12 +-
.../views/apps/streamingapp/overview.js | 10 +-
.../views/apps/streamingapp/popups/dag_edit.js | 10 +-
.../views/apps/streamingapp/processor.html | 11 +-
.../views/apps/streamingapp/processor.js | 28 +-
.../apps/streamingapp/processor_task_charts.js | 20 +-
.../apps/streamingapp/processor_task_pager.js | 6 +-
.../apps/streamingapp/processor_task_table.js | 6 +-
.../views/apps/streamingapp/streamingapp.html | 6 +-
.../views/apps/streamingapp/streamingapp.js | 33 +-
.../apps/streamingapp/vis/network_style.js | 22 +-
.../dashboard/views/apps/submit/submit.html | 9 +-
services/dashboard/views/apps/submit/submit.js | 12 +-
.../dashboard/views/cluster/master/master.js | 10 +-
.../views/cluster/workers/add_worker.html | 1 +
.../views/cluster/workers/add_worker.js | 4 +-
.../views/cluster/workers/worker/worker.html | 3 +-
.../views/cluster/workers/worker/worker.js | 20 +-
.../views/cluster/workers/workers_listview.html | 1 +
.../views/cluster/workers/workers_listview.js | 32 +-
services/dashboard/views/helper.js | 22 +-
.../dashboard/views/jvm/jvm_metrics_view.js | 26 +-
services/dashboard/views/landing/breadcrumbs.js | 6 +-
services/dashboard/views/landing/header.html | 1 -
services/dashboard/views/landing/header.js | 6 +-
.../views/service_unreachable_notice.html | 3 +-
.../widgets/metrics_period_switcher.js | 6 +-
services/dashboard/widgets/radio_group.js | 6 +-
.../io/gearpump/dashboard/DashboardApp.scala | 2 +-
.../io/gearpump/services/AdminService.scala | 22 +-
.../io/gearpump/services/AppMasterService.scala | 88 ++---
.../io/gearpump/services/BasicService.scala | 23 +-
.../io/gearpump/services/MasterService.scala | 77 +++--
.../io/gearpump/services/RestServices.scala | 25 +-
.../io/gearpump/services/SecurityService.scala | 117 ++++---
.../io/gearpump/services/StaticService.scala | 18 +-
.../gearpump/services/SupervisorService.scala | 30 +-
.../io/gearpump/services/WorkerService.scala | 35 +-
.../io/gearpump/services/main/Services.scala | 67 ++--
.../scala/io/gearpump/services/package.scala | 2 +-
.../security/oauth2/OAuth2Authenticator.scala | 61 ++--
.../oauth2/impl/BaseOAuth2Authenticator.scala | 37 +-
.../CloudFoundryUAAOAuth2Authenticator.scala | 91 +++--
.../oauth2/impl/GoogleOAuth2Authenticator.scala | 40 ++-
.../io/gearpump/services/util/UpickleUtil.scala | 24 +-
.../io/gearpump/services/AdminServiceSpec.scala | 28 +-
.../services/AppMasterServiceSpec.scala | 44 ++-
.../gearpump/services/MasterServiceSpec.scala | 62 ++--
.../gearpump/services/SecurityServiceSpec.scala | 70 ++--
.../gearpump/services/StaticServiceSpec.scala | 32 +-
.../gearpump/services/WorkerServiceSpec.scala | 42 +--
...CloudFoundryUAAOAuth2AuthenticatorSpec.scala | 53 +--
.../oauth2/GoogleOAuth2AuthenticatorSpec.scala | 69 ++--
.../security/oauth2/MockOAuth2Server.scala | 25 +-
.../io/gearpump/services/util/UpickleSpec.scala | 7 +-
streaming/README.md | 116 -------
.../io/gearpump/streaming/javaapi/Graph.java | 11 +-
.../gearpump/streaming/javaapi/Processor.java | 50 +--
.../streaming/javaapi/StreamApplication.java | 19 +-
.../io/gearpump/streaming/javaapi/Task.java | 16 +-
.../javaapi/dsl/functions/FilterFunction.java | 6 +-
.../javaapi/dsl/functions/FlatMapFunction.java | 7 +-
.../javaapi/dsl/functions/GroupByFunction.java | 7 +-
.../javaapi/dsl/functions/MapFunction.java | 7 +-
.../javaapi/dsl/functions/ReduceFunction.java | 6 +-
.../io/gearpump/streaming/ClusterMessage.scala | 31 +-
.../scala/io/gearpump/streaming/Constants.scala | 25 +-
.../main/scala/io/gearpump/streaming/DAG.scala | 18 +-
.../gearpump/streaming/MessageSerializer.scala | 4 +-
.../gearpump/streaming/StreamApplication.scala | 52 +--
.../streaming/appmaster/AppMaster.scala | 140 ++++----
.../streaming/appmaster/ClockService.scala | 113 ++++---
.../streaming/appmaster/DagManager.scala | 63 ++--
.../streaming/appmaster/ExecutorManager.scala | 92 ++---
.../streaming/appmaster/JarScheduler.scala | 98 ++++--
.../appmaster/StreamAppMasterSummary.scala | 27 +-
.../streaming/appmaster/TaskLocator.scala | 39 ++-
.../streaming/appmaster/TaskManager.scala | 126 +++----
.../streaming/appmaster/TaskRegistry.scala | 36 +-
.../streaming/appmaster/TaskSchedulerImpl.scala | 72 ++--
.../io/gearpump/streaming/dsl/Stream.scala | 116 ++++---
.../io/gearpump/streaming/dsl/StreamApp.scala | 50 +--
.../streaming/dsl/javaapi/JavaStream.scala | 31 +-
.../streaming/dsl/javaapi/JavaStreamApp.scala | 10 +-
.../scala/io/gearpump/streaming/dsl/op/OP.scala | 36 +-
.../dsl/partitioner/GroupbyPartitioner.scala | 17 +-
.../streaming/dsl/plan/OpTranslator.scala | 103 ++++--
.../gearpump/streaming/dsl/plan/Planner.scala | 26 +-
.../gearpump/streaming/executor/Executor.scala | 155 +++++----
.../executor/ExecutorRestartPolicy.scala | 24 +-
.../streaming/executor/TaskLauncher.scala | 23 +-
.../streaming/metrics/ProcessorAggregator.scala | 62 ++--
.../metrics/TaskFilterAggregator.scala | 64 ++--
.../scala/io/gearpump/streaming/package.scala | 18 +
.../io/gearpump/streaming/sink/DataSink.scala | 16 +-
.../streaming/sink/DataSinkProcessor.scala | 17 +-
.../gearpump/streaming/sink/DataSinkTask.scala | 11 +-
.../gearpump/streaming/source/DataSource.scala | 14 +-
.../streaming/source/DataSourceConfig.scala | 3 +-
.../streaming/source/DataSourceProcessor.scala | 19 +-
.../streaming/source/DataSourceTask.scala | 21 +-
.../source/DefaultTimeStampFilter.scala | 6 +-
.../gearpump/streaming/state/api/Monoid.scala | 10 +-
.../streaming/state/api/MonoidState.scala | 6 +-
.../streaming/state/api/PersistentState.scala | 13 +-
.../streaming/state/api/PersistentTask.scala | 32 +-
.../streaming/state/api/Serializer.scala | 8 +-
.../state/impl/CheckpointManager.scala | 5 +-
.../state/impl/InMemoryCheckpointStore.scala | 5 +-
.../streaming/state/impl/NonWindowState.scala | 12 +-
.../state/impl/PersistentStateConfig.scala | 2 +-
.../gearpump/streaming/state/impl/Window.scala | 4 +-
.../streaming/state/impl/WindowConfig.scala | 2 +-
.../streaming/state/impl/WindowState.scala | 56 +--
.../streaming/storage/AppDataStore.scala | 6 +-
.../storage/InMemoryAppStoreOnMaster.scala | 13 +-
.../streaming/task/ExpressTransport.scala | 22 +-
.../streaming/task/SerializedMessage.scala | 2 +-
.../streaming/task/SerializerResolver.scala | 16 +-
.../io/gearpump/streaming/task/StartTime.scala | 21 +-
.../task/StreamingTransportSerializer.scala | 15 +-
.../io/gearpump/streaming/task/Subscriber.scala | 11 +-
.../gearpump/streaming/task/Subscription.scala | 67 ++--
.../scala/io/gearpump/streaming/task/Task.scala | 78 +++--
.../io/gearpump/streaming/task/TaskActor.scala | 132 ++++----
.../streaming/task/TaskContextData.scala | 7 +-
.../streaming/task/TaskControlMessage.scala | 11 +-
.../io/gearpump/streaming/task/TaskId.scala | 8 +-
.../streaming/task/TaskMessageSerializer.scala | 4 +-
.../io/gearpump/streaming/task/TaskUtil.scala | 5 +-
.../gearpump/streaming/task/TaskWrapper.scala | 28 +-
.../transaction/api/CheckpointStore.scala | 4 +-
.../transaction/api/MessageDecoder.scala | 5 +-
.../transaction/api/OffsetManager.scala | 15 +-
.../transaction/api/OffsetStorage.scala | 16 +-
.../transaction/api/TimeReplayableSource.scala | 9 +-
.../transaction/api/TimeStampFilter.scala | 5 +-
.../gearpump/streaming/util/ActorPathUtil.scala | 14 +-
.../scala/io/gearpump/streaming/DAGSpec.scala | 9 +-
.../streaming/MessageSerializerSpec.scala | 17 +-
.../scala/io/gearpump/streaming/MockUtil.scala | 16 +-
.../gearpump/streaming/StreamingTestUtil.scala | 8 +-
.../streaming/appmaster/AppMasterSpec.scala | 183 +++++-----
.../streaming/appmaster/ClockServiceSpec.scala | 77 +++--
.../streaming/appmaster/DagManagerSpec.scala | 10 +-
.../appmaster/ExecutorManagerSpec.scala | 52 +--
.../appmaster/ExecutorRestartPolicySpec.scala | 10 +-
.../appmaster/HistoryMetricsServiceSpec.scala | 55 +--
.../streaming/appmaster/JarSchedulerSpec.scala | 61 ++--
.../streaming/appmaster/TaskLocatorSpec.scala | 9 +-
.../streaming/appmaster/TaskManagerSpec.scala | 103 +++---
.../streaming/appmaster/TaskRegistrySpec.scala | 4 +-
.../streaming/appmaster/TaskSchedulerSpec.scala | 69 ++--
.../gearpump/streaming/dsl/StreamAppSpec.scala | 30 +-
.../io/gearpump/streaming/dsl/StreamSpec.scala | 38 ++-
.../partitioner/GroupByPartitionerSpec.scala | 7 +-
.../streaming/dsl/plan/OpTranslatorSpec.scala | 54 +--
.../streaming/executor/ExecutorSpec.scala | 65 ++--
.../executor/TaskArgumentStoreSpec.scala | 15 +-
.../streaming/executor/TaskLauncherSpec.scala | 28 +-
.../metrics/ProcessorAggregatorSpec.scala | 113 ++++---
.../metrics/TaskFilterAggregatorSpec.scala | 31 +-
.../source/DefaultTimeStampFilterSpec.scala | 7 +-
.../state/impl/CheckpointManagerSpec.scala | 14 +-
.../impl/InMemoryCheckpointStoreSpec.scala | 2 +-
.../state/impl/NonWindowStateSpec.scala | 11 +-
.../streaming/state/impl/WindowSpec.scala | 11 +-
.../streaming/state/impl/WindowStateSpec.scala | 42 +--
.../storage/InMemoryAppStoreOnMasterSpec.scala | 37 +-
.../streaming/task/SubscriberSpec.scala | 17 +-
.../streaming/task/SubscriptionSpec.scala | 58 ++--
.../gearpump/streaming/task/TaskActorSpec.scala | 74 ++--
unmanagedlibs/README.md | 6 +-
version.sbt | 18 +
yarnconf/README.md | 8 +-
674 files changed, 12042 insertions(+), 9473 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/c176e448/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 5b6b80f..2e5317e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -3,8 +3,8 @@ language:
- scala
sudo: false
before_script:
-- mkdir -p $HOME/.sbt/launchers/0.13.9/
-- curl -L -o $HOME/.sbt/launchers/0.13.9/sbt-launch.jar http://dl.bintray.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/0.13.9/sbt-launch.jar
+- mkdir -p $HOME/.sbt/launchers/0.13.11/
+- curl -L -o $HOME/.sbt/launchers/0.13.11/sbt-launch.jar http://dl.bintray.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/0.13.11/sbt-launch.jar
script:
- echo "TRAVIS_PULL_REQUEST" $TRAVIS_PULL_REQUEST
- echo "TRAVIS_BRANCH" $TRAVIS_BRANCH
@@ -12,14 +12,18 @@ script:
- echo "repo" $TRAVIS_REPO_SLUG
- set -o pipefail; skipLogs="Resolving |Compiling |Done updating|Updating |scoverage|coverage-report";
if [[ $TRAVIS_PULL_REQUEST != "false" || $TRAVIS_REPO_SLUG != "gearpump/gearpump" ]];
- then sbt -jvm-opts project/travis/jvmopts clean coverage +test | grep -v -E "$skipLogs";
+ then sbt -jvm-opts project/travis/jvmopts clean scalastyle test:scalastyle it:scalastyle unidoc coverage +test | grep -v -E "$skipLogs";
elif [[ $TRAVIS_BRANCH == "master" ]];
- then sbt -jvm-opts project/travis/jvmopts clean +publish | grep -v -E "$skipLogs"; sbt -jvm-opts project/travis/jvmopts coverage +test | grep -v -E "$skipLogs";
+ then sbt -jvm-opts project/travis/jvmopts clean +publish | grep -v -E "$skipLogs"; sbt -jvm-opts project/travis/jvmopts scalastyle test:scalastyle it:scalastyle unidoc coverage +test | grep -v -E "$skipLogs";
elif [[ $TRAVIS_TAG != "" ]];
then sbt -jvm-opts project/travis/jvmopts clean +assembly +packArchiveZip | grep -v -E "$skipLogs";
fi
jdk:
- oraclejdk8
+addons:
+ apt:
+ packages:
+ - oracle-java8-installer
scala:
- 2.11.8
cache:
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/c176e448/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 230ec66..a5681fc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -16,7 +16,7 @@ Mail list:
What are the changes for the process for Apache Incubation?
------------------
-The code importing to Apache Git is still in process. During this transtion time,
+The code importing to Apache Git is still in process. During this transition time,
We will follow these procedures:
1. For new issue, we should no longer open issue in Github issues. Instead, we
@@ -43,7 +43,6 @@ After the transition period, the proposed process is like this:
Before completing importing source code to Apache Git, we will still use
-
Why we make a tag release now, but not wait until Apache importing complete?
------------------
There are quite a lot of open task at https://issues.apache.org/jira/browse/GEARPUMP-1, which
@@ -92,7 +91,7 @@ From now on, new issues should be submitted to https://issues.apache.org/jira/br
- #1892: added instruction text to operate network graph (2) minor tweaks of context menu
- #1963, rename CommitGuideline.md to CONTRIBUTING.md
- #1958: better test report organizing
-- #1906: some visual glichtes found in mobile view
+- #1906: some visual glitches found in mobile view
Gearpump 0.7.6
===================
@@ -232,13 +231,13 @@ Change logs:
- #1701: wrong dirty check of modify processor and more
- #1384: rest service url was not set correctly
- #1682: allow specify a transition time when changing parallelism
-- #1691: dashboard layout and textual changes and update to dashing…
-- #1640 YARN deployment - no easy way of obtaining master address…
+- #1691: dashboard layout and textual changes and update to dashing...
+- #1640 YARN deployment - no easy way of obtaining master address...
- #1688, Reduce unimportant logs in travis UT console
- #1671, make default timeout shorter to be more UI responsive.
- #1685, Should not publish gearpump-integration-test to maven
- #1683, Allow metrics to be aggregated before sending to UI
-- #1223: will mark application with a warning when stalling task de…
+- #1223: will mark application with a warning when stalling task de...
- #1670, #1653, fix dynamic dag bug
- #1659 add a rest api to terminate service process
- #1672: when worker is killed, its detail page will no longer be updated
@@ -251,7 +250,7 @@ Change logs:
- #1639 refine daemon's classpath
- #1652, submit app with no arguments from dashboard returns warning
- #1547 add a SequenceFileSink
-- #1424: when application is terminated, related status indicators …
+- #1424: when application is terminated, related status indicators ...
- #1645: parse appid in submission response better
- #1514, add an experimental module to support akka-stream 1.0
- #1643, fix UI timeout issue when metrics is disabled
@@ -322,9 +321,9 @@ Change log:
- #1194 add graph cycle detection
- #491, add a rest api to acquire Gearpump's built-in partitioners
- #1405: now the number of executors is back
-- #1371: a right way to show historical metrics (2) added committed…
+- #1371: a right way to show historical metrics (2) added committed�?
- #1402, fix NoneGroupingPartitioner
-- #1399: in application overview page the task count of executor wa…
+- #1399: in application overview page the task count of executor wa...
- #1397, allow KafkaSink to receive Message of bytes
- #1395, cross publishSigned
- #1374 remove jars of yarn in executor's classpath
@@ -337,15 +336,13 @@ Gearpump 0.6.1
==========================
Highlight: new version UI dashboard.
-
Change log:
--------------------
- #1369: object path and metrics name are divided by ':' now
- #1369: fixed data initialization issue of echart
- #1364, add default value for BYTES_PER_MESSAGE of SOLStreamProducer
-- #1361: able to show multiple tasks in chart (better not select mo…
-- #1358: properties will be updated (2) moved roboto font and echar…
-
+- #1361: able to show multiple tasks in chart (better not select mo�?
+- #1358: properties will be updated (2) moved roboto font and echart�?
Gearpump 0.6
==========================
@@ -357,7 +354,6 @@ Change log:
- #1352, return appmaster config
- #1344 fix Master HA bug
-
Gearpump 0.5.0
==========================
Highlights:
@@ -466,7 +462,7 @@ Change logs:
- #1149: Shell tools printed too much detail on console
- #1146: actor hungry when worker use block-io to wait response from FileServer.
- #1088: move hbase code to external/
- - #1140：pass app name to task
+ - #1140: pass app name to task
- #1017: Split daemon dependencies with core dependencies
- #1144: fix out of memory when trying to scale gearpump to 2000 task on a 2 core machine
- #995: Command line parser should be able to fall back to MAIN-CLASS definition in MANIFEST.IN when mainClass is not specified in command line options.
@@ -540,7 +536,7 @@ Change list
- #775, fix netty config
- #778, log improvements
- #781 when launching lots of tasks, the application failed to transfer message across hosts
- - #782, a) add wildcard match to return metrics(support glob char … and *), b) add diagnosis message if the clock stop advancing
+ - #782, a) add wildcard match to return metrics(support glob char . and *), b) add diagnosis message if the clock stop advancing
- #786, Read user config from classpath, the appmaster/executor wil use the dynamic user config
- #773: skew chart will show relative skew
- #790, 1) return detail task data in appmaster REST. 2) bind executor id with executor system id
@@ -567,8 +563,8 @@ Change list
- #843, Can't put custom user config in application.conf
- #849, set default hostname to 127.0.0.1 in UT
- #851, JVM not exited when there is exception due to akka create non-daemon threads
- - #854， fix storm connector performance
- - #856， Service launch failed
+ - #854, fix storm connector performance
+ - #856, Service launch failed
- #853, fix resource leak(thread not closed, process not killed in UT. Also increase the PermGen size to avoid Permgen OOM.
- #859, random UT fail due to akka bug, "akka cluster deadlock when initializing"
- #865, Change the default timeout setting in akka test expectMsg
@@ -595,7 +591,7 @@ gearpump-0.3.4
====================
Change List:
----------------
- - #768, Serious performance degration of ui server on windows
+ - #768, Serious performance degrade of ui server on windows
- #765, improve the graph type inference so we don't need to set template argument type explicitly
gearpump-0.3.3
@@ -718,7 +714,7 @@ Change List:
- #479, publish test jar as artifacts
- #419, Reorder the application log by master startup timestamp
- #456, Use visdag to render the graph DAG
- - #464， Travis bower integration
+ - #464, Travis bower integration
- #394, fix kafka producer hang issue
- #468, For test code, the ClockService will throw exception when the DAG is not defined
- #465, fix appname prefix bug
@@ -751,13 +747,12 @@ Change List:
- #378, construct the serializer explicitly, instead of implicitly via the kryo serializer akka extension
- #380, set the context class loader as the URL class loader for ActorSystem.
-
gearpump-0.2.3
====================
Change List:
---------------
- #333, KafkaUtilSpec causes out of memory on travis
- - #335, #359， Enable auto-deployment to sonatype
+ - #335, #359, Enable auto-deployment to sonatype
- #299, Some UT may fail randomly, most because of the expectMsg time out
- #338, fix kafka leader not available exception
- #349, scoverage dependencies get into snapshot build binaries.
@@ -797,7 +792,7 @@ Highlights:
- Add support for general replay-able data source to support at least once delivery.
- More robust streaming. Be resilient to message loss, message duplication, and zombie processes.
- Refactor Kafka data source for at least once delivery.
- - Support general applications besides streaming, add an experimental distrubtedshell application under experiments/.
+ - Support general applications besides streaming, add an experimental distributedshell application under experiments/.
- Re-defined the AppMaster, and Task interface, It is much easier to write an application now.
- Support submitting and distributing large applications jars.
- Add CI tool, nightly build, code coverage, and defined a formal commit guideline.
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/c176e448/CONTRIBUTING.md
----------------------------------------------------------------------
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index a523344..a39da53 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -35,7 +35,7 @@ No work should ever be done in the forked master. Another way to do this is to
```
git branch branch_issueId
- git checkout branch_issusId
+ git checkout branch_issueId
```
Work on the branch, make changes, then push to your forked gearpump repo http://github.com/<git-user_id>/gearpump
@@ -78,7 +78,6 @@ No work should ever be done in the forked master. Another way to do this is to
6. Open a PR, which is a one-click thing in github.com; it knows you likely are opening a PR against upstream master. [Guide](https://help.github.com/articles/creating-a-pull-request) is here.
7. [Merge PR](https://help.github.com/articles/merging-a-pull-request), [delete branch](https://help.github.com/articles/deleting-unused-branches).
-
You can skip using branches in your fork if desired, and just work out of your master. Less overhead, but, you can't pursue different unrelated changes then.
The key here is using rebase, not merge. The downstream forks never need to use merge unless the work is a long-lived collaboration with someone else who is pulling from your fork.
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/c176e448/LICENSE
----------------------------------------------------------------------
diff --git a/LICENSE b/LICENSE
index 106b49b..97a5197 100644
--- a/LICENSE
+++ b/LICENSE
@@ -200,7 +200,6 @@
See the License for the specific language governing permissions and
limitations under the License.
-
-----------------------------------------------------------------------
For bootstrap 3.3.5/3.3.6
@@ -300,13 +299,11 @@ WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations under
the License.
-
-----------------------------------------------------------------------
For angular 1.5.0 (angular-1.5.0-rc.0.jar)
https://github.com/angular/angular.js/blob/master/LICENSE
-
The MIT License
Copyright (c) 2010-2016 Google, Inc. http://angularjs.org
@@ -329,7 +326,6 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
-
-----------------------------------------------------------------------
For angular-loading-bar 0.8.0 (angular-loading-bar-0.8.0.jar)
@@ -371,7 +367,6 @@ The above copyright notice and this permission notice shall be included in all c
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-----------------------------------------------------------------------
For angular-smart-table-2.1.6 (angular-smart-table-2.1.6.jar)
@@ -387,7 +382,6 @@ The above copyright notice and this permission notice shall be included in all c
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-----------------------------------------------------------------------
For angular-strap-2.3.5 (angular-strap-2.3.5.jar)
@@ -403,7 +397,6 @@ The above copyright notice and this permission notice shall be included in all c
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-----------------------------------------------------------------------
For angular-ui-router-0.2.15 (angular-ui-router-0.2.15.jar)
@@ -431,7 +424,6 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
-
-----------------------------------------------------------------------
For angular-ui-select-0.13.2.jar (angular-ui-select-0.13.2.jar)
@@ -458,7 +450,6 @@ COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-----------------------------------------------------------------------
For angularjs-1.4.8(angularjs-1.4.8.jar)
@@ -486,7 +477,6 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
-
-----------------------------------------------------------------------
For aopalliance-1.0 (aopalliance-1.0.jar)
@@ -501,7 +491,6 @@ LICENCE: all the source code provided by AOP Alliance is Public Domain.
For asm-3.1 (asm-3.1.jar)
http://asm.ow2.org/license.html
-
Copyright (c) 2000-2011 INRIA, France Telecom
All rights reserved.
@@ -537,7 +526,6 @@ THE POSSIBILITY OF SUCH DAMAGE.
For bootstrap-additions-0.3.1 (bootstrap-additions-0.3.1.jar)
https://github.com/twbs/bootstrap/blob/master/LICENSE
-
The MIT License
Copyright (c) 2014 Olivier Louvignes http://olouv.com
@@ -548,7 +536,6 @@ The above copyright notice and this permission notice shall be included in all c
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-----------------------------------------------------------------------
For clipboard.js-0.1.1 (clipboard.js-0.1.1.jar)
@@ -587,7 +574,6 @@ The above copyright notice and this permission notice shall be included in all c
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-----------------------------------------------------------------------
For font-awesome-4.5.0 (font-awesome-4.5.0.jar)
@@ -614,7 +600,6 @@ All code is available to you under the MIT license, available at http://opensour
Copyright Erik Osheim, 2012-2015.
-
-----------------------------------------------------------------------
For jaxb 2.2.2, jaxb 2.2.3 (jaxb-api-2.2.2.jar, jaxb-impl-2.2.3-1.jar)
@@ -633,7 +618,6 @@ COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL - Version 1.1)
GNU General Public License (GPL - Version 2, June 1991) with the [Classpath Exception]
-
-----------------------------------------------------------------------
For jgrapht-core-0.9.0 (jgrapht-core-0.9.0.jar)
@@ -907,7 +891,6 @@ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
OF THE POSSIBILITY OF SUCH DAMAGE.
-
-----------------------------------------------------------------------
For jquery-cookie-1.4.1 (jquery-cookie-1.4.1.jar)
@@ -971,7 +954,6 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-----------------------------------------------------------------------
For jsr305-3.0.0 (jsr305-3.0.0.jar)
@@ -1006,13 +988,11 @@ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
-
-----------------------------------------------------------------------
For kryo-2.21.jar
https://code.google.com/p/kryo/
-
Code license
New BSD License
@@ -1021,7 +1001,6 @@ New BSD License
For leveldbjni-all-1.8.jar
https://github.com/fusesource/leveldbjni
-
Copyright (c) 2011 FuseSource Corp. All rights reserved.
Redistribution and use in source and binary forms, with or without
@@ -1111,7 +1090,6 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
-
-----------------------------------------------------------------------
For ng-file-upload-5.0.9.jar
@@ -1138,7 +1116,6 @@ COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-----------------------------------------------------------------------
For leveldbjniparanamer-2.3.jar
@@ -1280,7 +1257,6 @@ License
API is licensed under standard BSD license, which is compatible with all Free and Open Software (OSS) licenses.
-
-----------------------------------------------------------------------
For upickle_2.11-0.3.4.jar
@@ -1297,13 +1273,11 @@ The above copyright notice and this permission notice shall be included in all c
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-----------------------------------------------------------------------
For xmlenc-0.52.jar
https://github.com/digidentity/xmlenc/blob/master/LICENSE.txt
-
Copyright (c) 2013 Benoist
MIT License
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/c176e448/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 32b3837..17f0a0f 100644
--- a/README.md
+++ b/README.md
@@ -78,6 +78,14 @@ export HTTPS_PROXY= http://host:port
## How to run Gearpump integration test
Gearpump has an integration test system which is based on Docker. Please check [the instructions](integrationtest/README.md).
+## How to do style check before submitting a pull request?
+
+Before submitting a PR, you should always run style check first:
+```
+ ## Run style check for compile, test, and integration test.
+ sbt scalastyle test:scalastyle it:scalastyle
+```
+
## Contributors (time order)
* [Sean Zhong](https://github.com/clockfly)
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/c176e448/ReleaseProcess.md
----------------------------------------------------------------------
diff --git a/ReleaseProcess.md b/ReleaseProcess.md
index 7bf9e3c..bc035c3 100644
--- a/ReleaseProcess.md
+++ b/ReleaseProcess.md
@@ -38,5 +38,4 @@ Step3: Post-Release
where NEXT_SNPASHOT_VERSION must end with "-SNAPSHOT". For example, 0.2.3-SNPASHOT is a good snapshot version, 0.2.3 is NOT
2. Make PR by following https://github.com/gearpump/gearpump/blob/master/doc/CommitGuideline.md
-
Done!!!
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/c176e448/conf/gear.conf
----------------------------------------------------------------------
diff --git a/conf/gear.conf b/conf/gear.conf
index c2352fe..b21f59a 100644
--- a/conf/gear.conf
+++ b/conf/gear.conf
@@ -166,8 +166,6 @@ gearpump {
# "yourpackage.YourClass" = ""
}
-
-
### client's timeout (in second) to connect to master and wait for the response
masterclient.timeout = 90
@@ -316,7 +314,6 @@ gearpump {
}
}
-
### Configuration only visible to master nodes..
gearpump-master {
extensions = [
@@ -369,13 +366,11 @@ gearpump-worker {
}
}
-
#########################################
### For log level of Akka class, you need to change both log4j.properties and this entry
#########################################
#akka.loglevel = "INFO"
-
### configurations only visible to UI server.
gearpump-ui {
## Security related settings
@@ -514,7 +509,6 @@ gearpump-ui {
## value set here serves as an example.
uaahost = "http://<cloud foundry login endpoint>"
-
## Whether to enable additional authorization check.
## If the user fails the check, then Gearpump would log user out.
additional-authenticator-enabled = true
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/c176e448/conf/log4j.properties
----------------------------------------------------------------------
diff --git a/conf/log4j.properties b/conf/log4j.properties
index a6013ca..7175778 100644
--- a/conf/log4j.properties
+++ b/conf/log4j.properties
@@ -7,7 +7,7 @@
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
@@ -61,7 +61,6 @@ log4j.rootLogger=${log4j.rootLevel},${log4j.rootAppender}
# Logging Threshold
log4j.threshhold=ALL
-
# =====================================================================
# Customize logger for a specific class
# For example, if you want to enable debug log for specific akka class
@@ -88,7 +87,6 @@ log4j.appender.RollingFileAppender.layout.ConversionPattern=%d{ISO8601} %p %c{1}
log4j.appender.RollingFileAppender.MaxFileSize=200MB
log4j.appender.RollingFileAppender.MaxBackupIndex=10
-
#
# console appender
# Add "console" to rootlogger above if you want to use this
@@ -110,4 +108,3 @@ log4j.appender.ApplicationLogAppender.MaxBackupIndex=30
log4j.appender.ApplicationLogAppender.layout=org.apache.log4j.PatternLayout
log4j.appender.ApplicationLogAppender.layout.ConversionPattern=[%p] [%d{MM/dd/yyyy HH:mm:ss.SSS}] [%c{1}] %m%n
-
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/c176e448/core/src/main/java/io/gearpump/transport/netty/ITransportMessageSerializer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/io/gearpump/transport/netty/ITransportMessageSerializer.java b/core/src/main/java/io/gearpump/transport/netty/ITransportMessageSerializer.java
index 9b0734a..c01135e 100644
--- a/core/src/main/java/io/gearpump/transport/netty/ITransportMessageSerializer.java
+++ b/core/src/main/java/io/gearpump/transport/netty/ITransportMessageSerializer.java
@@ -15,12 +15,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package io.gearpump.transport.netty;
import java.io.DataInput;
import java.io.DataOutput;
public interface ITransportMessageSerializer {
+
int getLength(Object obj);
void serialize(DataOutput dataOutput, Object transportMessage);
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/c176e448/core/src/main/java/io/gearpump/transport/netty/MessageBatch.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/io/gearpump/transport/netty/MessageBatch.java b/core/src/main/java/io/gearpump/transport/netty/MessageBatch.java
index 665c99f..f556161 100644
--- a/core/src/main/java/io/gearpump/transport/netty/MessageBatch.java
+++ b/core/src/main/java/io/gearpump/transport/netty/MessageBatch.java
@@ -7,7 +7,7 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -29,6 +29,9 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+/**
+ * Netty message on the wire is wrapped as MessageBatch
+ */
public class MessageBatch {
private static final Logger log = LoggerFactory.getLogger(MessageBatch.class);
@@ -83,7 +86,6 @@ public class MessageBatch {
}
/**
- *
* @return true, if allowed buffer is Full
*/
boolean isFull() {
@@ -91,7 +93,6 @@ public class MessageBatch {
}
/**
- *
* @return true, if no messages in this batch
*/
boolean isEmpty() {
@@ -99,7 +100,6 @@ public class MessageBatch {
}
/**
- *
* @return number of messages available in this batch
*/
int size() {
@@ -111,7 +111,7 @@ public class MessageBatch {
*/
ChannelBuffer buffer() throws IOException {
ChannelBufferOutputStream bout =
- new ChannelBufferOutputStream(ChannelBuffers.directBuffer(encoded_length));
+ new ChannelBufferOutputStream(ChannelBuffers.directBuffer(encoded_length));
try {
for (TaskMessage msg : messages) {
@@ -128,7 +128,7 @@ public class MessageBatch {
/**
* write a TaskMessage into a stream
- * <p/>
+ * <p>
* Each TaskMessage is encoded as:
* sessionId ... int(4)
* source task ... Long(8)
@@ -137,7 +137,7 @@ public class MessageBatch {
* payload ... byte[] *
*/
private void writeTaskMessage(ChannelBufferOutputStream bout,
- TaskMessage message) throws IOException {
+ TaskMessage message) throws IOException {
long target_id = message.targetTask();
long source_id = message.sourceTask();
int sessionId = message.sessionId();
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/c176e448/core/src/main/java/io/gearpump/transport/netty/MessageDecoder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/io/gearpump/transport/netty/MessageDecoder.java b/core/src/main/java/io/gearpump/transport/netty/MessageDecoder.java
index d9a74b1..7566014 100644
--- a/core/src/main/java/io/gearpump/transport/netty/MessageDecoder.java
+++ b/core/src/main/java/io/gearpump/transport/netty/MessageDecoder.java
@@ -7,7 +7,7 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -23,7 +23,6 @@ import org.jboss.netty.channel.Channel;
import org.jboss.netty.channel.ChannelHandlerContext;
import org.jboss.netty.handler.codec.frame.FrameDecoder;
-import java.io.DataInput;
import java.util.ArrayList;
import java.util.List;
@@ -31,7 +30,7 @@ public class MessageDecoder extends FrameDecoder {
private ITransportMessageSerializer serializer;
private WrappedChannelBuffer dataInput = new WrappedChannelBuffer();
- public MessageDecoder(ITransportMessageSerializer serializer){
+ public MessageDecoder(ITransportMessageSerializer serializer) {
this.serializer = serializer;
}
@@ -44,7 +43,7 @@ public class MessageDecoder extends FrameDecoder {
* payload ... byte[] *
*/
protected List<TaskMessage> decode(ChannelHandlerContext ctx, Channel channel,
- ChannelBuffer buf) {
+ ChannelBuffer buf) {
this.dataInput.setChannelBuffer(buf);
final int SESION_LENGTH = 4; //int
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/c176e448/core/src/main/java/io/gearpump/transport/netty/MessageEncoder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/io/gearpump/transport/netty/MessageEncoder.java b/core/src/main/java/io/gearpump/transport/netty/MessageEncoder.java
index 5bd512d..674e96f 100644
--- a/core/src/main/java/io/gearpump/transport/netty/MessageEncoder.java
+++ b/core/src/main/java/io/gearpump/transport/netty/MessageEncoder.java
@@ -7,7 +7,7 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -26,7 +26,7 @@ public class MessageEncoder extends OneToOneEncoder {
@Override
protected Object encode(ChannelHandlerContext ctx, Channel channel, Object obj) throws Exception {
if (obj instanceof MessageBatch) {
- return ((MessageBatch)obj).buffer();
+ return ((MessageBatch) obj).buffer();
}
throw new RuntimeException("Unsupported encoding of object of class " + obj.getClass().getName());
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/c176e448/core/src/main/java/io/gearpump/transport/netty/NettyRenameThreadFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/io/gearpump/transport/netty/NettyRenameThreadFactory.java b/core/src/main/java/io/gearpump/transport/netty/NettyRenameThreadFactory.java
index f6f234c..026684e 100644
--- a/core/src/main/java/io/gearpump/transport/netty/NettyRenameThreadFactory.java
+++ b/core/src/main/java/io/gearpump/transport/netty/NettyRenameThreadFactory.java
@@ -7,7 +7,7 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -38,7 +38,7 @@ public class NettyRenameThreadFactory implements ThreadFactory {
NettyRenameThreadFactory(String name) {
SecurityManager s = System.getSecurityManager();
group = (s != null) ? s.getThreadGroup() :
- Thread.currentThread().getThreadGroup();
+ Thread.currentThread().getThreadGroup();
this.name = name;
}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/c176e448/core/src/main/java/io/gearpump/transport/netty/TaskMessage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/io/gearpump/transport/netty/TaskMessage.java b/core/src/main/java/io/gearpump/transport/netty/TaskMessage.java
index 5a5f43c..243d3f0 100644
--- a/core/src/main/java/io/gearpump/transport/netty/TaskMessage.java
+++ b/core/src/main/java/io/gearpump/transport/netty/TaskMessage.java
@@ -7,7 +7,7 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -38,13 +38,15 @@ public class TaskMessage {
_message = message;
}
- public int sessionId() {return _sessionId; }
+ public int sessionId() {
+ return _sessionId;
+ }
public long targetTask() {
return _targetTask;
}
- public long sourceTask(){
+ public long sourceTask() {
return _sourceTask;
}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/c176e448/core/src/main/java/io/gearpump/transport/netty/WrappedChannelBuffer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/io/gearpump/transport/netty/WrappedChannelBuffer.java b/core/src/main/java/io/gearpump/transport/netty/WrappedChannelBuffer.java
index a89d103..3531f3b 100644
--- a/core/src/main/java/io/gearpump/transport/netty/WrappedChannelBuffer.java
+++ b/core/src/main/java/io/gearpump/transport/netty/WrappedChannelBuffer.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package io.gearpump.transport.netty;
import org.jboss.netty.buffer.ChannelBuffer;
@@ -22,16 +23,18 @@ import org.jboss.netty.buffer.ChannelBuffer;
import java.io.DataInput;
import java.io.IOException;
-public class WrappedChannelBuffer implements DataInput{
+/** Wrap ChannelBuffer as a DataInput */
+public class WrappedChannelBuffer implements DataInput {
private ChannelBuffer channelBuffer;
- public WrappedChannelBuffer(){}
+ public WrappedChannelBuffer() {
+ }
- public WrappedChannelBuffer(ChannelBuffer channelBuffer){
+ public WrappedChannelBuffer(ChannelBuffer channelBuffer) {
this.channelBuffer = channelBuffer;
}
- public void setChannelBuffer(ChannelBuffer channelBuffer){
+ public void setChannelBuffer(ChannelBuffer channelBuffer) {
this.channelBuffer = channelBuffer;
}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/c176e448/core/src/main/java/io/gearpump/util/AkkaHelper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/io/gearpump/util/AkkaHelper.java b/core/src/main/java/io/gearpump/util/AkkaHelper.java
index f4772c1..286fabf 100644
--- a/core/src/main/java/io/gearpump/util/AkkaHelper.java
+++ b/core/src/main/java/io/gearpump/util/AkkaHelper.java
@@ -7,7 +7,7 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -28,7 +28,12 @@ public class AkkaHelper {
*
* This is used for performance optimization, we encode the session Id
* in the ActorRef path. Session Id is used to identity sender Task.
+ *
+ * @param system ActorSystem
+ * @param path Relative or absolute path of this Actor System.
+ * @return Full qualified ActorRef.
*/
+ @SuppressWarnings("deprecation")
public static ActorRef actorFor(ActorSystem system, String path) {
return system.actorFor(path);
}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/c176e448/core/src/main/java/io/gearpump/util/HadoopFSLogAppender.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/io/gearpump/util/HadoopFSLogAppender.java b/core/src/main/java/io/gearpump/util/HadoopFSLogAppender.java
index ab7ee5e..93b65e8 100644
--- a/core/src/main/java/io/gearpump/util/HadoopFSLogAppender.java
+++ b/core/src/main/java/io/gearpump/util/HadoopFSLogAppender.java
@@ -7,7 +7,7 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/c176e448/core/src/main/java/io/gearpump/util/RecreateRollingFileAppender.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/io/gearpump/util/RecreateRollingFileAppender.java b/core/src/main/java/io/gearpump/util/RecreateRollingFileAppender.java
index 7447eb4..acbd1cb 100644
--- a/core/src/main/java/io/gearpump/util/RecreateRollingFileAppender.java
+++ b/core/src/main/java/io/gearpump/util/RecreateRollingFileAppender.java
@@ -7,7 +7,7 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -23,41 +23,42 @@ import org.apache.log4j.spi.LoggingEvent;
import java.io.File;
+/** When log file is deleted, tried to recreate it in file system */
public class RecreateRollingFileAppender extends RollingFileAppender {
- protected long checkFileInterval = 60L;
- private long lastCheckTime = 0L;
+ protected long checkFileInterval = 60L;
+ private long lastCheckTime = 0L;
- @Override
- public void append(LoggingEvent event) {
- checkInterval();
- super.append(event);
- }
+ @Override
+ public void append(LoggingEvent event) {
+ checkInterval();
+ super.append(event);
+ }
- private void checkInterval() {
- long currentTime = System.currentTimeMillis();
- if ((currentTime - lastCheckTime) > (checkFileInterval * 1000)) {
- checkLogFileExist();
- lastCheckTime = currentTime;
- }
+ private void checkInterval() {
+ long currentTime = System.currentTimeMillis();
+ if ((currentTime - lastCheckTime) > (checkFileInterval * 1000)) {
+ checkLogFileExist();
+ lastCheckTime = currentTime;
}
+ }
- private void checkLogFileExist() {
- String fileName = super.fileName;
- if (fileName != null) {
- File logFile = new File(fileName);
- if (!logFile.exists()) {
- this.setFile(fileName);
- this.activateOptions();
- }
- }
+ private void checkLogFileExist() {
+ String fileName = super.fileName;
+ if (fileName != null) {
+ File logFile = new File(fileName);
+ if (!logFile.exists()) {
+ this.setFile(fileName);
+ this.activateOptions();
+ }
}
+ }
- public long getCheckFileInterval() {
- return this.checkFileInterval;
- }
+ public long getCheckFileInterval() {
+ return this.checkFileInterval;
+ }
- public void setCheckFileInterval(long checkFileInterval) {
- this.checkFileInterval = checkFileInterval;
- }
+ public void setCheckFileInterval(long checkFileInterval) {
+ this.checkFileInterval = checkFileInterval;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/c176e448/core/src/main/resources/geardefault.conf
----------------------------------------------------------------------
diff --git a/core/src/main/resources/geardefault.conf b/core/src/main/resources/geardefault.conf
index d34a656..4eecf3a 100644
--- a/core/src/main/resources/geardefault.conf
+++ b/core/src/main/resources/geardefault.conf
@@ -27,7 +27,6 @@ gearpump {
###
hostname = "127.0.0.1"
-
## The installation folder of gearpump
home = ""
@@ -43,7 +42,6 @@ gearpump {
## Number of executors to launch when starting an application
application.executor-num = 1
-
## Unique Id to identity this worker instance in low level resource manager like YARN.
##
## This value is typically configured by resource manager integration module, like gearpump-yarn in this case.
@@ -86,7 +84,7 @@ gearpump {
### When the resource cannot be allocated in the timeout, then
### the appmaster will shutdown itself.
resource-allocation-timeout-seconds = 120
-
+
##
## Executor share same process of worker
worker.executor-share-same-jvm-as-worker = false
@@ -216,8 +214,6 @@ gearpump {
"akka.actor.LocalActorRef" = ""
}
-
-
### client's timeout (in second) to connect to master and wait for the response
masterclient.timeout = 90
@@ -366,7 +362,6 @@ gearpump {
}
}
-
### Configuration only visible to master nodes..
gearpump-master {
## NOTE: Please add akka-related settings in gear.conf to avoid confusion in
@@ -379,13 +374,11 @@ gearpump-worker {
## config overriding.
}
-
#########################################
### For log level of Akka class, you need to change both log4j.properties and this entry
#########################################
#akka.loglevel = "INFO"
-
### configurations only visible to UI server.
gearpump-ui {
## Security related settings
@@ -542,7 +535,6 @@ gearpump-ui {
}
}
-
###################################################################
###################################################################
## The following configurations supercede the default akka config
@@ -552,7 +544,7 @@ gearpump-ui {
## work-around to fix akka io performance issue on windows
## reference: http://stackoverflow.com/questions/29453848/high-cpu-on-akka-io-pinned-dispatcher-on-windows
## ticket: https://github.com/akka/akka/issues/16295
-akka.io.tcp.windows-connection-abort-workaround-enabled=off
+akka.io.tcp.windows-connection-abort-workaround-enabled = off
### On windows, the value must be larger than 10ms, check
### https://github.com/akka/akka/blob/master/akka-actor/src/main/scala/akka/actor/Scheduler.scala#L204
@@ -576,22 +568,20 @@ akka {
## Akka-http session related settings
session {
- serverSecret = "!!!please change this to a value only you know!!!"
-
- clientSession {
- cookie {
- name = "gearpump_token"
- ## domain = "..."
- path = "/"
- ## maxAge = 0
- secure = false
- httpOnly = true
- }
+ server-secret = "Please change this to a value only you know! At least 64 chars!!!"
- ## Session lifetime. Default value is about 1 week
- maxAgeSeconds = 604800
- encryptData = true
+ cookie {
+ name = "gearpump_token"
+ ## domain = "..."
+ path = "/"
+ ## maxAge = 0
+ secure = false
+ http-only = true
}
+
+ ## Session lifetime. Default value is about 1 week
+ max-age = 604800
+ encrypt-data = true
}
}
@@ -628,7 +618,6 @@ akka {
mailbox-type = "akka.dispatch.SingleConsumerOnlyUnboundedMailbox"
}
default-dispatcher {
- mailbox-type = "akka.dispatch.SingleConsumerOnlyUnboundedMailbox"
throughput = 10
fork-join-executor {
parallelism-factor = 2
@@ -668,13 +657,11 @@ akka {
default-remote-dispatcher {
throughput = 5
type = Dispatcher
- mailbox-type = "akka.dispatch.SingleConsumerOnlyUnboundedMailbox"
executor = "fork-join-executor"
}
}
}
-
## Configurations only visible on Linux or Mac.
gearpump-linux {
### On windows, the value must be larger than 10ms, check
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/c176e448/core/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/core/src/main/resources/log4j.properties b/core/src/main/resources/log4j.properties
index 937905a..682a11b 100644
--- a/core/src/main/resources/log4j.properties
+++ b/core/src/main/resources/log4j.properties
@@ -7,7 +7,7 @@
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
@@ -54,7 +54,6 @@ log4j.rootLogger=INFO,console
# Logging Threshold
log4j.threshhold=ALL
-
# =====================================================================
# Appenders
# =====================================================================
@@ -75,7 +74,6 @@ log4j.appender.RollingFileAppender.layout.ConversionPattern=%d{ISO8601} %p %c{1}
log4j.appender.RollingFileAppender.MaxFileSize=200MB
log4j.appender.RollingFileAppender.MaxBackupIndex=10
-
#
# console appender
# Add "console" to rootlogger above if you want to use this
@@ -100,4 +98,3 @@ log4j.appender.ApplicationLogAppender.layout=org.apache.log4j.PatternLayout
#log4j.appender.ApplicationLogAppender.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
log4j.appender.ApplicationLogAppender.layout.ConversionPattern=[%p] [%d{MM/dd/yyyy HH:mm:ss.SSS}] [%c{1}] %m%n
-
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/c176e448/core/src/main/scala/io/gearpump/Message.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/io/gearpump/Message.scala b/core/src/main/scala/io/gearpump/Message.scala
index b442e33..aec8089 100644
--- a/core/src/main/scala/io/gearpump/Message.scala
+++ b/core/src/main/scala/io/gearpump/Message.scala
@@ -7,7 +7,7 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -22,11 +22,11 @@ package io.gearpump
* Each message contains a immutable timestamp.
*
* For example, if you take a picture, the time you take the picture is the
- * message's timestmap.
- * @param msg Accept any type except Null, Nothing and Unit
+ * message's timestamp.
+ * @param msg Accept any type except Null, Nothing and Unit
*/
case class Message(msg: Any, timestamp: TimeStamp = Message.noTimeStamp)
object Message {
- val noTimeStamp : TimeStamp = 0L
+ val noTimeStamp: TimeStamp = 0L
}