You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2014/05/30 21:07:29 UTC
svn commit: r1598702 [1/23] - in /pig/trunk: ./ ivy/
shims/src/hadoop23/org/apache/pig/backend/hadoop23/
shims/test/hadoop20/org/apache/pig/test/
shims/test/hadoop23/org/apache/pig/test/ src/META-INF/services/
src/org/apache/pig/ src/org/apache/pig/bac...
Author: daijy
Date: Fri May 30 19:07:23 2014
New Revision: 1598702
URL: http://svn.apache.org/r1598702
Log:
PIG-3970: Merge Tez branch into trunk
Added:
pig/trunk/shims/test/hadoop20/org/apache/pig/test/TezMiniCluster.java
pig/trunk/shims/test/hadoop23/org/apache/pig/test/TezMiniCluster.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SecondaryKeyOptimizerMR.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/optimizer/
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/optimizer/SecondaryKeyOptimizer.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java.orig
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPoissonSample.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POReservoirSample.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/AccumulatorOptimizer.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/CombinerOptimizer.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/MultiQueryOptimizerTez.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/ObjectCache.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/POFRJoinTez.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/POIdentityInOutTez.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/POLocalRearrangeTez.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/POLocalRearrangeTezFactory.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/POPartitionRearrangeTez.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/POShuffleTezLoad.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/POSimpleTezLoad.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/POStoreTez.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/POValueInputTez.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/POValueOutputTez.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/PigProcessor.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/ReadScalarsTez.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/RoundRobinPartitioner.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/SecondaryKeyOptimizerTez.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/SkewedPartitionerTez.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezCompiler.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezCompilerException.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezEdgeDescriptor.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezExecType.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezExecutionEngine.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezInput.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJob.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobControl.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobControlCompiler.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobNotifier.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLocalExecType.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezOpPlanVisitor.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezOperPlan.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezOperator.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezOutput.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezPOPackageAnnotator.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezPOStreamVisitor.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezPlanContainer.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezPlanContainerNode.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezPlanContainerPrinter.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezPlanContainerUDFCollector.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezPlanContainerVisitor.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezPrinter.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezResourceManager.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezSessionManager.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezTaskConfigurable.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezTaskContext.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/WeightedRangePartitionerTez.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/operators/
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/operators/POCounterStatsTez.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/operators/POCounterTez.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/operators/PORankTez.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/operators/POShuffledValueInputTez.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/optimizers/
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/optimizers/NoopFilterRemover.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/optimizers/UnionOptimizer.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/util/
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/util/MRToTezHelper.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/util/SecurityHelper.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/util/TezCompilerUtil.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/util/AccumulatorOptimizerUtil.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/util/CombinerOptimizerUtil.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/util/SecondaryKeyOptimizerUtil.java
pig/trunk/src/org/apache/pig/tools/pigstats/tez/
pig/trunk/src/org/apache/pig/tools/pigstats/tez/TezScriptState.java
pig/trunk/src/org/apache/pig/tools/pigstats/tez/TezStats.java
pig/trunk/src/org/apache/pig/tools/pigstats/tez/TezTaskStats.java
pig/trunk/test/e2e/pig/conf/tez.conf
pig/trunk/test/org/apache/pig/test/TestCustomPartitioner.java
pig/trunk/test/org/apache/pig/test/TestSecondarySortMR.java
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC1.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC10.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC11.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC12.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC13.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC14.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC15.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC16.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC17.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC18.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC2.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC3.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC4.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC5.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-1-OPTOFF.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-1.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3-OPTOFF.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-4-OPTOFF.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-4.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-5-OPTOFF.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-5.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Rank-1.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Rank-2.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-1-OPTOFF.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-1.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-10-OPTOFF.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-10.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-2-OPTOFF.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-2.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-3-OPTOFF.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-3.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-4-OPTOFF.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-4.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-5-OPTOFF.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-5.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6-OPTOFF.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7-OPTOFF.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-8-OPTOFF.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-8.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9-OPTOFF.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9.gld
pig/trunk/test/org/apache/pig/tez/
pig/trunk/test/org/apache/pig/tez/TestSecondarySortTez.java
pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java
pig/trunk/test/org/apache/pig/tez/TestTezJobControlCompiler.java
pig/trunk/test/org/apache/pig/tez/TestTezLauncher.java
pig/trunk/test/tez-tests
Modified:
pig/trunk/CHANGES.txt
pig/trunk/build.xml
pig/trunk/ivy.xml
pig/trunk/ivy/libraries.properties
pig/trunk/shims/src/hadoop23/org/apache/pig/backend/hadoop23/PigJobControl.java
pig/trunk/shims/test/hadoop20/org/apache/pig/test/MiniCluster.java
pig/trunk/shims/test/hadoop23/org/apache/pig/test/MiniCluster.java
pig/trunk/src/META-INF/services/org.apache.pig.ExecType
pig/trunk/src/org/apache/pig/Main.java
pig/trunk/src/org/apache/pig/PigConfiguration.java
pig/trunk/src/org/apache/pig/PigServer.java
pig/trunk/src/org/apache/pig/PigWarning.java
pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java
pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/HPath.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/AccumulatorOptimizer.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/ColumnInfo.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/CombinerOptimizer.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRExecType.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRUtil.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MultiQueryOptimizer.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PhyPlanSetter.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBigDecimalRawComparator.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBigIntegerRawComparator.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBooleanRawComparator.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBytesRawComparator.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigCombiner.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigDateTimeRawComparator.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigDoubleRawComparator.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigFloatRawComparator.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigIntRawComparator.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigLongRawComparator.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapReduceCounter.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigSecondaryKeyComparator.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigTextRawComparator.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigTupleDefaultRawComparator.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigTupleSortComparator.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/partitioners/SkewedPartitioner.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/partitioners/WeightedRangePartitioner.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/plans/EndOfAllInputSetter.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/plans/POPackageAnnotator.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PhysicalOperator.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PigLogger.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/PhyPlanVisitor.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/PhysicalPlan.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/PlanPrinter.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/XMLPhysicalPlanPrinter.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/CombinerPackager.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/JoinPackager.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/LitePackager.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POCounter.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POFRJoin.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POLimit.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POLocalRearrange.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POMergeJoin.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPackage.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPartitionRearrange.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/PORank.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POSkewedJoin.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POStore.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POStream.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/Packager.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/util/PlanHelper.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/util/MapRedUtil.java
pig/trunk/src/org/apache/pig/backend/hadoop/streaming/HadoopExecutableManager.java
pig/trunk/src/org/apache/pig/data/BinInterSedes.java
pig/trunk/src/org/apache/pig/data/DefaultAbstractBag.java
pig/trunk/src/org/apache/pig/data/DefaultTuple.java
pig/trunk/src/org/apache/pig/data/ReadOnceBag.java
pig/trunk/src/org/apache/pig/impl/PigContext.java
pig/trunk/src/org/apache/pig/impl/builtin/PartitionSkewedKeys.java
pig/trunk/src/org/apache/pig/impl/builtin/PoissonSampleLoader.java
pig/trunk/src/org/apache/pig/impl/io/NullablePartitionWritable.java
pig/trunk/src/org/apache/pig/impl/io/NullableTuple.java
pig/trunk/src/org/apache/pig/impl/io/PigNullableWritable.java
pig/trunk/src/org/apache/pig/impl/plan/NodeIdGenerator.java
pig/trunk/src/org/apache/pig/impl/plan/OperatorPlan.java
pig/trunk/src/org/apache/pig/impl/util/JarManager.java
pig/trunk/src/org/apache/pig/impl/util/UDFContext.java
pig/trunk/src/org/apache/pig/impl/util/Utils.java
pig/trunk/src/org/apache/pig/newplan/BaseOperatorPlan.java
pig/trunk/src/org/apache/pig/newplan/logical/relational/LogToPhyTranslationVisitor.java
pig/trunk/src/org/apache/pig/newplan/logical/visitor/ScalarVisitor.java
pig/trunk/src/org/apache/pig/pen/IllustratorAttacher.java
pig/trunk/src/org/apache/pig/pen/PhysicalPlanResetter.java
pig/trunk/src/org/apache/pig/scripting/ScriptEngine.java
pig/trunk/src/org/apache/pig/tools/pigstats/mapreduce/SimplePigStats.java
pig/trunk/test/e2e/pig/build.xml
pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm
pig/trunk/test/e2e/pig/tests/bigdata.conf
pig/trunk/test/e2e/pig/tests/cmdline.conf
pig/trunk/test/e2e/pig/tests/grunt.conf
pig/trunk/test/e2e/pig/tests/hcat.conf
pig/trunk/test/e2e/pig/tests/multiquery.conf
pig/trunk/test/e2e/pig/tests/negative.conf
pig/trunk/test/e2e/pig/tests/nightly.conf
pig/trunk/test/e2e/pig/tests/streaming.conf
pig/trunk/test/org/apache/pig/test/MiniGenericCluster.java
pig/trunk/test/org/apache/pig/test/TestAccumulator.java
pig/trunk/test/org/apache/pig/test/TestAlgebraicEval.java
pig/trunk/test/org/apache/pig/test/TestBZip.java
pig/trunk/test/org/apache/pig/test/TestBatchAliases.java
pig/trunk/test/org/apache/pig/test/TestBestFitCast.java
pig/trunk/test/org/apache/pig/test/TestBinaryExpressionOps.java
pig/trunk/test/org/apache/pig/test/TestBuiltin.java
pig/trunk/test/org/apache/pig/test/TestCharArrayToNumeric.java
pig/trunk/test/org/apache/pig/test/TestCombiner.java
pig/trunk/test/org/apache/pig/test/TestCompressedFiles.java
pig/trunk/test/org/apache/pig/test/TestEvalPipeline.java
pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java
pig/trunk/test/org/apache/pig/test/TestFRJoin.java
pig/trunk/test/org/apache/pig/test/TestFRJoin2.java
pig/trunk/test/org/apache/pig/test/TestFRJoinNullValue.java
pig/trunk/test/org/apache/pig/test/TestFilterUDF.java
pig/trunk/test/org/apache/pig/test/TestFinish.java
pig/trunk/test/org/apache/pig/test/TestForEachNestedPlan.java
pig/trunk/test/org/apache/pig/test/TestGrunt.java
pig/trunk/test/org/apache/pig/test/TestImplicitSplit.java
pig/trunk/test/org/apache/pig/test/TestInputOutputMiniClusterFileValidator.java
pig/trunk/test/org/apache/pig/test/TestJoin.java
pig/trunk/test/org/apache/pig/test/TestJoinSmoke.java
pig/trunk/test/org/apache/pig/test/TestKeyTypeDiscoveryVisitor.java
pig/trunk/test/org/apache/pig/test/TestLoad.java
pig/trunk/test/org/apache/pig/test/TestLocalRearrange.java
pig/trunk/test/org/apache/pig/test/TestMRCompiler.java
pig/trunk/test/org/apache/pig/test/TestMapReduce.java
pig/trunk/test/org/apache/pig/test/TestMapReduce2.java
pig/trunk/test/org/apache/pig/test/TestMultiQuery.java
pig/trunk/test/org/apache/pig/test/TestMultiQueryBasic.java
pig/trunk/test/org/apache/pig/test/TestMultiQueryCompiler.java
pig/trunk/test/org/apache/pig/test/TestMultiQueryLocal.java
pig/trunk/test/org/apache/pig/test/TestNestedForeach.java
pig/trunk/test/org/apache/pig/test/TestNewPlanImplicitSplit.java
pig/trunk/test/org/apache/pig/test/TestPackage.java
pig/trunk/test/org/apache/pig/test/TestPigContext.java
pig/trunk/test/org/apache/pig/test/TestPigRunner.java
pig/trunk/test/org/apache/pig/test/TestPigServer.java
pig/trunk/test/org/apache/pig/test/TestPigServerWithMacros.java
pig/trunk/test/org/apache/pig/test/TestPigStorage.java
pig/trunk/test/org/apache/pig/test/TestPoissonSampleLoader.java
pig/trunk/test/org/apache/pig/test/TestRegisteredJarVisibility.java
pig/trunk/test/org/apache/pig/test/TestSample.java
pig/trunk/test/org/apache/pig/test/TestSchema.java
pig/trunk/test/org/apache/pig/test/TestScriptLanguageJavaScript.java
pig/trunk/test/org/apache/pig/test/TestScriptUDF.java
pig/trunk/test/org/apache/pig/test/TestSecondarySort.java
pig/trunk/test/org/apache/pig/test/TestSkewedJoin.java
pig/trunk/test/org/apache/pig/test/TestSplitStore.java
pig/trunk/test/org/apache/pig/test/TestStore.java
pig/trunk/test/org/apache/pig/test/TestStoreInstances.java
pig/trunk/test/org/apache/pig/test/TestStoreOld.java
pig/trunk/test/org/apache/pig/test/TestStreaming.java
pig/trunk/test/org/apache/pig/test/TestToolsPigServer.java
pig/trunk/test/org/apache/pig/test/TestUDF.java
pig/trunk/test/org/apache/pig/test/TestUDFContext.java
pig/trunk/test/org/apache/pig/test/Util.java
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC1.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC10.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC11.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC12.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC13.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC14.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC2.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC3.gld
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1598702&r1=1598701&r2=1598702&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri May 30 19:07:23 2014
@@ -23,6 +23,8 @@ Trunk (unreleased changes)
INCOMPATIBLE CHANGES
IMPROVEMENTS
+
+PIG-3970: Merge Tez branch into trunk (daijy)
OPTIMIZATIONS
Modified: pig/trunk/build.xml
URL: http://svn.apache.org/viewvc/pig/trunk/build.xml?rev=1598702&r1=1598701&r2=1598702&view=diff
==============================================================================
--- pig/trunk/build.xml (original)
+++ pig/trunk/build.xml Fri May 30 19:07:23 2014
@@ -90,6 +90,10 @@
<property name="output.jarfile.backcompat.withouthadoop" value="${basedir}/${name}-withouthadoop.jar" />
<!-- test properties -->
+ <condition property="test.exec.type" value="${exectype}" else="mr">
+ <!-- By default, test.exec.type is mr -->
+ <isset property="exectype"/>
+ </condition>
<property name="test.src.dir" value="${basedir}/test" />
<property name="test.build.dir" value="${build.dir}/test" />
<property name="test.build.classes" value="${test.build.dir}/classes" />
@@ -100,6 +104,7 @@
<property name="test.unit.file" value="${test.src.dir}/unit-tests"/>
<property name="test.smoke.file" value="${test.src.dir}/smoke-tests"/>
<property name="test.all.file" value="${test.src.dir}/all-tests"/>
+ <property name="test.tez.file" value="${test.src.dir}/tez-tests"/>
<property name="test.exclude.file" value="${test.src.dir}/excluded-tests"/>
<property name="test.exclude.file.23" value="${test.src.dir}/excluded-tests-23"/>
<property name="pigunit.jarfile" value="pigunit.jar" />
@@ -185,10 +190,6 @@
<property name="ivy.dir" location="ivy" />
<loadproperties srcfile="${ivy.dir}/libraries.properties"/>
- <!--propertyregex property="hadoopversion"
- input="${hadoop-core.version}"
- regexp="\d+\.(\d+)\.\d+"
- select="\1" /-->
<property name="hadoopversion" value="20" />
<condition property="isHadoop23">
@@ -206,6 +207,13 @@
-->
<property name="hbaseversion" value="94" />
+ <!-- exclude tez code if not hadoop20 -->
+ <condition property="src.exclude.dir" value="**/tez/**" else="">
+ <not>
+ <equals arg1="${hadoopversion}" arg2="23"/>
+ </not>
+ </condition>
+
<property name="src.shims.dir" value="${basedir}/shims/src/hadoop${hadoopversion}" />
<property name="src.shims.test.dir" value="${basedir}/shims/test/hadoop${hadoopversion}" />
@@ -357,9 +365,9 @@
<include name="guava-${guava.version}.jar"/>
<include name="automaton-${automaton.version}.jar"/>
<include name="jansi-${jansi.version}.jar"/>
- <include name="avro-${avro.version}.jar"/>
- <include name="avro-mapred-${avro.version}.jar"/>
- <include name="trevni-core-${avro.version}.jar"/>
+ <include name="avro-${avro.version}.jar"/>
+ <include name="avro-mapred-${avro.version}.jar"/>
+ <include name="trevni-core-${avro.version}.jar"/>
<include name="trevni-avro-${avro.version}.jar"/>
<include name="snappy-java-${snappy.version}.jar"/>
<include name="asm*.jar"/>
@@ -379,11 +387,12 @@
<include name="slf4j*.jar"/>
<include name="jsp-api*.jar"/>
<include name="avro-${avro.version}.jar"/>
- <include name="avro-mapred-${avro.version}.jar"/>
- <include name="trevni-core-${avro.version}.jar"/>
+ <include name="avro-mapred-${avro.version}.jar"/>
+ <include name="trevni-core-${avro.version}.jar"/>
<include name="trevni-avro-${avro.version}.jar"/>
<include name="snappy-java-${snappy.version}.jar"/>
<include name="asm*.jar"/>
+ <include name="tez-*-${tez.version}.jar" if="isHadoop23"/>
</patternset>
</fileset>
@@ -551,7 +560,7 @@
<echo>*** To compile with all warnings enabled, supply -Dall.warnings=1 on command line ***</echo>
<echo>*** Else, you will only be warned about deprecations ***</echo>
<compileSources sources="${src.dir};${src.gen.dir};${src.lib.dir}/bzip2;${src.shims.dir}"
- excludes="" dist="${build.classes}" cp="classpath" warnings="${javac.args.warnings}" />
+ excludes="${src.exclude.dir}" dist="${build.classes}" cp="classpath" warnings="${javac.args.warnings}" />
<copy todir="${build.classes}/META-INF">
<fileset dir="${src.dir}/META-INF" includes="**"/>
</copy>
@@ -562,7 +571,8 @@
<echo>*** To compile with all warnings enabled, supply -Dall.warnings=1 on command line ***</echo>
<echo>*** Else, you will only be warned about deprecations ***</echo>
<compileSources sources="${test.src.dir};${src.shims.test.dir}"
- excludes="**/PigTestLoader.java **/resources/** perf/**" dist="${test.build.classes}" cp="test.classpath" warnings="${javac.args.warnings}" />
+ excludes="**/PigTestLoader.java **/resources/** perf/** ${src.exclude.dir}"
+ dist="${test.build.classes}" cp="test.classpath" warnings="${javac.args.warnings}" />
<copy file="${basedir}/test/hbase-site.xml" tofile="${test.build.classes}/hbase-site.xml"/>
<ivy:cachepath pathid="mr-apps-test-ivy.classpath" conf="test" />
<path id="mr-apps-test.classpath">
@@ -839,6 +849,10 @@
<macro-test-runner test.file="${test.smoke.file}" />
</target>
+ <target name="test-tez" depends="setWindowsPath,setLinuxPath,compile-test,jar-withouthadoop,debugger.check" description="Run tez unit tests">
+ <macro-test-runner test.file="${test.tez.file}" />
+ </target>
+
<target name="debugger.check" depends="debugger.set,debugger.unset"/>
<target name="debugger.set" if="debugPort">
<property name="debugArgs" value="-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=${debugPort}"/>
@@ -855,12 +869,13 @@
<mkdir dir="${test.log.dir}"/>
<tempfile property="junit.tmp.dir" prefix="pig_junit_tmp" destDir="${java.io.tmpdir}" />
<mkdir dir="${junit.tmp.dir}/"/>
- <junit showoutput="${test.output}" printsummary="yes" haltonfailure="no" fork="yes" maxmemory="1024m" dir="${basedir}" timeout="${test.timeout}" errorProperty="tests.failed" failureProperty="tests.failed">
+ <junit showoutput="${test.output}" printsummary="yes" haltonfailure="no" fork="yes" maxmemory="2048m" dir="${basedir}" timeout="${test.timeout}" errorProperty="tests.failed" failureProperty="tests.failed">
+ <sysproperty key="test.exec.type" value="${test.exec.type}" />
<sysproperty key="ssh.gateway" value="${ssh.gateway}" />
<sysproperty key="hod.server" value="${hod.server}" />
<sysproperty key="java.io.tmpdir" value="${junit.tmp.dir}" />
<sysproperty key="hadoop.log.dir" value="${test.log.dir}"/>
- <jvmarg line="${debugArgs}"/>
+ <jvmarg line="-XX:+CMSClassUnloadingEnabled -XX:MaxPermSize=128M ${debugArgs}"/>
<sysproperty key="java.security.krb5.realm" value="" />
<sysproperty key="java.security.krb5.kdc" value="" />
<sysproperty key="log4j.configuration" value="file:${basedir}/conf/test-log4j.properties"/>
@@ -921,6 +936,10 @@
<ant dir="${test.e2e.dir}"/>
</target>
+ <target name="test-e2e-tez" depends="jar, jar-withouthadoop, piggybank" description="run end-to-end tests in tez mode">
+ <ant dir="${test.e2e.dir}" target="test-tez"/>
+ </target>
+
<target name="test-e2e-deploy" depends="jar, jar-withouthadoop" description="deploy end-to-end tests to existing cluster">
<ant dir="${test.e2e.dir}" target="deploy"/>
</target>
Modified: pig/trunk/ivy.xml
URL: http://svn.apache.org/viewvc/pig/trunk/ivy.xml?rev=1598702&r1=1598701&r2=1598702&view=diff
==============================================================================
--- pig/trunk/ivy.xml (original)
+++ pig/trunk/ivy.xml Fri May 30 19:07:23 2014
@@ -166,6 +166,8 @@
rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
<dependency org="org.apache.hadoop" name="hadoop-yarn-client"
rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
+ <dependency org="org.apache.hadoop" name="hadoop-yarn-server-applicationhistoryservice"
+ rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
<dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-hs"
rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
<dependency org="org.mortbay.jetty" name="jetty" rev="${jetty.version}"
@@ -403,9 +405,29 @@
</dependency>
<dependency org="org.mockito" name="mockito-all" rev="${mockito.version}" conf="compile->default"/>
-
+
<dependency org="com.twitter" name="parquet-pig-bundle" rev="${parquet-pig-bundle.version}" conf="compile->master"/>
+ <!-- for Tez integration -->
+ <dependency org="org.apache.tez" name="tez" rev="${tez.version}"
+ conf="hadoop23->master"/>
+ <dependency org="org.apache.tez" name="tez-common" rev="${tez.version}"
+ conf="hadoop23->master"/>
+ <dependency org="org.apache.tez" name="tez-api" rev="${tez.version}"
+ conf="hadoop23->master"/>
+ <dependency org="org.apache.tez" name="tez-dag" rev="${tez.version}"
+ conf="hadoop23->master"/>
+ <dependency org="org.apache.tez" name="tez-runtime-internals" rev="${tez.version}"
+ conf="hadoop23->master"/>
+ <dependency org="org.apache.tez" name="tez-runtime-library" rev="${tez.version}"
+ conf="hadoop23->master"/>
+ <dependency org="org.apache.tez" name="tez-mapreduce" rev="${tez.version}"
+ conf="hadoop23->master"/>
+ <dependency org="org.apache.commons" name="commons-collections4" rev="${commons-collections4.version}"
+ conf="hadoop23->master"/>
+ <dependency org="org.codehaus.jettison" name="jettison" rev="${jettison.version}"
+ conf="hadoop23->master"/>
+
</dependencies>
</ivy-module>
Modified: pig/trunk/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/pig/trunk/ivy/libraries.properties?rev=1598702&r1=1598701&r2=1598702&view=diff
==============================================================================
--- pig/trunk/ivy/libraries.properties (original)
+++ pig/trunk/ivy/libraries.properties Fri May 30 19:07:23 2014
@@ -20,7 +20,7 @@ apacherat.version=0.8
automaton.version=1.11-8
avro.version=1.7.5
commons-beanutils.version=1.7.0
-commons-cli.version=1.0
+commons-cli.version=1.2
commons-codec.version=1.4
commons-io.version=2.3
commons-el.version=1.0
@@ -28,6 +28,7 @@ commons-logging.version=1.1.1
commons-lang.version=2.4
commons-configuration.version=1.6
commons-collections.version=3.2.1
+commons-collections4.version=4.0
commons-httpclient.version=3.1
xmlenc.version=0.52
jersey.version=1.8
@@ -39,9 +40,9 @@ guava.version=11.0
jersey-core.version=1.8
hadoop-core.version=1.0.4
hadoop-test.version=1.0.4
-hadoop-common.version=2.0.3-alpha
-hadoop-hdfs.version=2.0.3-alpha
-hadoop-mapreduce.version=2.0.3-alpha
+hadoop-common.version=2.4.0
+hadoop-hdfs.version=2.4.0
+hadoop-mapreduce.version=2.4.0
hbase94.version=0.94.1
hbase95.version=0.96.0-hadoop1
hsqldb.version=1.8.0.10
@@ -55,6 +56,7 @@ jaxb-api.version=2.2.2
jaxb-impl.version=2.2.3-1
jdeb.version=0.8
jdiff.version=1.0.9
+jettison.version=1.3.4
jetty.version=6.1.26
jetty-util.version=6.1.26
jline.version=1.0
@@ -78,7 +80,7 @@ wagon-http.version=1.0-beta-2
zookeeper.version=3.4.5
servlet.version=4.0.6
servlet-api.version=2.5
-protobuf-java.version=2.4.0a
+protobuf-java.version=2.5.0
high-scale-lib.version=1.1.1
metrics-core.version=2.1.2
guice.version=3.0
@@ -89,4 +91,5 @@ mockito.version=1.8.4
jansi.version=1.9
asm.version=3.3.1
snappy.version=1.1.0.1
+tez.version=0.5.0-incubating-SNAPSHOT
parquet-pig-bundle.version=1.2.3
Modified: pig/trunk/shims/src/hadoop23/org/apache/pig/backend/hadoop23/PigJobControl.java
URL: http://svn.apache.org/viewvc/pig/trunk/shims/src/hadoop23/org/apache/pig/backend/hadoop23/PigJobControl.java?rev=1598702&r1=1598701&r2=1598702&view=diff
==============================================================================
--- pig/trunk/shims/src/hadoop23/org/apache/pig/backend/hadoop23/PigJobControl.java (original)
+++ pig/trunk/shims/src/hadoop23/org/apache/pig/backend/hadoop23/PigJobControl.java Fri May 30 19:07:23 2014
@@ -77,7 +77,7 @@ public class PigJobControl extends JobCo
}
}
- private int timeToSleep;
+ protected int timeToSleep;
/**
* Construct a job control for a group of jobs.
@@ -174,6 +174,9 @@ public class PigJobControl extends JobCo
synchronized(this) {
Iterator<ControlledJob> it = getJobs(jobsInProgress).iterator();
+ if (!it.hasNext()) {
+ stop();
+ }
while(it.hasNext()) {
ControlledJob j = it.next();
log.debug("Checking state of job "+j);
Modified: pig/trunk/shims/test/hadoop20/org/apache/pig/test/MiniCluster.java
URL: http://svn.apache.org/viewvc/pig/trunk/shims/test/hadoop20/org/apache/pig/test/MiniCluster.java?rev=1598702&r1=1598701&r2=1598702&view=diff
==============================================================================
--- pig/trunk/shims/test/hadoop20/org/apache/pig/test/MiniCluster.java (original)
+++ pig/trunk/shims/test/hadoop20/org/apache/pig/test/MiniCluster.java Fri May 30 19:07:23 2014
@@ -24,14 +24,26 @@ import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.mapred.MiniMRCluster;
+import org.apache.pig.ExecType;
public class MiniCluster extends MiniGenericCluster {
private static final File CONF_DIR = new File("build/classes");
private static final File CONF_FILE = new File(CONF_DIR, "hadoop-site.xml");
-
+
private MiniMRCluster m_mr = null;
- public MiniCluster() {
- super();
+
+ /**
+ * @deprecated use {@link org.apache.pig.test.MiniGenericCluster.buildCluster() instead.
+ */
+ @Deprecated
+ public static MiniCluster buildCluster() {
+ System.setProperty("test.exec.type", "mr");
+ return (MiniCluster)MiniGenericCluster.buildCluster();
+ }
+
+ @Override
+ protected ExecType getExecType() {
+ return ExecType.MAPREDUCE;
}
@Override
Added: pig/trunk/shims/test/hadoop20/org/apache/pig/test/TezMiniCluster.java
URL: http://svn.apache.org/viewvc/pig/trunk/shims/test/hadoop20/org/apache/pig/test/TezMiniCluster.java?rev=1598702&view=auto
==============================================================================
--- pig/trunk/shims/test/hadoop20/org/apache/pig/test/TezMiniCluster.java (added)
+++ pig/trunk/shims/test/hadoop20/org/apache/pig/test/TezMiniCluster.java Fri May 30 19:07:23 2014
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.test;
+
+import org.apache.pig.ExecType;
+import org.apache.pig.test.MiniGenericCluster;
+
+/**
+ * Dummy class for compile-time compatibility with Hadoop 1.x and 0.20.x
+ */
+public class TezMiniCluster extends MiniGenericCluster {
+ TezMiniCluster() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected ExecType getExecType() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected void setupMiniDfsAndMrClusters() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected void shutdownMiniMrClusters() {
+ throw new UnsupportedOperationException();
+ }
+}
Modified: pig/trunk/shims/test/hadoop23/org/apache/pig/test/MiniCluster.java
URL: http://svn.apache.org/viewvc/pig/trunk/shims/test/hadoop23/org/apache/pig/test/MiniCluster.java?rev=1598702&r1=1598701&r2=1598702&view=diff
==============================================================================
--- pig/trunk/shims/test/hadoop23/org/apache/pig/test/MiniCluster.java (original)
+++ pig/trunk/shims/test/hadoop23/org/apache/pig/test/MiniCluster.java Fri May 30 19:07:23 2014
@@ -22,17 +22,11 @@ import java.io.FileOutputStream;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapreduce.filecache.DistributedCache;
-import org.apache.hadoop.mapreduce.v2.MiniMRYarnCluster;
-import org.apache.hadoop.mapreduce.v2.TestMRJobs;
-import org.apache.hadoop.yarn.server.MiniYARNCluster;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
+import org.apache.hadoop.mapreduce.filecache.DistributedCache;
+import org.apache.hadoop.mapreduce.v2.MiniMRYarnCluster;
+import org.apache.pig.ExecType;
/**
* This class builds a single instance of itself with the Singleton
@@ -49,13 +43,23 @@ public class MiniCluster extends MiniGen
private Configuration m_dfs_conf = null;
private Configuration m_mr_conf = null;
- public MiniCluster() {
- super();
+ /**
+ * @deprecated use {@link org.apache.pig.test.MiniGenericCluster.buildCluster() instead.
+ */
+ @Deprecated
+ public static MiniCluster buildCluster() {
+ System.setProperty("test.exec.type", "mr");
+ return (MiniCluster)MiniGenericCluster.buildCluster("mr");
+ }
+
+ @Override
+ protected ExecType getExecType() {
+ return ExecType.MAPREDUCE;
}
@Override
protected void setupMiniDfsAndMrClusters() {
- try {
+ try {
final int dataNodes = 4; // There will be 4 data nodes
final int taskTrackers = 4; // There will be 4 task tracker nodes
@@ -74,7 +78,7 @@ public class MiniCluster extends MiniGen
m_dfs = new MiniDFSCluster(config, dataNodes, true, null);
m_fileSys = m_dfs.getFileSystem();
m_dfs_conf = m_dfs.getConfiguration(0);
-
+
//Create user home directory
m_fileSys.mkdirs(m_fileSys.getWorkingDirectory());
@@ -86,7 +90,7 @@ public class MiniCluster extends MiniGen
m_mr_conf = new Configuration(m_mr.getConfig());
m_conf = m_mr_conf;
- m_conf.set("fs.default.name", m_dfs_conf.get("fs.default.name"));
+ m_conf.set("fs.default.name", m_dfs_conf.get("fs.default.name"));
m_conf.unset("mapreduce.job.cache.files");
m_conf.setInt("mapred.io.sort.mb", 200);
@@ -103,7 +107,7 @@ public class MiniCluster extends MiniGen
new Path("/pigtest/conf/hadoop-site.xml"));
DistributedCache.addFileToClassPath(new Path("/pigtest/conf/hadoop-site.xml"), m_conf);
- System.err.println("XXX: Setting fs.default.name to: " + m_dfs_conf.get("fs.default.name"));
+ System.err.println("XXX: Setting fs.default.name to: " + m_dfs_conf.get("fs.default.name"));
// Set the system properties needed by Pig
System.setProperty("cluster", m_conf.get("mapred.job.tracker"));
//System.setProperty("namenode", m_dfs_conf.get("fs.default.name"));
Added: pig/trunk/shims/test/hadoop23/org/apache/pig/test/TezMiniCluster.java
URL: http://svn.apache.org/viewvc/pig/trunk/shims/test/hadoop23/org/apache/pig/test/TezMiniCluster.java?rev=1598702&view=auto
==============================================================================
--- pig/trunk/shims/test/hadoop23/org/apache/pig/test/TezMiniCluster.java (added)
+++ pig/trunk/shims/test/hadoop23/org/apache/pig/test/TezMiniCluster.java Fri May 30 19:07:23 2014
@@ -0,0 +1,174 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.test;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+import org.apache.commons.io.filefilter.RegexFileFilter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.v2.MiniMRYarnCluster;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.pig.ExecType;
+import org.apache.pig.PigConfiguration;
+import org.apache.pig.backend.hadoop.executionengine.tez.TezExecType;
+import org.apache.pig.backend.hadoop.executionengine.tez.TezSessionManager;
+import org.apache.tez.common.TezJobConfig;
+
+public class TezMiniCluster extends MiniGenericCluster {
+ private static final File CONF_DIR = new File("build/classes");
+ private static final File TEZ_LIB_DIR = new File("build/ivy/lib/Pig");
+ private static final File TEZ_CONF_FILE = new File(CONF_DIR, "tez-site.xml");
+ private static final File CORE_CONF_FILE = new File(CONF_DIR, "core-site.xml");
+ private static final File HDFS_CONF_FILE = new File(CONF_DIR, "hdfs-site.xml");
+ private static final File MAPRED_CONF_FILE = new File(CONF_DIR, "mapred-site.xml");
+ private static final ExecType TEZ = new TezExecType();
+
+ protected MiniMRYarnCluster m_mr = null;
+ private Configuration m_dfs_conf = null;
+ private Configuration m_mr_conf = null;
+
+ @Override
+ protected ExecType getExecType() {
+ return TEZ;
+ }
+
+ @Override
+ public void setupMiniDfsAndMrClusters() {
+ try {
+ deleteConfFiles();
+ CONF_DIR.mkdirs();
+
+ // Build mini DFS cluster
+ Configuration hdfsConf = new Configuration(false);
+ hdfsConf.addResource("core-default.xml");
+ hdfsConf.addResource("hdfs-default.xml");
+ m_dfs = new MiniDFSCluster.Builder(hdfsConf)
+ .numDataNodes(2)
+ .format(true)
+ .racks(null)
+ .build();
+ m_fileSys = m_dfs.getFileSystem();
+ m_dfs_conf = m_dfs.getConfiguration(0);
+ //Create user home directory
+ m_fileSys.mkdirs(m_fileSys.getWorkingDirectory());
+
+ m_dfs_conf.writeXml(new FileOutputStream(HDFS_CONF_FILE));
+ m_fileSys.copyFromLocalFile(
+ new Path(HDFS_CONF_FILE.getAbsoluteFile().toString()),
+ new Path("/pigtest/conf/hdfs-site.xml"));
+ Job job = Job.getInstance(m_dfs_conf);
+ job.addFileToClassPath(new Path("/pigtest/conf/hdfs-site.xml"));
+
+ // Build mini YARN cluster
+ m_mr = new MiniMRYarnCluster("PigMiniCluster", 2);
+ m_mr.init(m_dfs_conf);
+ m_mr.start();
+ m_mr_conf = m_mr.getConfig();
+ m_mr_conf.set("mapreduce.framework.name", "yarn-tez");
+ m_mr_conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
+ System.getProperty("java.class.path"));
+
+ m_mr_conf.writeXml(new FileOutputStream(MAPRED_CONF_FILE));
+ m_fileSys.copyFromLocalFile(
+ new Path(MAPRED_CONF_FILE.getAbsoluteFile().toString()),
+ new Path("/pigtest/conf/mapred-site.xml"));
+ job.addFileToClassPath(new Path("/pigtest/conf/mapred-site.xml"));
+
+ // Write core-site.xml
+ m_conf = m_mr_conf;
+ m_conf.writeXml(new FileOutputStream(CORE_CONF_FILE));
+ m_fileSys.copyFromLocalFile(
+ new Path(CORE_CONF_FILE.getAbsoluteFile().toString()),
+ new Path("/pigtest/conf/core-site.xml"));
+ job.addFileToClassPath(new Path("/pigtest/conf/core-site.xml"));
+
+ // Write tez-site.xml
+ Configuration tez_conf = new Configuration(false);
+ // TODO Remove this once TezSession reuse in mini cluster is fixed
+ tez_conf.set(PigConfiguration.TEZ_SESSION_REUSE, "false");
+ // TODO PIG-3659 - Remove this once memory management is fixed
+ tez_conf.set(TezJobConfig.TEZ_RUNTIME_IO_SORT_MB, "20");
+ tez_conf.set("tez.lib.uris", "hdfs:///tez,hdfs:///tez/lib");
+ tez_conf.writeXml(new FileOutputStream(TEZ_CONF_FILE));
+ m_fileSys.copyFromLocalFile(
+ new Path(TEZ_CONF_FILE.getAbsoluteFile().toString()),
+ new Path("/pigtest/conf/tez-site.xml"));
+ job.addFileToClassPath(new Path("/pigtest/conf/tez-site.xml"));
+
+ // Copy tez jars to hdfs
+ m_fileSys.mkdirs(new Path("/tez/lib"));
+ FileFilter fileFilter = new RegexFileFilter("tez-.+\\.jar$");
+ File[] tezJars = TEZ_LIB_DIR.listFiles(fileFilter);
+ for (int i = 0; i < tezJars.length; i++) {
+ if (tezJars[i].getName().startsWith("tez-api")) {
+ m_fileSys.copyFromLocalFile(
+ new Path(tezJars[i].getAbsoluteFile().toString()),
+ new Path("/tez"));
+ } else {
+ m_fileSys.copyFromLocalFile(
+ new Path(tezJars[i].getAbsoluteFile().toString()),
+ new Path("/tez/lib"));
+ }
+ }
+
+ // Turn FetchOptimizer off so that we can actually test Tez
+ m_conf.set(PigConfiguration.OPT_FETCH, System.getProperty("test.opt.fetch", "false"));
+
+ System.setProperty("junit.hadoop.conf", CONF_DIR.getPath());
+ System.setProperty("hadoop.log.dir", "build/test/logs");
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ protected void shutdownMiniDfsAndMrClusters() {
+ TezSessionManager.shutdown();
+ super.shutdownMiniDfsAndMrClusters();
+ }
+
+ @Override
+ protected void shutdownMiniMrClusters() {
+ deleteConfFiles();
+ if (m_mr != null) {
+ m_mr.stop();
+ m_mr = null;
+ }
+ }
+
+ private void deleteConfFiles() {
+ if(TEZ_CONF_FILE.exists()) {
+ TEZ_CONF_FILE.delete();
+ }
+ if(CORE_CONF_FILE.exists()) {
+ CORE_CONF_FILE.delete();
+ }
+ if(HDFS_CONF_FILE.exists()) {
+ HDFS_CONF_FILE.delete();
+ }
+ if(MAPRED_CONF_FILE.exists()) {
+ MAPRED_CONF_FILE.delete();
+ }
+ }
+}
Modified: pig/trunk/src/META-INF/services/org.apache.pig.ExecType
URL: http://svn.apache.org/viewvc/pig/trunk/src/META-INF/services/org.apache.pig.ExecType?rev=1598702&r1=1598701&r2=1598702&view=diff
==============================================================================
--- pig/trunk/src/META-INF/services/org.apache.pig.ExecType (original)
+++ pig/trunk/src/META-INF/services/org.apache.pig.ExecType Fri May 30 19:07:23 2014
@@ -13,4 +13,6 @@
#
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.LocalExecType
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRExecType
+org.apache.pig.backend.hadoop.executionengine.tez.TezLocalExecType
+org.apache.pig.backend.hadoop.executionengine.tez.TezExecType
Modified: pig/trunk/src/org/apache/pig/Main.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/Main.java?rev=1598702&r1=1598701&r2=1598702&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/Main.java (original)
+++ pig/trunk/src/org/apache/pig/Main.java Fri May 30 19:07:23 2014
@@ -298,7 +298,7 @@ public class Main {
case 'M':
// turns off multiquery optimization
- properties.setProperty("opt.multiquery",""+false);
+ properties.setProperty(PigConfiguration.OPT_MULTIQUERY,""+false);
break;
case 'N':
Modified: pig/trunk/src/org/apache/pig/PigConfiguration.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/PigConfiguration.java?rev=1598702&r1=1598701&r2=1598702&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/PigConfiguration.java (original)
+++ pig/trunk/src/org/apache/pig/PigConfiguration.java Fri May 30 19:07:23 2014
@@ -71,7 +71,27 @@ public class PigConfiguration {
public static final String SCHEMA_TUPLE_SHOULD_ALLOW_FORCE = "pig.schematuple.force";
- /*
+ /**
+ * This key is used to enable multiquery optimization.
+ */
+ public static final String OPT_MULTIQUERY = "opt.multiquery";
+
+ /**
+ * This key is used to enable accumulator optimization.
+ */
+ public static final String OPT_ACCUMULATOR = "opt.accumulator";
+
+ /**
+ * This key is used to enable union optimization.
+ */
+ public static final String TEZ_OPT_UNION = "pig.tez.opt.union";
+
+ /**
+ * This key is used to define whether to reuse AM in Tez jobs.
+ */
+ public static final String TEZ_SESSION_REUSE = "pig.tez.session.reuse";
+
+ /**
* Turns off use of combiners in MapReduce jobs produced by Pig.
*/
public static final String PROP_NO_COMBINER = "pig.exec.nocombiner";
@@ -135,11 +155,44 @@ public class PigConfiguration {
public static final String PIG_DELETE_TEMP_FILE = "pig.delete.temp.files";
/**
+ * For a given mean and a confidence, a sample rate is obtained from a poisson udf
+ */
+ public static final String SAMPLE_RATE = "pig.sksampler.samplerate";
+
+ /**
+ * % of memory available for the input data. This is currently equal to the
+ * memory available for the skewed join
+ */
+ public static final String PERC_MEM_AVAIL = "pig.skewedjoin.reduce.memusage";
+
+ /**
* This key used to control the maximum size loaded into
* the distributed cache when doing fragment-replicated join
*/
public static final String PIG_JOIN_REPLICATED_MAX_BYTES = "pig.join.replicated.max.bytes";
-
+
+ /**
+ * Turns combine split files on or off
+ */
+ public static final String PIG_SPLIT_COMBINATION = "pig.splitCombination";
+
+ /**
+ * Whether turns combine split files off. This is for internal use only
+ */
+ public static final String PIG_NO_SPLIT_COMBINATION = "pig.noSplitCombination";
+
+ /**
+ * Specifies the size, in bytes, of data to be processed by a single map.
+ * Smaller files are combined untill this size is reached.
+ */
+ public static final String PIG_MAX_COMBINED_SPLIT_SIZE = "pig.maxCombinedSplitSize";
+
+ /**
+ * This key controls whether secondary sort key is used for optimization in case
+ * of nested distinct or sort
+ */
+ public static final String PIG_EXEC_NO_SECONDARY_KEY = "pig.exec.nosecondarykey";
+
/**
* This key used to control the sample size of RandomeSampleLoader for
* order-by. The default value is 100 rows per task.
Modified: pig/trunk/src/org/apache/pig/PigServer.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/PigServer.java?rev=1598702&r1=1598701&r2=1598702&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/PigServer.java (original)
+++ pig/trunk/src/org/apache/pig/PigServer.java Fri May 30 19:07:23 2014
@@ -97,6 +97,8 @@ import org.apache.pig.tools.pigstats.Scr
import org.apache.pig.validator.BlackAndWhitelistFilter;
import org.apache.pig.validator.PigCommandFilter;
+import com.google.common.annotations.VisibleForTesting;
+
/**
*
* A class for Java programs to connect to Pig. Typically a program will create a PigServer
@@ -159,6 +161,11 @@ public class PigServer {
return "" + scopeCounter.incrementAndGet();
}
+ @VisibleForTesting
+ public static void resetScope() {
+ scopeCounter.set(0);
+ }
+
/**
* @param execTypeString can be 'mapreduce' or 'local'. Local mode will
* use Hadoop's local job runner to execute the job on the local machine.
@@ -172,6 +179,10 @@ public class PigServer {
this(addExecTypeProperty(PropertiesUtil.loadDefaultProperties(), execTypeString));
}
+ public PigServer(String execTypeString, Properties properties) throws ExecException, IOException {
+ this(addExecTypeProperty(properties, execTypeString));
+ }
+
public PigServer(Properties properties) throws ExecException, IOException {
this(new PigContext(properties));
}
@@ -750,7 +761,7 @@ public class PigServer {
registerScript(fis, params, paramsFiles);
}catch (FileNotFoundException e){
log.error(e.getLocalizedMessage());
- throw new IOException(e.getCause());
+ throw new IOException(e);
} finally {
if (fis != null) {
fis.close();
@@ -1366,9 +1377,9 @@ public class PigServer {
try {
stats = pigContext.getExecutionEngine().launchPig(lp, jobName, pigContext);
} catch (ExecException e) {
- throw (ExecException) e;
+ throw e;
} catch (FrontendException e) {
- throw (FrontendException) e;
+ throw e;
} catch (Exception e) {
// There are a lot of exceptions thrown by the launcher. If this
// is an ExecException, just let it through. Else wrap it.
Modified: pig/trunk/src/org/apache/pig/PigWarning.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/PigWarning.java?rev=1598702&r1=1598701&r2=1598702&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/PigWarning.java (original)
+++ pig/trunk/src/org/apache/pig/PigWarning.java Fri May 30 19:07:23 2014
@@ -41,7 +41,7 @@ public enum PigWarning {
IMPLICIT_CAST_TO_TUPLE,
TOO_LARGE_FOR_INT,
MULTI_LEAF_MAP,
- MULTI_LEAF_REDUCE,
+ MULTI_ROOT_REDUCE,
NON_PACKAGE_REDUCE_PLAN_ROOT,
NON_EMPTY_COMBINE_PLAN,
PROGRESS_REPORTER_NOT_PROVIDED,
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java?rev=1598702&r1=1598701&r2=1598702&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java Fri May 30 19:07:23 2014
@@ -34,8 +34,12 @@ import org.apache.pig.backend.hadoop.exe
public class ConfigurationUtil {
public static Configuration toConfiguration(Properties properties) {
+ return toConfiguration(properties, false);
+ }
+
+ public static Configuration toConfiguration(Properties properties, boolean loadDefaults) {
assert properties != null;
- final Configuration config = new Configuration(false);
+ final Configuration config = new Configuration(loadDefaults);
final Enumeration<Object> iter = properties.keys();
while (iter.hasMoreElements()) {
final String key = (String) iter.nextElement();
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/HPath.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/HPath.java?rev=1598702&r1=1598701&r2=1598702&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/HPath.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/HPath.java Fri May 30 19:07:23 2014
@@ -1,208 +1,208 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.pig.backend.hadoop.datastorage;
-
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.IOException;
-import java.util.Properties;
-import java.util.Map;
-import java.util.HashMap;
-
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.pig.PigException;
-import org.apache.pig.backend.datastorage.*;
-import org.apache.pig.backend.executionengine.ExecException;
-
-public abstract class HPath implements ElementDescriptor {
-
- protected Path path;
- protected HDataStorage fs;
-
- public HPath(HDataStorage fs, Path parent, Path child) {
- this.path = new Path(parent, child);
- this.fs = fs;
- }
-
- public HPath(HDataStorage fs, String parent, String child) {
- this(fs, new Path(parent), new Path(child));
- }
-
- public HPath(HDataStorage fs, Path parent, String child) {
- this(fs, parent, new Path(child));
- }
-
- public HPath(HDataStorage fs, String parent, Path child) {
- this(fs, new Path(parent), child);
- }
-
- public HPath(HDataStorage fs, String pathString) {
- this(fs, new Path(pathString));
- }
-
- public HPath(HDataStorage fs, Path path) {
- this.path = path;
- this.fs = fs;
- }
-
- public DataStorage getDataStorage() {
- return fs;
- }
-
- public abstract OutputStream create(Properties configuration)
- throws IOException;
-
- public void copy(ElementDescriptor dstName,
- Properties dstConfiguration,
- boolean removeSrc)
- throws IOException {
- FileSystem srcFS = this.fs.getHFS();
- FileSystem dstFS = ((HPath)dstName).fs.getHFS();
-
- Path srcPath = this.path;
- Path dstPath = ((HPath)dstName).path;
-
- boolean result = FileUtil.copy(srcFS,
- srcPath,
- dstFS,
- dstPath,
- false,
- new Configuration());
-
- if (!result) {
- int errCode = 2097;
- String msg = "Failed to copy from: " + this.toString() +
- " to: " + dstName.toString();
- throw new ExecException(msg, errCode, PigException.BUG);
- }
- }
-
- public abstract InputStream open() throws IOException;
-
- public abstract SeekableInputStream sopen() throws IOException;
-
- public boolean exists() throws IOException {
- return fs.getHFS().exists(path);
- }
-
- public void rename(ElementDescriptor newName)
- throws IOException {
- if (newName != null) {
- fs.getHFS().rename(path, ((HPath)newName).path);
- }
- }
-
- public void delete() throws IOException {
- // the file is removed and not placed in the trash bin
- fs.getHFS().delete(path, true);
- }
-
- public void setPermission(FsPermission permission) throws IOException {
- fs.getHFS().setPermission(path, permission);
- }
-
- public Properties getConfiguration() throws IOException {
- HConfiguration props = new HConfiguration();
-
- long blockSize = fs.getHFS().getFileStatus(path).getBlockSize();
-
- short replication = fs.getHFS().getFileStatus(path).getReplication();
-
- props.setProperty(BLOCK_SIZE_KEY, (Long.valueOf(blockSize)).toString());
- props.setProperty(BLOCK_REPLICATION_KEY, (Short.valueOf(replication)).toString());
-
- return props;
- }
-
- public void updateConfiguration(Properties newConfig) throws IOException {
- if (newConfig == null) {
- return;
- }
-
- String blkReplStr = newConfig.getProperty(BLOCK_REPLICATION_KEY);
-
- fs.getHFS().setReplication(path,
- new Short(blkReplStr).shortValue());
- }
-
- public Map<String, Object> getStatistics() throws IOException {
- HashMap<String, Object> props = new HashMap<String, Object>();
-
- FileStatus fileStatus = fs.getHFS().getFileStatus(path);
-
- props.put(BLOCK_SIZE_KEY, fileStatus.getBlockSize());
- props.put(BLOCK_REPLICATION_KEY, fileStatus.getReplication());
- props.put(LENGTH_KEY, fileStatus.getLen());
- props.put(MODIFICATION_TIME_KEY, fileStatus.getModificationTime());
-
- return props;
- }
-
- public OutputStream create() throws IOException {
- return create(null);
- }
-
- public void copy(ElementDescriptor dstName,
- boolean removeSrc)
- throws IOException {
- copy(dstName, null, removeSrc);
- }
-
- public Path getPath() {
- return path;
- }
-
- public FileSystem getHFS() {
- return fs.getHFS();
- }
-
- public boolean systemElement() {
- return (path != null &&
- (path.getName().startsWith("_") ||
- path.getName().startsWith(".")));
- }
-
- @Override
- public String toString() {
- return path.makeQualified(getHFS()).toString();
- }
-
- @Override
- public boolean equals(Object obj) {
- if (! (obj instanceof HPath)) {
- return false;
- }
-
- return this.path.equals(((HPath)obj).path);
- }
-
- public int compareTo(ElementDescriptor other) {
- return path.compareTo(((HPath)other).path);
- }
-
- @Override
- public int hashCode() {
- return this.path.hashCode();
- }
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.backend.hadoop.datastorage;
+
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.IOException;
+import java.util.Properties;
+import java.util.Map;
+import java.util.HashMap;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.pig.PigException;
+import org.apache.pig.backend.datastorage.*;
+import org.apache.pig.backend.executionengine.ExecException;
+
+public abstract class HPath implements ElementDescriptor {
+
+ protected Path path;
+ protected HDataStorage fs;
+
+ public HPath(HDataStorage fs, Path parent, Path child) {
+ this.path = new Path(parent, child);
+ this.fs = fs;
+ }
+
+ public HPath(HDataStorage fs, String parent, String child) {
+ this(fs, new Path(parent), new Path(child));
+ }
+
+ public HPath(HDataStorage fs, Path parent, String child) {
+ this(fs, parent, new Path(child));
+ }
+
+ public HPath(HDataStorage fs, String parent, Path child) {
+ this(fs, new Path(parent), child);
+ }
+
+ public HPath(HDataStorage fs, String pathString) {
+ this(fs, new Path(pathString));
+ }
+
+ public HPath(HDataStorage fs, Path path) {
+ this.path = path;
+ this.fs = fs;
+ }
+
+ public DataStorage getDataStorage() {
+ return fs;
+ }
+
+ public abstract OutputStream create(Properties configuration)
+ throws IOException;
+
+ public void copy(ElementDescriptor dstName,
+ Properties dstConfiguration,
+ boolean removeSrc)
+ throws IOException {
+ FileSystem srcFS = this.fs.getHFS();
+ FileSystem dstFS = ((HPath)dstName).fs.getHFS();
+
+ Path srcPath = this.path;
+ Path dstPath = ((HPath)dstName).path;
+
+ boolean result = FileUtil.copy(srcFS,
+ srcPath,
+ dstFS,
+ dstPath,
+ false,
+ new Configuration());
+
+ if (!result) {
+ int errCode = 2097;
+ String msg = "Failed to copy from: " + this.toString() +
+ " to: " + dstName.toString();
+ throw new ExecException(msg, errCode, PigException.BUG);
+ }
+ }
+
+ public abstract InputStream open() throws IOException;
+
+ public abstract SeekableInputStream sopen() throws IOException;
+
+ public boolean exists() throws IOException {
+ return fs.getHFS().exists(path);
+ }
+
+ public void rename(ElementDescriptor newName)
+ throws IOException {
+ if (newName != null) {
+ fs.getHFS().rename(path, ((HPath)newName).path);
+ }
+ }
+
+ public void delete() throws IOException {
+ // the file is removed and not placed in the trash bin
+ fs.getHFS().delete(path, true);
+ }
+
+ public void setPermission(FsPermission permission) throws IOException {
+ fs.getHFS().setPermission(path, permission);
+ }
+
+ public Properties getConfiguration() throws IOException {
+ HConfiguration props = new HConfiguration();
+
+ long blockSize = fs.getHFS().getFileStatus(path).getBlockSize();
+
+ short replication = fs.getHFS().getFileStatus(path).getReplication();
+
+ props.setProperty(BLOCK_SIZE_KEY, (Long.valueOf(blockSize)).toString());
+ props.setProperty(BLOCK_REPLICATION_KEY, (Short.valueOf(replication)).toString());
+
+ return props;
+ }
+
+ public void updateConfiguration(Properties newConfig) throws IOException {
+ if (newConfig == null) {
+ return;
+ }
+
+ String blkReplStr = newConfig.getProperty(BLOCK_REPLICATION_KEY);
+
+ fs.getHFS().setReplication(path,
+ new Short(blkReplStr).shortValue());
+ }
+
+ public Map<String, Object> getStatistics() throws IOException {
+ HashMap<String, Object> props = new HashMap<String, Object>();
+
+ FileStatus fileStatus = fs.getHFS().getFileStatus(path);
+
+ props.put(BLOCK_SIZE_KEY, fileStatus.getBlockSize());
+ props.put(BLOCK_REPLICATION_KEY, fileStatus.getReplication());
+ props.put(LENGTH_KEY, fileStatus.getLen());
+ props.put(MODIFICATION_TIME_KEY, fileStatus.getModificationTime());
+
+ return props;
+ }
+
+ public OutputStream create() throws IOException {
+ return create(null);
+ }
+
+ public void copy(ElementDescriptor dstName,
+ boolean removeSrc)
+ throws IOException {
+ copy(dstName, null, removeSrc);
+ }
+
+ public Path getPath() {
+ return path;
+ }
+
+ public FileSystem getHFS() {
+ return fs.getHFS();
+ }
+
+ public boolean systemElement() {
+ return (path != null &&
+ (path.getName().startsWith("_") ||
+ path.getName().startsWith(".")));
+ }
+
+ @Override
+ public String toString() {
+ return path.makeQualified(getHFS()).toString();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (! (obj instanceof HPath)) {
+ return false;
+ }
+
+ return this.path.equals(((HPath)obj).path);
+ }
+
+ public int compareTo(ElementDescriptor other) {
+ return path.compareTo(((HPath)other).path);
+ }
+
+ @Override
+ public int hashCode() {
+ return this.path.hashCode();
+ }
+}
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/AccumulatorOptimizer.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/AccumulatorOptimizer.java?rev=1598702&r1=1598701&r2=1598702&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/AccumulatorOptimizer.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/AccumulatorOptimizer.java Fri May 30 19:07:23 2014
@@ -18,274 +18,23 @@
package org.apache.pig.backend.hadoop.executionengine.mapReduceLayer;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.pig.Accumulator;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROpPlanVisitor;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.BinaryExpressionOperator;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.ConstantExpression;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.ExpressionOperator;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POBinCond;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POCast;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POMapLookUp;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POProject;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.PORelationToExprProject;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POUserFunc;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.UnaryExpressionOperator;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POForEach;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSortedDistinct;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.Packager;
-import org.apache.pig.data.DataType;
-import org.apache.pig.impl.PigContext;
+import org.apache.pig.backend.hadoop.executionengine.util.AccumulatorOptimizerUtil;
import org.apache.pig.impl.plan.DepthFirstWalker;
import org.apache.pig.impl.plan.VisitorException;
/**
- * A visitor to optimize plans that determines if a reduce plan
- * can run in accumulative mode.
+ * A visitor to optimize plans that determines if a reduce plan can run in
+ * accumulative mode.
*/
public class AccumulatorOptimizer extends MROpPlanVisitor {
- private Log log = LogFactory.getLog(getClass());
-
public AccumulatorOptimizer(MROperPlan plan) {
super(plan, new DepthFirstWalker<MapReduceOper, MROperPlan>(plan));
}
public void visitMROp(MapReduceOper mr) throws VisitorException {
- // See if this is a map-reduce job
- List<PhysicalOperator> pos = mr.reducePlan.getRoots();
- if (pos == null || pos.size() == 0) {
- return;
- }
-
- // See if this is a POPackage
- PhysicalOperator po_package = pos.get(0);
- if (!po_package.getClass().equals(POPackage.class)) {
- return;
- }
-
- Packager pkgr = ((POPackage) po_package).getPkgr();
- // Check that this is a standard package, not a subclass
- if (!pkgr.getClass().equals(Packager.class)) {
- return;
- }
-
- // if POPackage is for distinct, just return
- if (pkgr.isDistinct()) {
- return;
- }
-
- // if any input to POPackage is inner, just return
- boolean[] isInner = pkgr.getInner();
- for(boolean b: isInner) {
- if (b) {
- return;
- }
- }
-
- List<PhysicalOperator> l = mr.reducePlan.getSuccessors(po_package);
- // there should be only one POForEach
- if (l == null || l.size() == 0 || l.size() > 1) {
- return;
- }
-
- PhysicalOperator po_foreach = l.get(0);
- if (!(po_foreach instanceof POForEach)) {
- return;
- }
-
- boolean foundUDF = false;
- List<PhysicalPlan> list = ((POForEach)po_foreach).getInputPlans();
- for(PhysicalPlan p: list) {
- PhysicalOperator po = p.getLeaves().get(0);
-
- // only expression operators are allowed
- if (!(po instanceof ExpressionOperator)) {
- return;
- }
-
- if (((ExpressionOperator)po).containUDF()) {
- foundUDF = true;
- }
-
- if (!check(po)) {
- return;
- }
- }
-
- if (foundUDF) {
- // if all tests are passed, reducer can run in accumulative mode
- log.info("Reducer is to run in accumulative mode.");
- po_package.setAccumulative();
- po_foreach.setAccumulative();
- }
- }
-
- /**
- * Check if an operator is qualified to be under POForEach
- * to turn on accumulator. The operator must be in the following list or
- * an <code>POUserFunc</code>.
- *
- * If the operator has sub-operators, they must also belong to this list.
- * <li>ConstantExpression</li>
- * <li>POProject, whose result type is not BAG, or TUPLE and overloaded</li>
- * <li>POMapLookup</li>
- * <li>POCase</li>
- * <li>UnaryExpressionOperator</li>
- * <li>BinaryExpressionOperator</li>
- * <li>POBinCond</li>
- *
- * If the operator is <code>POUserFunc</code>, it must implement
- * <code>Accumulator</code> interface and its inputs pass the check
- * by calling <code>checkUDFInput()</code>
- *
- * @param po the operator to be checked on
- * @return <code>true</code> if it is ok, <code>false</code>
- * if not.
- */
- @SuppressWarnings("unchecked")
- private boolean check(PhysicalOperator po) {
- if (po instanceof ConstantExpression) {
- return true;
- }
-
- if (po instanceof POCast) {
- return check(po.getInputs().get(0));
- }
-
- if (po instanceof POMapLookUp) {
- return check(po.getInputs().get(0));
- }
-
- if (po instanceof POProject) {
- // POProject can not project data bag
- if (((POProject)po).getResultType() == DataType.BAG) {
- return false;
- }
-
- // POProject can not overload a data bag
- if (((POProject)po).getResultType() == DataType.TUPLE && ((POProject)po).isOverloaded()) {
- return false;
- }
-
- return true;
- }
-
- if (po instanceof UnaryExpressionOperator) {
- return check(((UnaryExpressionOperator)po).getExpr());
- }
-
- if (po instanceof BinaryExpressionOperator) {
- return check(((BinaryExpressionOperator)po).getLhs()) &&
- check(((BinaryExpressionOperator)po).getRhs());
- }
-
- if (po instanceof POBinCond) {
- return check(((POBinCond)po).getLhs()) &&
- check(((POBinCond)po).getRhs()) && check(((POBinCond)po).getCond());
- }
-
- if (po instanceof POUserFunc) {
- String className = ((POUserFunc)po).getFuncSpec().getClassName();
- Class c = null;
- try {
- c = PigContext.resolveClassName(className);
- }catch(Exception e) {
- return false;
- }
- if (!Accumulator.class.isAssignableFrom(c)) {
- return false;
- }
-
- // check input of UDF
- List<PhysicalOperator> inputs = po.getInputs();
- for(PhysicalOperator p: inputs) {
- if (!checkUDFInput(p)) {
- return false;
- }
- }
-
- return true;
- }
-
- return false;
- }
-
- /**
- * Check operators under POUserFunc to verify if this
- * is a valid UDF to run as accumulator. The inputs to
- * <code>POUserFunc</code> must be in the following list.
- * If the operator has sub-operators, they must also belong
- * to this list.
- *
- * <li>PORelationToExprProject</li>
- * <li>ConstantExpression</li>
- * <li>POProject</li>
- * <li>POCase</li>
- * <li>UnaryExpressionOperator</li>
- * <li>BinaryExpressionOperator</li>
- * <li>POBinCond</li>
- * <li>POSortedDistinct</li>
- * <li>POForEach</li>
- *
- */
- private boolean checkUDFInput(PhysicalOperator po) {
- if (po instanceof PORelationToExprProject) {
- return checkUDFInput(po.getInputs().get(0));
- }
-
- if (po instanceof POProject) {
- if(po.getInputs() == null )
- return true;
- else
- return checkUDFInput(po.getInputs().get(0));
- }
-
- if (po instanceof ConstantExpression) {
- return true;
- }
-
- if (po instanceof UnaryExpressionOperator) {
- return checkUDFInput(((UnaryExpressionOperator)po).getExpr());
- }
-
- if (po instanceof BinaryExpressionOperator) {
- return checkUDFInput(((BinaryExpressionOperator)po).getLhs()) ||
- checkUDFInput(((BinaryExpressionOperator)po).getRhs());
- }
-
- if (po instanceof POCast) {
- return checkUDFInput(po.getInputs().get(0));
- }
-
- if (po instanceof POBinCond) {
- return checkUDFInput(((POBinCond)po).getLhs()) &&
- checkUDFInput(((POBinCond)po).getRhs()) && checkUDFInput(((POBinCond)po).getCond());
- }
-
- if (po instanceof POSortedDistinct) {
- return true;
- }
-
- if (po instanceof POForEach) {
- List<PhysicalPlan> list = ((POForEach)po).getInputPlans();
- if (list.size() != 1) {
- return false;
- }
-
- PhysicalOperator p = list.get(0).getLeaves().get(0);
- if (checkUDFInput(p)) {
- return checkUDFInput(po.getInputs().get(0));
- }
- }
-
- return false;
+ AccumulatorOptimizerUtil.addAccumulator(mr.reducePlan);
}
}
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/ColumnInfo.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/ColumnInfo.java?rev=1598702&r1=1598701&r2=1598702&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/ColumnInfo.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/ColumnInfo.java Fri May 30 19:07:23 2014
@@ -28,7 +28,7 @@ public class ColumnInfo implements Clone
byte resultType;
int startCol = -1;
boolean isRangeProject = false;
-
+
public ColumnInfo(List<Integer> columns, byte type) {
this.columns = columns;
this.resultType = type;
@@ -44,7 +44,24 @@ public class ColumnInfo implements Clone
this.resultType = type;
this.isRangeProject = true;
}
-
+
+ public byte getResultType() {
+ return resultType;
+ }
+
+ public int getStartCol() {
+ return startCol;
+ }
+
+ public boolean isRangeProject() {
+ return isRangeProject;
+ }
+
+ public List<Integer> getColumns() {
+ return columns;
+ }
+
+ @Override
public String toString() {
String result;
if (isStar())
@@ -53,32 +70,33 @@ public class ColumnInfo implements Clone
result+=DataType.findTypeName(resultType);
return result;
}
-
+
private boolean isStar() {
return isRangeProject && startCol == 0;
}
+ @Override
public boolean equals(Object o2)
{
-
+
if (o2 == null || !(o2 instanceof ColumnInfo))
return false;
ColumnInfo c2 = (ColumnInfo)o2;
if (
isRangeProject == c2.isRangeProject &&
startCol == c2.startCol &&
- ((columns == null && c2.columns == null) ||
+ ((columns == null && c2.columns == null) ||
(columns != null && columns.equals(c2.columns)))
)
return true;
-
+
return false;
}
@Override
public int hashCode() {
return toString().hashCode();
}
-
+
@Override
public Object clone() throws CloneNotSupportedException{
ColumnInfo newColInfo = (ColumnInfo)super.clone();
@@ -87,6 +105,6 @@ public class ColumnInfo implements Clone
cols.addAll(this.columns);
newColInfo.columns = cols;
return newColInfo;
-
+
}
}