You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by pr...@apache.org on 2009/12/17 19:36:36 UTC
svn commit: r891828 [1/15] - in /hadoop/pig/branches/load-store-redesign: ./
contrib/
contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/
contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/evaluation/string/
contrib/z...
Author: pradeepkth
Date: Thu Dec 17 18:36:28 2009
New Revision: 891828
URL: http://svn.apache.org/viewvc?rev=891828&view=rev
Log:
svn merge -r882818:890596 http://svn.apache.org/repos/asf/hadoop/pig/trunk
Added:
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/ZebraOutputPartition.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/ZebraProjection.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/ZebraSchema.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/ZebraSortInfo.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/ZebraStorageHint.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/types/CsvZebraTupleOutput.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/types/ZebraTuple.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/types/ZebraTupleFactory.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/e2e/
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/e2e/pruning/
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/e2e/pruning/1.txt
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/e2e/pruning/2.txt
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/e2e/pruning/foreach.pig
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/e2e/pruning/foreach2.pig
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/e2e/pruning/j.pig
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/e2e/pruning/join.pig
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/e2e/pruning/join2.pig
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/e2e/pruning/m.pig
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/e2e/pruning/map1.pig
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/e2e/pruning/map2.pig
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/e2e/pruning/map3.pig
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/e2e/pruning/map4.pig
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/e2e/pruning/merge.pig
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/e2e/pruning/merge1.pig
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/e2e/pruning/merge2.pig
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/e2e/pruning/merge3.pig
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/e2e/pruning/merge_question.pig
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/e2e/pruning/merge_rh.pig
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/e2e/pruning/mforeach.pig
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/e2e/pruning/mforeach2.pig
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/e2e/pruning/mforeach3.pig
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/e2e/pruning/orderby.pig
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/e2e/pruning/orderby2.pig
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/e2e/pruning/union.pig
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/e2e/pruning/union1.pig
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/e2e/pruning/union2.pig
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/e2e/pruning/union3.pig
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnNameGroup.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestMultipleOutputs.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestMultipleOutputs2.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestMultipleOutputs2TypedApi.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestMultipleOutputs3.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestMultipleOutputs3TypedApi.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestMultipleOutputs4.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestMultipleOutputs4TypedApi.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestMultipleOutputsTypeApi.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestMultipleOutputsTypedApiNeg.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestTypedApi.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestTypedApi2.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestCogroup.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestMapTypePrune.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestMergeJoin2.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestMergeJoinEmpty.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestMergeJoinPrune.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestOrderPreserveMultiTable.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableLoaderPrune.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestSchemaAnonymousCollection.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestZebraTupleTostring.java
hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/images/zebra.jpg
hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/zebra_mapreduce.xml
hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/zebra_overview.xml
hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/zebra_pig.xml
hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/zebra_reference.xml
hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/zebra_stream.xml
hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/zebra_users.xml
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/ColumnPruner.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/optimizer/PruneColumns.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/plan/MapKeysInfo.java
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestPinOptions.java
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestPruneColumn.java
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/ExpectedResult4.pig
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/ExpectedResultCmdLnPriorDeclare.pig
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/ExpectedResultComment.pig
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/ExpectedResultDefault.pig
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/ExpectedResultMulDecs.pig
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/inputComment.pig
Modified:
hadoop/pig/branches/load-store-redesign/CHANGES.txt
hadoop/pig/branches/load-store-redesign/contrib/CHANGES.txt
hadoop/pig/branches/load-store-redesign/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/RegExLoader.java
hadoop/pig/branches/load-store-redesign/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/SequenceFileLoader.java
hadoop/pig/branches/load-store-redesign/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/evaluation/string/TestLookupInFiles.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/CHANGES.txt
hadoop/pig/branches/load-store-redesign/contrib/zebra/build-contrib.xml
hadoop/pig/branches/load-store-redesign/contrib/zebra/build.xml
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/io/BasicTable.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/io/BlockDistribution.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/io/ColumnGroup.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/BasicTableOutputFormat.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/TableInputFormat.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/TableRecordReader.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/TableLoader.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/TableStorer.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/schema/Schema.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/schema/SchemaParser.jjt
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/types/Partition.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/types/SortInfo.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/types/SubColumnExtraction.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/types/TableStorageParser.jjt
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/types/TypesUtils.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName2.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestSchema.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestOrderPreserveProjectionNegative.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestCheckin.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestColumnGroupName.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestColumnSecurity.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestSchemaMap.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestSchemaPrimitive.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageGrammar.java
hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageMisc1.java
hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/cookbook.xml
hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/piglatin_reference.xml
hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/piglatin_users.xml
hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/site.xml
hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/udf.xml
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/LoadFunc.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/PigServer.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/executionengine/PigSlice.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MultiQueryOptimizer.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapBase.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapReduce.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigRecordReader.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SampleOptimizer.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SliceWrapper.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/partitioners/SkewedPartitioner.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/LogToPhyTranslationVisitor.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ExpressionOperator.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/PODemux.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POFRJoin.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POLoad.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POMergeJoin.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POMultiQueryPackage.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPartitionRearrange.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POSort.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/util/MapRedUtil.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/local/executionengine/LocalPigLauncher.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/builtin/BinStorage.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/builtin/BinaryStorage.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/builtin/COUNT_STAR.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/builtin/PigStorage.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/data/InternalSortedBag.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/io/NullablePartitionWritable.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/io/PigLineRecordReader.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/LOCogroup.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/LOCross.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/LODistinct.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/LOFilter.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/LOForEach.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/LOJoin.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/LOLimit.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/LOLoad.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/LOSort.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/LOSplit.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/LOSplitOutput.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/LOStore.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/LOStream.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/LOUnion.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/LogicalOperator.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/RelationalOperator.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/optimizer/ImplicitSplitInserter.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/optimizer/LogicalOptimizer.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/optimizer/LogicalTransformer.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/optimizer/OpLimitOptimizer.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/optimizer/PushDownForeachFlatten.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/optimizer/PushUpFilter.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/optimizer/StreamOptimizer.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/optimizer/TypeCastInserter.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/schema/Schema.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/plan/OperatorPlan.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/plan/RequiredFields.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/plan/optimizer/Rule.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/plan/optimizer/RuleMatcher.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/plan/optimizer/Transformer.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/util/UDFContext.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/tools/parameters/ParamLoader.jj
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/tools/parameters/ParameterSubstitutionPreprocessor.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/tools/parameters/PigFileParser.jj
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/tools/parameters/PreprocessorContext.java
hadoop/pig/branches/load-store-redesign/test/commit-tests
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestAccumulator.java
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestForEachNestedPlan.java
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestInputOutputFileValidator.java
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestLogicalPlanBuilder.java
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestMRCompiler.java
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestMergeJoin.java
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestMultiQuery.java
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestOperatorPlan.java
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestPOCast.java
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestParamSubPreproc.java
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestPigLineRecordReader.java
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestPushUpFilter.java
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestRelevantFields.java
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestSkewedJoin.java
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestTypeChecking.java
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestTypeCheckingValidatorNoSchema.java
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/Util.java
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/DotFiles/optlimitplan10.dot
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/DotFiles/optlimitplan4.dot
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/DotFiles/optlimitplan5.dot
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/DotFiles/optlimitplan6.dot
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/DotFiles/optlimitplan7.dot
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/DotFiles/optlimitplan8.dot
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/DotFiles/optlimitplan9.dot
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/ExpectedResult.pig
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/ExpectedResult2.pig
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/GoldenFiles/MRC18.gld
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/input5.pig
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/inputDefault.pig
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/inputEscape.pig
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/inputSubstitutionWithinShellCommand.pig
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/inputThreeParams.pig
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/output1.pig
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/utils/TypeCheckingTestUtil.java
Modified: hadoop/pig/branches/load-store-redesign/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/CHANGES.txt?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/CHANGES.txt (original)
+++ hadoop/pig/branches/load-store-redesign/CHANGES.txt Thu Dec 17 18:36:28 2009
@@ -42,6 +42,24 @@
IMPROVEMENTS
+PIG-1106: FR join should not spill (ankit.modi via olgan)
+
+PIG-1147: Zebra Docs for Pig 0.6.0 (chandec via olgan)
+
+PIG-1129: Pig UDF doc: fieldsToRead function (chandec via olgan)
+
+PIG-978: MQ docs update (chandec via olgan)
+
+PIG-990: Provide a way to pin LogicalOperator Options (dvryaboy via gates)
+
+PIG-1103: refactoring of commit tests (olgan)
+
+PIG-1101: Allow arugment to limit to be long in addition to int (ashutoshc via
+ gates)
+
+PIG-872: use distributed cache for the replicated data set in FR join
+(sriranjan via olgan)
+
PIG-1053: Consider moving to Hadoop for local mode (ankit.modi via olgan)
PIG-1085: Pass JobConf and UDF specific configuration information to UDFs
@@ -51,15 +69,58 @@
BUG FIXES
+PIG-973: type resolution inconsistency (rding via olgan)
+
+PIG-1135: skewed join partitioner returns negative partition index (yinghe
+via olgan)
+
+PIG-1134: Skewed Join sampling job overwhelms the name node (sriranjan via
+olgan)
+
+PIG-1105: COUNT_STAR accumulate interface implementation cases failure
+(sriranjan via olgan)
+
+PIG-1118: expression with aggregate functions returning null, with accumulate
+interface (yinghe via olgan)
+
+PIG-1068: COGROUP fails with 'Type mismatch in key from map: expected
+ org.apache.pig.impl.io.NullableText, recieved
+ org.apache.pig.impl.io.NullableTuple' (rding via gates)
+
+PIG-1113: Diamond query optimization throws error in JOIN (rding via olgan)
+
+PIG-1116: Remove redundant map-reduce job for merge join (pradeepkth)
+
+PIG-1114: MultiQuery optimization throws error when merging 2 level spl (rding
+via olgan)
+
+PIG-1108: Incorrect map output key type in MultiQuery optimiza (rding via
+olgan)
+
+PIG-1022: optimizer pushes filter before the foreach that generates column
+used by filter (daijy via gates)
+
+PIG-1107: PigLineRecordReader bails out on an empty line for compressed data
+(ankit.modi via olgan)
+
+PIG-598: Parameter substitution ($PARAMETER) should not be performed in
+comments (thejas via olgan)
+
PIG-1064: Behaviour of COGROUP with and without schema when using "*" operator
(pradeepkth)
+PIG-1086: Nested sort by * throw exception (rding via daijy)
+
Release 0.6.0 - Unreleased
INCOMPATIBLE CHANGES
+PIG-922: Logical optimizer: push up project (daijy)
+
IMPROVEMENTS
+PIG-1084: Pig 0.6.0 Documentation improvements (chandec via olgan)
+
PIG-1089: Pig 0.6.0 Documentation (chandec via olgan)
PIG-958: Splitting output data on key field (ankur via pradeepkth)
@@ -155,6 +216,8 @@
OPTIMIZATIONS
+PIG-922: Logical optimizer: push up project (daijy)
+
BUG FIXES
PIG-1080: PigStorage may miss records when loading a file (rding via olgan)
@@ -224,6 +287,18 @@
PIG-1060: MultiQuery optimization throws error for multi-level splits (rding via daijy)
+PIG-1128: column pruning causing failure when foreach has user-specified
+schema (daijy)
+
+PIG-1127: Logical operator should contains individual copy of schema object
+(daijy)
+
+PIG-1133: UDFContext should be made available to LoadFunc.bindTo (daijy)
+
+PIG-1132: Column Pruner issues in dealing with unprunable loader (daijy)
+
+PIG-1142: Got NullPointerException merge join with pruning (daijy)
+
Release 0.5.0
INCOMPATIBLE CHANGES
Modified: hadoop/pig/branches/load-store-redesign/contrib/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/CHANGES.txt?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/contrib/CHANGES.txt (original)
+++ hadoop/pig/branches/load-store-redesign/contrib/CHANGES.txt Thu Dec 17 18:36:28 2009
@@ -1,3 +1,4 @@
+PIG-1126: updated fieldsToRead function (olgan)
PIG-1015: [piggybank] DateExtractor should take into account timezones
(dryaboy via olgan)
PIG-911: Added SequenceFileLoader (dryaboy via gates)
Modified: hadoop/pig/branches/load-store-redesign/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/RegExLoader.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/RegExLoader.java?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/RegExLoader.java (original)
+++ hadoop/pig/branches/load-store-redesign/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/RegExLoader.java Thu Dec 17 18:36:28 2009
@@ -31,6 +31,7 @@
import org.apache.pig.data.TupleFactory;
import org.apache.pig.impl.io.BufferedPositionedInputStream;
import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.impl.logicalLayer.FrontendException;
/**
* RegExLoader is an abstract class used to parse logs based on a regular expression.
@@ -110,6 +111,10 @@
public Schema determineSchema(String fileName, ExecType execType, DataStorage storage) throws IOException {
return null;
}
- public void fieldsToRead(Schema schema) {
+
+ @Override
+ public LoadFunc.RequiredFieldResponse fieldsToRead(LoadFunc.RequiredFieldList requiredFieldList) throws FrontendException {
+ return new LoadFunc.RequiredFieldResponse(false);
}
+
}
Modified: hadoop/pig/branches/load-store-redesign/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/SequenceFileLoader.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/SequenceFileLoader.java?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/SequenceFileLoader.java (original)
+++ hadoop/pig/branches/load-store-redesign/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/SequenceFileLoader.java Thu Dec 17 18:36:28 2009
@@ -38,6 +38,7 @@
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.serializer.SerializationFactory;
import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.ExecType;
import org.apache.pig.LoadFunc;
@@ -168,9 +169,8 @@
}
@Override
- public void fieldsToRead(Schema schema) {
- // not implemented
-
+ public LoadFunc.RequiredFieldResponse fieldsToRead(LoadFunc.RequiredFieldList requiredFieldList) throws FrontendException {
+ return new LoadFunc.RequiredFieldResponse(false);
}
@Override
Modified: hadoop/pig/branches/load-store-redesign/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/evaluation/string/TestLookupInFiles.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/evaluation/string/TestLookupInFiles.java?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/evaluation/string/TestLookupInFiles.java (original)
+++ hadoop/pig/branches/load-store-redesign/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/evaluation/string/TestLookupInFiles.java Thu Dec 17 18:36:28 2009
@@ -38,6 +38,7 @@
MiniCluster cluster = MiniCluster.buildCluster();
private PigServer pigServer;
+ @Override
public void setUp() throws Exception{
pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
}
@@ -70,7 +71,7 @@
FileSystem fs = FileSystem.get(ConfigurationUtil.toConfiguration(pigServer.getPigContext().getProperties()));
fs.copyFromLocalFile(new Path(lookupFile1.toString()), new Path("lookup1"));
fs.copyFromLocalFile(new Path(lookupFile1.toString()), new Path("lookup2"));
- pigServer.registerQuery("A = LOAD '" + Util.generateURI(tmpFile.toString()) + "' AS (key:chararray);");
+ pigServer.registerQuery("A = LOAD '" + Util.generateURI(tmpFile.toString(), pigServer.getPigContext()) + "' AS (key:chararray);");
pigServer.registerQuery("B = FOREACH A GENERATE org.apache.pig.piggybank.evaluation.string.LookupInFiles(key, 'lookup1', 'lookup2');");
Iterator<Tuple> iter = pigServer.openIterator("B");
Modified: hadoop/pig/branches/load-store-redesign/contrib/zebra/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/zebra/CHANGES.txt?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/contrib/zebra/CHANGES.txt (original)
+++ hadoop/pig/branches/load-store-redesign/contrib/zebra/CHANGES.txt Thu Dec 17 18:36:28 2009
@@ -3,11 +3,30 @@
Trunk (unreleased changes)
INCOMPATIBLE CHANGES
+
+ PIG-1122 Changed version number of pig dev core jar used in Zebra build from 0.6.0 to 0.7.0
+ to match Pig version number (yanz)
+
PIG-1099 Changed version number to be 0.7.0 to match Pig version number
change (yanz via gates)
IMPROVEMENTS
+ PIG-1125 Map/Reduce API Changes (Chao Wang via yanz)
+
+ PIG-1104 Streaming Support (Chao Wang via yanz)
+
+ PIG-1119 Support of "group" as a column name (Gaurav Jain via yanz)
+
+ PIG-653 Pig Projection Push Down (Gaurav Jain via yanz)
+
+ PIG-1111 Multiple Outputs Support (Gaurav Jain via yanz)
+
+ PIG-1098 Zebra Performance Optimizations (yanz via gates)
+
+ PIG-1074 Zebra store function should allow '::' in column names in output
+ schema (yanz via gates)
+
PIG-1077 Support record(row)-based file split in Zebra's
TableInputFormat (chaow via gates)
@@ -29,6 +48,13 @@
BUG FIXES
+ PIG-1145: Merge Join on Large Table throws an EOF exception (yanz)
+
+ PIG-1095: Schema support of anonymous fields in COLECTION fails (yanz via
+ gates)
+
+ PIG-1078: merge join with empty table failed (yanz via gates)
+
PIG-1091: Exception when load with projection of map keys on a map column
that is not map split (yanz via gates).
Modified: hadoop/pig/branches/load-store-redesign/contrib/zebra/build-contrib.xml
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/zebra/build-contrib.xml?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/contrib/zebra/build-contrib.xml (original)
+++ hadoop/pig/branches/load-store-redesign/contrib/zebra/build-contrib.xml Thu Dec 17 18:36:28 2009
@@ -53,7 +53,7 @@
<property name="build.examples" location="${build.dir}/examples"/>
<property name="pig.log.dir" location="${build.dir}/test/logs"/>
<property name="hadoop.jarfile" value="hadoop20.jar" />
- <property name="pig.jarfile" value="pig-0.6.0-dev-core.jar" />
+ <property name="pig.jarfile" value="pig-0.7.0-dev-core.jar" />
<property name="hbase.jarfile" value="hbase-0.18.1.jar" />
<property name="hbase.test.jarfile" value="hbase-0.18.1-test.jar" />
Modified: hadoop/pig/branches/load-store-redesign/contrib/zebra/build.xml
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/zebra/build.xml?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/contrib/zebra/build.xml (original)
+++ hadoop/pig/branches/load-store-redesign/contrib/zebra/build.xml Thu Dec 17 18:36:28 2009
@@ -116,7 +116,8 @@
<sysproperty key="build.test" value="${build.test}"/>
<sysproperty key="contrib.name" value="${name}"/>
<sysproperty key="hadoop.log.dir" value="${pig.log.dir}"/>
-
+ <sysproperty key="whichCluster" value="miniCluster"/>
+
<!-- requires fork=yes for:
relative File paths to use the specified user.dir
classpath to use build/contrib/*.jar
@@ -160,7 +161,7 @@
<sysproperty key="build.test" value="${build.test}"/>
<sysproperty key="contrib.name" value="${name}"/>
<sysproperty key="hadoop.log.dir" value="${pig.log.dir}"/>
-
+ <sysproperty key="whichCluster" value="miniCluster"/>
<!-- requires fork=yes for:
relative File paths to use the specified user.dir
classpath to use build/contrib/*.jar
@@ -182,6 +183,7 @@
<not>
<filename name="**/TestColumnSecurity.java"/>
</not>
+
</fileset>
</batchtest>
</junit>
Modified: hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/io/BasicTable.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/io/BasicTable.java?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/io/BasicTable.java (original)
+++ hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/io/BasicTable.java Thu Dec 17 18:36:28 2009
@@ -52,6 +52,7 @@
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.zebra.io.ColumnGroup.Reader.CGRangeSplit;
import org.apache.hadoop.zebra.io.ColumnGroup.Reader.CGRowSplit;
+import org.apache.hadoop.zebra.io.ColumnGroup.Reader.CGScanner;
import org.apache.hadoop.zebra.types.CGSchema;
import org.apache.hadoop.zebra.parser.ParseException;
import org.apache.hadoop.zebra.types.Partition;
@@ -442,7 +443,6 @@
* @param split The row-based split. <i>Cannot</i> be null.
* @return An object that conveys how blocks fall into the split are
* distributed across hosts.
- * @see #rowSplit(int)
*/
public BlockDistribution getBlockDistribution(RowSplit split)
throws IOException {
@@ -874,7 +874,7 @@
*/
private class BTScanner implements TableScanner {
private Projection schema;
- private TableScanner[] cgScanners;
+ private CGScanner[] cgScanners;
private int opCount = 0;
Random random = new Random(System.nanoTime());
// checking for consistency once every 1000 times.
@@ -936,7 +936,7 @@
}
// Helper function for initialization.
- private TableScanner createCGScanner(int cgIndex, CGRowSplit cgRowSplit,
+ private CGScanner createCGScanner(int cgIndex, CGRowSplit cgRowSplit,
RangeSplit rangeSplit,
BytesWritable beginKey,
BytesWritable endKey)
@@ -972,7 +972,7 @@
try {
schema = partition.getProjection();
- cgScanners = new TableScanner[colGroups.length];
+ cgScanners = new CGScanner[colGroups.length];
for (int i = 0; i < colGroups.length; ++i) {
if (!isCGDeleted(i) && partition.isCGNeeded(i))
{
@@ -1020,7 +1020,7 @@
for (int nx = 0; nx < cgScanners.length; nx++) {
if (cgScanners[nx] != null)
{
- cur = cgScanners[nx].advance();
+ cur = cgScanners[nx].advanceCG();
if (!firstAdvance) {
if (cur != first) {
throw new IOException(
@@ -1038,9 +1038,6 @@
@Override
public boolean atEnd() throws IOException {
- if (cgScanners.length == 0) {
- return true;
- }
boolean ret = true;
int i;
for (i = 0; i < cgScanners.length; i++)
@@ -1077,16 +1074,12 @@
@Override
public void getKey(BytesWritable key) throws IOException {
- if (cgScanners.length == 0) {
- return;
- }
-
int i;
for (i = 0; i < cgScanners.length; i++)
{
if (cgScanners[i] != null)
{
- cgScanners[i].getKey(key);
+ cgScanners[i].getCGKey(key);
break;
}
}
@@ -1104,7 +1097,7 @@
if (cgScanners[index] != null)
{
BytesWritable key2 = new BytesWritable();
- cgScanners[index].getKey(key2);
+ cgScanners[index].getCGKey(key2);
if (key.equals(key2)) {
return;
}
@@ -1129,7 +1122,7 @@
{
if (cgTuples[i] == null)
throw new AssertionError("cgTuples["+i+"] is null");
- cgScanners[i].getValue(cgTuples[i]);
+ cgScanners[i].getCGValue(cgTuples[i]);
}
}
}
@@ -1144,12 +1137,12 @@
@Override
public boolean seekTo(BytesWritable key) throws IOException {
- boolean first = false, cur;
+ boolean first = false, cur, firstset = false;
for (int nx = 0; nx < cgScanners.length; nx++) {
if (cgScanners[nx] == null)
continue;
cur = cgScanners[nx].seekTo(key);
- if (nx != 0) {
+ if (firstset) {
if (cur != first) {
throw new IOException(
"seekTo() failed: Column Groups are not evenly positioned.");
@@ -1157,6 +1150,7 @@
}
else {
first = cur;
+ firstset = true;
}
}
return first;
Modified: hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/io/BlockDistribution.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/io/BlockDistribution.java?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/io/BlockDistribution.java (original)
+++ hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/io/BlockDistribution.java Thu Dec 17 18:36:28 2009
@@ -136,8 +136,8 @@
@Override
public int compare(Entry<String, Long> o1, Entry<String, Long> o2) {
long diff = o1.getValue() - o2.getValue();
- if (diff < 0) return -1;
- if (diff > 0) return 1;
+ if (diff < 0) return 1;
+ if (diff > 0) return -1;
return 0;
}
});
Modified: hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/io/ColumnGroup.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/io/ColumnGroup.java?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/io/ColumnGroup.java (original)
+++ hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/io/ColumnGroup.java Thu Dec 17 18:36:28 2009
@@ -381,7 +381,7 @@
* @return A scanner object.
* @throws IOException
*/
- public synchronized TableScanner getScanner(BytesWritable beginKey,
+ public synchronized CGScanner getScanner(BytesWritable beginKey,
BytesWritable endKey, boolean closeReader) throws IOException,
ParseException {
if (closed) {
@@ -422,7 +422,7 @@
* @return A scanner object.
* @throws IOException
*/
- public synchronized TableScanner getScanner(CGRangeSplit split,
+ public synchronized CGScanner getScanner(CGRangeSplit split,
boolean closeReader) throws IOException, ParseException {
if (closed) {
throw new EOFException("Reader already closed");
@@ -449,7 +449,7 @@
* @param rowSplit specifies part index, start row, and end row.
* @return A scanner object.
*/
- public synchronized TableScanner getScanner(boolean closeReader,
+ public synchronized CGScanner getScanner(boolean closeReader,
CGRowSplit rowSplit)
throws IOException, ParseException {
if (closed) {
@@ -509,14 +509,16 @@
}
BlockDistribution ret = new BlockDistribution();
- CGIndexEntry entry = cgindex.get(split.fileIndex);
- FileStatus tfileStatus = fs.getFileStatus(new Path(path, entry.getName()));
-
- BlockLocation[] locations = fs.getFileBlockLocations(tfileStatus, split.startByte, split.numBytes);
- for (BlockLocation l : locations) {
- ret.add(l);
- }
-
+ if (split.fileIndex >= 0)
+ {
+ CGIndexEntry entry = cgindex.get(split.fileIndex);
+ FileStatus tfileStatus = fs.getFileStatus(new Path(path, entry.getName()));
+
+ BlockLocation[] locations = fs.getFileBlockLocations(tfileStatus, split.startByte, split.numBytes);
+ for (BlockLocation l : locations) {
+ ret.add(l);
+ }
+ }
return ret;
}
@@ -530,6 +532,9 @@
void fillRowSplit(CGRowSplit rowSplit, long startOffset, long length)
throws IOException {
+ if (rowSplit.fileIndex < 0)
+ return;
+
Path tfPath = new Path(path, cgindex.get(rowSplit.fileIndex).getName());
FileStatus tfile = fs.getFileStatus(tfPath);
@@ -748,7 +753,6 @@
long length = lengths[i];
Path path = paths[i];
int idx = cgindex.getFileIndex(path);
-
lst.add(new CGRowSplit(idx, start, length));
}
@@ -1009,22 +1013,34 @@
@Override
public void getKey(BytesWritable key) throws IOException {
- if (atEnd()) {
- throw new EOFException("No more key-value to read");
+ if (atEnd()) {
+ throw new EOFException("No more key-value to read");
+ }
+ scanners[current].getKey(key);
+ }
+
+ @Override
+ public void getValue(Tuple row) throws IOException {
+ if (atEnd()) {
+ throw new EOFException("No more key-value to read");
+ }
+ try {
+ scanners[current].getValue(row);
+ } catch (ParseException e) {
+ throw new IOException("Invalid Projection: "+e.getMessage());
+ }
}
+
+ public void getCGKey(BytesWritable key) throws IOException {
scanners[current].getKey(key);
}
- @Override
- public void getValue(Tuple row) throws IOException {
- if (atEnd()) {
- throw new EOFException("No more key-value to read");
- }
+ public void getCGValue(Tuple row) throws IOException {
try {
- scanners[current].getValue(row);
- } catch (ParseException e) {
- throw new IOException("Invalid Projection: "+e.getMessage());
- }
+ scanners[current].getValue(row);
+ } catch (ParseException e) {
+ throw new IOException("Invalid Projection: "+e.getMessage());
+ }
}
@Override
@@ -1038,18 +1054,29 @@
@Override
public boolean advance() throws IOException {
- if (atEnd()) {
- return false;
+ if (atEnd()) {
+ return false;
+ }
+ scanners[current].advance();
+ if (scanners[current].atEnd()) {
+ ++current;
+ if (!atEnd()) {
+ scanners[current].rewind();
+ }
+ }
+ return true;
}
- scanners[current].advance();
- if (scanners[current].atEnd()) {
- ++current;
- if (!atEnd()) {
- scanners[current].rewind();
+
+ public boolean advanceCG() throws IOException {
+ scanners[current].advance();
+ if (scanners[current].atEnd()) {
+ ++current;
+ if (!atEnd()) {
+ scanners[current].rewind();
+ }
}
+ return true;
}
- return true;
- }
@Override
public boolean atEnd() throws IOException {
@@ -1061,10 +1088,14 @@
if (!isSorted()) {
throw new IOException("Cannot seek in unsorted Column Gruop");
}
+ if (atEnd())
+ {
+ return false;
+ }
int index =
cgindex.lowerBound(new ByteArray(key.get(), 0, key.getSize()),
comparator);
- if (index > endIndex) {
+ if (index >= endIndex) {
seekToEnd();
return false;
}
@@ -1764,6 +1795,8 @@
int getFileIndex(Path path) throws IOException {
String filename = path.getName();
+ if (index.isEmpty())
+ return -1;
for (CGIndexEntry cgie : index) {
if (cgie.getName().equals(filename)) {
return cgie.getIndex();
Modified: hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/BasicTableOutputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/BasicTableOutputFormat.java?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/BasicTableOutputFormat.java (original)
+++ hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/BasicTableOutputFormat.java Thu Dec 17 18:36:28 2009
@@ -17,11 +17,13 @@
package org.apache.hadoop.zebra.mapred;
import java.io.IOException;
-
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.RawComparator;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputFormat;
import org.apache.hadoop.mapred.RecordWriter;
import org.apache.hadoop.mapred.Reporter;
@@ -32,8 +34,10 @@
import org.apache.hadoop.zebra.types.Partition;
import org.apache.hadoop.zebra.types.SortInfo;
import org.apache.hadoop.zebra.schema.Schema;
+import org.apache.hadoop.zebra.tfile.TFile;
import org.apache.pig.data.Tuple;
import org.apache.hadoop.zebra.pig.comparator.*;
+import org.apache.hadoop.util.ReflectionUtils;
/**
@@ -54,11 +58,38 @@
* The above code does the following things:
* <UL>
* <LI>Set the output format class to BasicTableOutputFormat.
- * <LI>Set the path to the BasicTable to be created.
+ * <LI>Set the single path to the BasicTable to be created.
* <LI>Set the schema of the BasicTable to be created. In this case, the
* to-be-created BasicTable contains three columns with names "Name", "Age",
* "Salary", "BonusPct".
* </UL>
+ *
+ * To create multiple output paths. ZebraOutputPartitoner interface needs to be implemented
+ * <pre>
+ * String multiLocs = "commaSeparatedPaths"
+ * jobConf.setOutputFormat(BasicTableOutputFormat.class);
+ * BasicTableOutputFormat.setMultipleOutputPaths(jobConf, multiLocs);
+ * jobConf.setOutputFormat(BasicTableOutputFormat.class);
+ * BasicTableOutputFormat.setSchema(jobConf, "Name, Age, Salary, BonusPct");
+ * BasicTableOutputFormat.setZebraOutputPartitionClass(
+ * jobConf, MultipleOutputsTest.OutputPartitionerClass.class);
+ * </pre>
+ *
+ *
+ * The user ZebraOutputPartitionClass should like this
+ *
+ * <pre>
+ *
+ * static class OutputPartitionerClass implements ZebraOutputPartition {
+ * @Override
+ * public int getOutputPartition(BytesWritable key, Tuple value) {
+ *
+ * return someIndexInOutputParitionlist0;
+ * }
+ *
+ * </pre>
+ *
+ *
* The user Reducer code (or similarly Mapper code if it is a Map-only job)
* should look like the following:
*
@@ -113,6 +144,7 @@
public class BasicTableOutputFormat implements
OutputFormat<BytesWritable, Tuple> {
private static final String OUTPUT_PATH = "mapred.lib.table.output.dir";
+ public static final String MULTI_OUTPUT_PATH = "mapred.lib.table.multi.output.dirs";
private static final String OUTPUT_SCHEMA = "mapred.lib.table.output.schema";
private static final String OUTPUT_STORAGEHINT =
"mapred.lib.table.output.storagehint";
@@ -120,7 +152,142 @@
"mapred.lib.table.output.sortcolumns";
private static final String OUTPUT_COMPARATOR =
"mapred.lib.table.output.comparator";
+ static final String IS_MULTI = "multi";
+ public static final String ZEBRA_OUTPUT_PARTITIONER_CLASS = "zebra.output.partitioner.class";
+
+
+ /**
+ * Set the multiple output paths of the BasicTable in JobConf
+ *
+ * @param conf
+ * The JobConf object.
+ * @param commaSeparatedLocations
+ * The comma separated output paths to the tables.
+ * The path must either not existent, or must be an empty directory.
+ * @param theClass
+ * Zebra output partitoner class
+ */
+
+ public static void setMultipleOutputs(JobConf conf, String commaSeparatedLocations, Class<? extends ZebraOutputPartition> theClass)
+ throws IOException {
+
+ conf.set(MULTI_OUTPUT_PATH, commaSeparatedLocations);
+
+ if(conf.getBoolean(IS_MULTI, true) == false) {
+ throw new IllegalArgumentException("Job has been setup as single output path");
+ }
+ conf.setBoolean(IS_MULTI, true);
+ setZebraOutputPartitionClass(conf, theClass);
+
+ }
+
+ /**
+ * Set the multiple output paths of the BasicTable in JobConf
+ *
+ * @param conf
+ * The JobConf object.
+ * @param paths
+ * The list of paths
+ * The path must either not existent, or must be an empty directory.
+ * @param theClass
+ * Zebra output partitioner class
+ */
+
+ public static void setMultipleOutputs(JobConf conf, Class<? extends ZebraOutputPartition> theClass, Path... paths)
+ throws IOException {
+ FileSystem fs = FileSystem.get(conf);
+ Path path = paths[0].makeQualified(fs);
+ StringBuffer str = new StringBuffer(StringUtils.escapeString(path.toString()));
+ for(int i = 1; i < paths.length;i++) {
+ str.append(StringUtils.COMMA_STR);
+ path = paths[i].makeQualified(fs);
+ str.append(StringUtils.escapeString(path.toString()));
+ }
+ conf.set(MULTI_OUTPUT_PATH, str.toString());
+
+ if(conf.getBoolean(IS_MULTI, true) == false) {
+ throw new IllegalArgumentException("Job has been setup as single output path");
+ }
+ conf.setBoolean(IS_MULTI, true);
+ setZebraOutputPartitionClass(conf, theClass);
+
+ }
+
+
+ /**
+ * Set the multiple output paths of the BasicTable in JobConf
+ *
+ * @param conf
+ * The JobConf object.
+ * @return path
+ * The comma separated output paths to the tables.
+ * The path must either not existent, or must be an empty directory.
+ */
+
+ public static Path[] getOutputPaths(JobConf conf)
+ throws IOException {
+
+ Path[] result;
+ String paths = conf.get(MULTI_OUTPUT_PATH);
+ String path = conf.get(OUTPUT_PATH);
+
+ if(paths != null && path != null) {
+ throw new IllegalArgumentException("Illegal output paths specs. Both multi and single output locs are set");
+ }
+
+ if(conf.getBoolean(IS_MULTI, false) == true) {
+ if (paths == null || paths.equals("")) {
+ throw new IllegalArgumentException("Illegal multi output paths");
+ }
+ String [] list = StringUtils.split(paths);
+ result = new Path[list.length];
+ for (int i = 0; i < list.length; i++) {
+ result[i] = new Path(StringUtils.unEscapeString(list[i]));
+ }
+ } else {
+ if (path == null || path.equals("")) {
+ throw new IllegalArgumentException("Cannot find output path");
+ }
+ result = new Path[1];
+ result[0] = new Path(path);
+ }
+
+ return result;
+
+ }
+
+
+ private static void setZebraOutputPartitionClass(
+ JobConf conf, Class<? extends ZebraOutputPartition> theClass) throws IOException {
+ if (!ZebraOutputPartition.class.isAssignableFrom(theClass))
+ throw new IOException(theClass+" not "+ZebraOutputPartition.class.getName());
+ conf.set(ZEBRA_OUTPUT_PARTITIONER_CLASS, theClass.getName());
+ }
+
+
+ public static Class<? extends ZebraOutputPartition> getZebraOutputPartitionClass(JobConf conf) throws IOException {
+
+ Class<?> theClass;
+ String valueString = conf.get(ZEBRA_OUTPUT_PARTITIONER_CLASS);
+ if (valueString == null)
+ throw new IOException("zebra output partitioner class not found");
+ try {
+ theClass = conf.getClassByName(valueString);
+ } catch (ClassNotFoundException e) {
+ throw new IOException(e);
+ }
+ if (theClass != null && !ZebraOutputPartition.class.isAssignableFrom(theClass))
+ throw new IOException(theClass+" not "+ZebraOutputPartition.class.getName());
+ else if (theClass != null)
+ return theClass.asSubclass(ZebraOutputPartition.class);
+ else
+ return null;
+
+ }
+
+
+
/**
* Set the output path of the BasicTable in JobConf
*
@@ -132,6 +299,11 @@
*/
public static void setOutputPath(JobConf conf, Path path) {
conf.set(OUTPUT_PATH, path.toString());
+ if(conf.getBoolean(IS_MULTI, false) == true) {
+ throw new IllegalArgumentException("Job has been setup as multi output paths");
+ }
+ conf.setBoolean(IS_MULTI, false);
+
}
/**
@@ -156,13 +328,12 @@
* implementation, the schema string is simply a comma separated list
* of column names, such as "Col1, Col2, Col3".
*
- * @see Schema#Schema(String)
+ * @deprecated Use {@link #setStorageInfo(JobConf, ZebraSchema, ZebraStorageHint, ZebraSortInfo)} instead.
*/
- public static void setSchema(JobConf conf, String schema)
- throws ParseException {
+ public static void setSchema(JobConf conf, String schema) {
conf.set(OUTPUT_SCHEMA, Schema.normalize(schema));
}
-
+
/**
* Get the table schema in JobConf.
*
@@ -176,7 +347,7 @@
if (schema == null) {
return null;
}
- schema = schema.replaceAll(";", ",");
+ //schema = schema.replaceAll(";", ",");
return new Schema(schema);
}
@@ -246,21 +417,20 @@
* The storage hint of the BasicTable to be created. The format would
* be like "[f1, f2.subfld]; [f3, f4]".
*
- * @see Schema#Schema(String)
+ * @deprecated Use {@link #setStorageInfo(JobConf, ZebraSchema, ZebraStorageHint, ZebraSortInfo)} instead.
*/
- public static void setStorageHint(JobConf conf, String storehint)
- throws ParseException, IOException {
+ public static void setStorageHint(JobConf conf, String storehint) throws ParseException, IOException {
String schema = conf.get(OUTPUT_SCHEMA);
if (schema == null)
throw new ParseException("Schema has not been set");
// for sanity check purpose only
- Partition partition = new Partition(schema, storehint, null);
+ new Partition(schema, storehint, null);
conf.set(OUTPUT_STORAGEHINT, storehint);
}
-
+
/**
* Get the table storage hint in JobConf.
*
@@ -284,13 +454,15 @@
* @param sortColumns
* Comma-separated sort column names
*
- * @param comparator
+ * @param comparatorClass
* comparator class name; null for default
*
+ * @deprecated Use {@link #setStorageInfo(JobConf, ZebraSchema, ZebraStorageHint, ZebraSortInfo)} instead.
*/
- public static void setSortInfo(JobConf conf, String sortColumns, String comparator) {
+ public static void setSortInfo(JobConf conf, String sortColumns, Class<? extends RawComparator<Object>> comparatorClass) {
conf.set(OUTPUT_SORTCOLUMNS, sortColumns);
- conf.set(OUTPUT_COMPARATOR, comparator);
+ if (comparatorClass != null)
+ conf.set(OUTPUT_COMPARATOR, TFile.COMPARATOR_JCLASS+comparatorClass.getName());
}
/**
@@ -301,9 +473,89 @@
*
* @param sortColumns
* Comma-separated sort column names
+ *
+ * @deprecated Use {@link #setStorageInfo(JobConf, ZebraSchema, ZebraStorageHint, ZebraSortInfo)} instead.
*/
public static void setSortInfo(JobConf conf, String sortColumns) {
conf.set(OUTPUT_SORTCOLUMNS, sortColumns);
+ }
+
+ /**
+ * Set the table storage info including ZebraSchema,
+ *
+ * @param conf
+ * The JobConf object.
+ *
+ * @param zSchema The ZebraSchema object containing schema information.
+ *
+ * @param zStorageHint The ZebraStorageHint object containing storage hint information.
+ *
+ * @param zSortInfo The ZebraSortInfo object containing sorting information.
+ *
+ */
+ public static void setStorageInfo(JobConf conf, ZebraSchema zSchema, ZebraStorageHint zStorageHint, ZebraSortInfo zSortInfo)
+ throws ParseException, IOException {
+ String schemaStr = null;
+ String storageHintStr = null;
+
+ /* validity check on schema*/
+ if (zSchema == null) {
+ throw new IllegalArgumentException("ZebraSchema object cannot be null.");
+ } else {
+ schemaStr = zSchema.toString();
+ }
+
+ Schema schema = null;
+ try {
+ schema = new Schema(schemaStr);
+ } catch (ParseException e) {
+ throw new ParseException("[" + zSchema + "] " + " is not a valid schema string: " + e.getMessage());
+ }
+
+ /* validity check on storage hint*/
+ if (zStorageHint == null) {
+ storageHintStr = "";
+ } else {
+ storageHintStr = zStorageHint.toString();
+ }
+
+ try {
+ new Partition(schemaStr, storageHintStr, null);
+ } catch (ParseException e) {
+ throw new ParseException("[" + zStorageHint + "] " + " is not a valid storage hint string: " + e.getMessage() );
+ } catch (IOException e) {
+ throw new ParseException("[" + zStorageHint + "] " + " is not a valid storage hint string: " + e.getMessage() );
+ }
+
+ conf.set(OUTPUT_SCHEMA, schemaStr);
+ conf.set(OUTPUT_STORAGEHINT, storageHintStr);
+
+ /* validity check on sort info if user specifies it */
+ if (zSortInfo != null) {
+ String sortColumnsStr = zSortInfo.getSortColumns();
+ String comparatorStr = zSortInfo.getComparator();
+
+ /* Check existence of comparable class if user specifies it */
+ if (comparatorStr != null && comparatorStr != "") {
+ try {
+ conf.getClassByName(comparatorStr.substring(TFile.COMPARATOR_JCLASS.length()).trim());
+ } catch (ClassNotFoundException e) {
+ throw new IOException("comparator Class cannot be found : " + e.getMessage());
+ }
+ }
+
+ try {
+ SortInfo.parse(sortColumnsStr, schema, comparatorStr);
+ } catch (IOException e) {
+ throw new IOException("[" + sortColumnsStr + " + " + comparatorStr + "] "
+ + "is not a valid sort configuration: " + e.getMessage());
+ }
+
+ if (sortColumnsStr != null)
+ conf.set(OUTPUT_SORTCOLUMNS, sortColumnsStr);
+ if (comparatorStr != null)
+ conf.set(OUTPUT_COMPARATOR, comparatorStr);
+ }
}
/**
@@ -353,12 +605,14 @@
* @return The output BasicTable.Writer object.
* @throws IOException
*/
- private static BasicTable.Writer getOutput(JobConf conf) throws IOException {
- String path = conf.get(OUTPUT_PATH);
- if (path == null) {
- throw new IllegalArgumentException("Cannot find output path");
- }
- return new BasicTable.Writer(new Path(path), conf);
+ private static BasicTable.Writer[] getOutput(JobConf conf) throws IOException {
+ Path[] paths = getOutputPaths(conf);
+ BasicTable.Writer[] writers = new BasicTable.Writer[paths.length];
+ for(int i = 0; i < paths.length; i++) {
+ writers[i] = new BasicTable.Writer(paths[i], conf);
+ }
+
+ return writers;
}
/**
@@ -371,10 +625,6 @@
@Override
public void checkOutputSpecs(FileSystem ignored, JobConf conf)
throws IOException {
- String path = conf.get(OUTPUT_PATH);
- if (path == null) {
- throw new IllegalArgumentException("Cannot find output path");
- }
String schema = conf.get(OUTPUT_SCHEMA);
if (schema == null) {
@@ -389,10 +639,14 @@
catch (ParseException e) {
throw new IOException(e);
}
- BasicTable.Writer writer =
- new BasicTable.Writer(new Path(path), schema, storehint, sortColumns, comparator, conf);
-
- writer.finish();
+
+ Path [] paths = getOutputPaths(conf);
+
+ for(int i = 0; i < paths.length; ++i) {
+ BasicTable.Writer writer =
+ new BasicTable.Writer(paths[i], schema, storehint, sortColumns, comparator, conf);
+ writer.finish();
+ }
}
/**
@@ -403,10 +657,6 @@
public RecordWriter<BytesWritable, Tuple> getRecordWriter(FileSystem ignored,
JobConf conf, String name, Progressable progress) throws IOException {
String path = conf.get(OUTPUT_PATH);
- if (path == null) {
- throw new IllegalArgumentException("Cannot find output path");
- }
-
return new TableRecordWriter(path, name, conf, progress);
}
@@ -419,8 +669,12 @@
* @throws IOException
*/
public static void close(JobConf conf) throws IOException {
- BasicTable.Writer table = getOutput(conf);
- table.close();
+ BasicTable.Writer tables[] = getOutput(conf);
+
+ for(int i =0; i < tables.length; ++i) {
+ tables[i].close();
+ }
+
}
}
@@ -428,25 +682,51 @@
* Adaptor class for BasicTable RecordWriter.
*/
class TableRecordWriter implements RecordWriter<BytesWritable, Tuple> {
- private final TableInserter inserter;
+ private final TableInserter inserter[];
private final Progressable progress;
+ private org.apache.hadoop.zebra.mapred.ZebraOutputPartition op = null;
+
public TableRecordWriter(String path, String name, JobConf conf,
Progressable progress) throws IOException {
- BasicTable.Writer writer = new BasicTable.Writer(new Path(path), conf);
- this.inserter = writer.getInserter(name, true);
+
+ if(conf.getBoolean(BasicTableOutputFormat.IS_MULTI, false) == true) {
+ op = (org.apache.hadoop.zebra.mapred.ZebraOutputPartition)
+ ReflectionUtils.newInstance(BasicTableOutputFormat.getZebraOutputPartitionClass(conf), conf);
+
+ }
+ Path [] paths = BasicTableOutputFormat.getOutputPaths(conf);
+ inserter = new TableInserter[paths.length];
+ for(int i = 0; i < paths.length; ++i) {
+ BasicTable.Writer writer =
+ new BasicTable.Writer(paths[i], conf);
+ this.inserter[i] = writer.getInserter(name, true);
+ }
+
this.progress = progress;
}
@Override
public void close(Reporter reporter) throws IOException {
- inserter.close();
+ for(int i = 0; i < this.inserter.length; ++i) {
+ inserter[i].close();
+ }
reporter.progress();
}
@Override
public void write(BytesWritable key, Tuple value) throws IOException {
- inserter.insert(key, value);
- progress.progress();
+
+ if(op != null ) {
+ int idx = op.getOutputPartition(key, value);
+ if(idx < 0 || (idx >= inserter.length)) {
+ throw new IllegalArgumentException("index returned by getOutputPartition is out of range");
+ }
+ inserter[idx].insert(key, value);
+ } else {
+ inserter[0].insert(key, value);
+ }
+ progress.progress();
+
}
}
Modified: hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/TableInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/TableInputFormat.java?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/TableInputFormat.java (original)
+++ hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/TableInputFormat.java Thu Dec 17 18:36:28 2009
@@ -31,6 +31,7 @@
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.io.RawComparator;
import org.apache.hadoop.zebra.tfile.RawComparable;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.FileSplit;
@@ -51,6 +52,7 @@
import org.apache.hadoop.zebra.types.Projection;
import org.apache.hadoop.zebra.schema.Schema;
import org.apache.hadoop.zebra.types.SortInfo;
+import org.apache.hadoop.zebra.tfile.TFile;
import org.apache.pig.data.DataBag;
import org.apache.pig.data.DataByteArray;
import org.apache.pig.data.Tuple;
@@ -184,6 +186,15 @@
static TableExpr getInputExpr(JobConf conf) throws IOException {
String expr = conf.get(INPUT_EXPR);
if (expr == null) {
+ // try setting from input path
+ Path[] paths = FileInputFormat.getInputPaths(conf);
+ if (paths != null) {
+ setInputPaths(conf, paths);
+ }
+ expr = conf.get(INPUT_EXPR);
+ }
+
+ if (expr == null) {
throw new IllegalArgumentException("Input expression not defined.");
}
StringReader in = new StringReader(expr);
@@ -195,13 +206,14 @@
*
* @param conf
* JobConf object.
+ *
*/
public static Schema getSchema(JobConf conf) throws IOException
{
TableExpr expr = getInputExpr(conf);
return expr.getSchema(conf);
- }
-
+ }
+
/**
* Set the input projection in the JobConf object.
*
@@ -211,7 +223,7 @@
* A common separated list of column names. If we want select all
* columns, pass projection==null. The syntax of the projection
* conforms to the {@link Schema} string.
- * @see Schema#Schema(String)
+ * @deprecated Use {@link #setProjection(JobConf, ZebraProjection)} instead.
*/
public static void setProjection(JobConf conf, String projection) throws ParseException {
conf.set(INPUT_PROJ, Schema.normalize(projection));
@@ -220,6 +232,37 @@
if (Projection.getVirtualColumnIndices(projection) != null && !getSorted(conf))
throw new ParseException("The source_table virtual column is only availabe for sorted table unions.");
}
+
+ /**
+ * Set the input projection in the JobConf object.
+ *
+ * @param conf
+ * JobConf object.
+ * @param projection
+ * A common separated list of column names. If we want select all
+ * columns, pass projection==null. The syntax of the projection
+ * conforms to the {@link Schema} string.
+ *
+ */
+ public static void setProjection(JobConf conf, ZebraProjection projection) throws ParseException {
+ /* validity check on projection */
+ Schema schema = null;
+ String normalizedProjectionString = Schema.normalize(projection.toString());
+ try {
+ schema = getSchema(conf);
+ new org.apache.hadoop.zebra.types.Projection(schema, normalizedProjectionString);
+ } catch (ParseException e) {
+ throw new ParseException("[" + projection + "] " + "is not a valid Zebra projection string " + e.getMessage());
+ } catch (IOException e) {
+ throw new ParseException("[" + projection + "] " + "is not a valid Zebra projection string " + e.getMessage());
+ }
+
+ conf.set(INPUT_PROJ, normalizedProjectionString);
+
+ // virtual source_table columns require sorted table
+ if (Projection.getVirtualColumnIndices(projection.toString()) != null && !getSorted(conf))
+ throw new ParseException("The source_table virtual column is only availabe for sorted table unions.");
+ }
/**
* Get the projection from the JobConf
@@ -231,6 +274,7 @@
* when this method is called in Mapper code, the projection must
* already be known.
* @throws IOException
+ *
*/
public static String getProjection(JobConf conf) throws IOException, ParseException {
String strProj = conf.get(INPUT_PROJ);
@@ -242,7 +286,7 @@
}
return null;
}
-
+
/**
* Set requirement for sorted table
*
@@ -296,97 +340,70 @@
*
* @param conf
* JobConf object.
- * @param sortcolumns
- * Sort column names.
- * @param comparator
- * Comparator name. Null means the caller does not care but table union will check
- * identical comparators regardless as a minimum sanity check
+ * @param sortInfo
+ * ZebraSortInfo object containing sorting information.
*
*/
- public static void requireSortedTable(JobConf conf, String sortcolumns, String comparator) throws IOException {
+ public static void requireSortedTable(JobConf conf, ZebraSortInfo sortInfo) throws IOException {
TableExpr expr = getInputExpr(conf);
+ String comparatorName = null;
+ String[] sortcolumns = null;
+ if (sortInfo != null)
+ {
+ comparatorName = TFile.COMPARATOR_JCLASS+sortInfo.getComparator();
+ String sortColumnNames = sortInfo.getSortColumns();
+ if (sortColumnNames != null)
+ sortcolumns = sortColumnNames.trim().split(SortInfo.SORTED_COLUMN_DELIMITER);
+ if (sortcolumns == null)
+ throw new IllegalArgumentException("No sort columns specified.");
+ }
+
if (expr instanceof BasicTableExpr)
{
BasicTable.Reader reader = new BasicTable.Reader(((BasicTableExpr) expr).getPath(), conf);
- SortInfo sortInfo = reader.getSortInfo();
+ SortInfo mySortInfo = reader.getSortInfo();
+
reader.close();
- if (comparator == null && sortInfo != null)
+ if (mySortInfo == null)
+ throw new IOException("The table is not sorted");
+ if (comparatorName == null)
// cheat the equals method's comparator comparison
- comparator = sortInfo.getComparator();
- if (sortInfo == null || !sortInfo.equals(sortcolumns, comparator))
+ comparatorName = mySortInfo.getComparator();
+ if (sortcolumns != null && !mySortInfo.equals(sortcolumns, comparatorName))
{
- throw new IOException("The table is not (properly) sorted");
+ throw new IOException("The table is not properly sorted");
}
- } else {
- List<LeafTableInfo> leaves = expr.getLeafTables(null);
- for (Iterator<LeafTableInfo> it = leaves.iterator(); it.hasNext(); )
- {
- LeafTableInfo leaf = it.next();
- BasicTable.Reader reader = new BasicTable.Reader(leaf.getPath(), conf);
- SortInfo sortInfo = reader.getSortInfo();
- reader.close();
- if (comparator == null && sortInfo != null)
- comparator = sortInfo.getComparator(); // use the first table's comparator as comparison base
- if (sortInfo == null || !sortInfo.equals(sortcolumns, comparator))
- {
- throw new IOException("The table is not (properly) sorted");
- }
- }
- // need key range input splits for sorted table union
setSorted(conf);
- }
- }
-
- /**
- * Requires sorted table or table union. For table union, leading sort columns
- * of component tables need to be the same
- *
- * @param conf
- * JobConf object.
- *
- */
- public static void requireSortedTable(JobConf conf) throws IOException {
- TableExpr expr = getInputExpr(conf);
- if (expr instanceof BasicTableExpr)
- {
- BasicTable.Reader reader = new BasicTable.Reader(((BasicTableExpr) expr).getPath(), conf);
- SortInfo sortInfo = reader.getSortInfo();
- reader.close();
- if (sortInfo == null)
- {
- throw new IOException("The table is not (properly) sorted");
- }
} else {
List<LeafTableInfo> leaves = expr.getLeafTables(null);
- String sortcolumns = null, comparator = null;
for (Iterator<LeafTableInfo> it = leaves.iterator(); it.hasNext(); )
{
LeafTableInfo leaf = it.next();
BasicTable.Reader reader = new BasicTable.Reader(leaf.getPath(), conf);
- SortInfo sortInfo = reader.getSortInfo();
+ SortInfo mySortInfo = reader.getSortInfo();
reader.close();
- if (sortInfo == null)
- {
- throw new IOException("The table is not (properly) sorted");
- }
- // check the compatible sort info among the member tables of a union
+ if (mySortInfo == null)
+ throw new IOException("The table is not sorted");
+ if (comparatorName == null)
+ comparatorName = mySortInfo.getComparator(); // use the first table's comparator as comparison base
if (sortcolumns == null)
- {
- sortcolumns = SortInfo.toSortString(sortInfo.getSortColumnNames());
- comparator = sortInfo.getComparator();
- } else {
- if (!sortInfo.equals(sortcolumns, comparator))
- {
- throw new IOException("The table is not (properly) sorted");
- }
- }
+ {
+ sortcolumns = mySortInfo.getSortColumnNames();
+ comparatorName = mySortInfo.getComparator();
+ } else {
+ if (!mySortInfo.equals(sortcolumns, comparatorName))
+ {
+ throw new IOException("The table is not properly sorted");
+ }
+ }
}
// need key range input splits for sorted table union
setSorted(conf);
}
}
+
/**
- * Set requirement for sorted table
+ * Get requirement for sorted table
*
*@param conf
* JobConf object.
@@ -400,8 +417,8 @@
*/
@Override
public RecordReader<BytesWritable, Tuple> getRecordReader(InputSplit split,
- JobConf conf, Reporter reporter) throws IOException {
- TableExpr expr = getInputExpr(conf);
+ JobConf conf, Reporter reporter) throws IOException {
+ TableExpr expr = getInputExpr(conf);
if (expr == null) {
throw new IOException("Table expression not defined");
}
@@ -410,6 +427,7 @@
expr.setSortedSplit();
String strProj = conf.get(INPUT_PROJ);
+
String projection = null;
try {
if (strProj == null) {
@@ -438,14 +456,14 @@
* comma-separated column names in projection. null means all columns in projection
*/
- public TableRecordReader getTableRecordReader(JobConf conf, String projection) throws IOException, ParseException
+ public static TableRecordReader getTableRecordReader(JobConf conf, String projection) throws IOException, ParseException
{
// a single split is needed
if (projection != null)
setProjection(conf, projection);
TableInputFormat inputFormat = new TableInputFormat();
InputSplit[] splits = inputFormat.getSplits(conf, 1);
- return (TableRecordReader) getRecordReader(splits[0], conf, Reporter.NULL);
+ return (TableRecordReader) inputFormat.getRecordReader(splits[0], conf, Reporter.NULL);
}
private static InputSplit[] getSortedSplits(JobConf conf, int numSplits,
@@ -486,11 +504,6 @@
bd = BlockDistribution.sum(bd, reader.getBlockDistribution((RangeSplit)null));
}
- if (bd == null) {
- // should never happen.
- return new InputSplit[0];
- }
-
SortedTableSplit split = new SortedTableSplit(null, null, bd, conf);
return new InputSplit[] { split };
}
@@ -509,7 +522,8 @@
if (keyDistri == null) {
// should never happen.
- return new InputSplit[0];
+ SortedTableSplit split = new SortedTableSplit(null, null, null, conf);
+ return new InputSplit[] { split };
}
if (numSplits > 0) {
@@ -571,13 +585,20 @@
ArrayList<InputSplit> ret = new ArrayList<InputSplit>();
if (numSplits <= 0) {
- for (int i = 0; i < readers.size(); ++i) {
- BasicTable.Reader reader = readers.get(i);
- List<RangeSplit> subSplits = reader.rangeSplit(-1);
- for (Iterator<RangeSplit> it = subSplits.iterator(); it.hasNext();) {
- UnsortedTableSplit split =
+ if (totalBytes <= 0) {
+ BasicTable.Reader reader = readers.get(0);
+ UnsortedTableSplit split =
+ new UnsortedTableSplit(reader, null, conf);
+ ret.add(split);
+ } else {
+ for (int i = 0; i < readers.size(); ++i) {
+ BasicTable.Reader reader = readers.get(i);
+ List<RangeSplit> subSplits = reader.rangeSplit(-1);
+ for (Iterator<RangeSplit> it = subSplits.iterator(); it.hasNext();) {
+ UnsortedTableSplit split =
new UnsortedTableSplit(reader, it.next(), conf);
- ret.add(split);
+ ret.add(split);
+ }
}
}
} else {
@@ -700,14 +721,8 @@
new BasicTable.Reader(leaf.getPath(), conf);
reader.setProjection(leaf.getProjection());
BasicTableStatus s = reader.getStatus();
- if ((s.getRows() != 0) && (s.getSize() > 0)) {
- // skipping empty tables.
- // getRows() may return -1 to indicate unknown row numbers.
- readers.add(reader);
- status.add(s);
- } else {
- reader.close();
- }
+ readers.add(reader);
+ status.add(s);
}
if (readers.isEmpty()) {
@@ -795,6 +810,12 @@
@Override
public String[] getLocations() throws IOException {
+ if (hosts == null)
+ {
+ String[] tmp = new String[1];
+ tmp[0] = "";
+ return tmp;
+ }
return hosts;
}
@@ -884,6 +905,12 @@
@Override
public String[] getLocations() throws IOException {
+ if (hosts == null)
+ {
+ String[] tmp = new String[1];
+ tmp[0] = "";
+ return tmp;
+ }
return hosts;
}
Modified: hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/TableRecordReader.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/TableRecordReader.java?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/TableRecordReader.java (original)
+++ hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/TableRecordReader.java Thu Dec 17 18:36:28 2009
@@ -94,7 +94,7 @@
@Override
public float getProgress() throws IOException {
- return (float) ((scanner.atEnd()) ? 1.0 : 0);
+ return (float)((scanner.atEnd()) ? 1.0 : 0);
}
@Override
Added: hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/ZebraOutputPartition.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/ZebraOutputPartition.java?rev=891828&view=auto
==============================================================================
--- hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/ZebraOutputPartition.java (added)
+++ hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/ZebraOutputPartition.java Thu Dec 17 18:36:28 2009
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.zebra.mapred;
+
+import java.io.IOException;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.pig.data.Tuple;
+
+
+public abstract class ZebraOutputPartition implements Configurable{
+
+ protected Configuration jobConf;
+
+ @Override
+ public void setConf( Configuration conf) {
+ jobConf = conf;
+ }
+
+ @Override
+ public Configuration getConf( ) {
+ return jobConf;
+ }
+
+ public abstract int getOutputPartition(BytesWritable key, Tuple Value)
+ throws IOException;
+}
Added: hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/ZebraProjection.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/ZebraProjection.java?rev=891828&view=auto
==============================================================================
--- hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/ZebraProjection.java (added)
+++ hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/ZebraProjection.java Thu Dec 17 18:36:28 2009
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.zebra.mapred;
+
+/**
+ * A wrapper class for Projection.
+ */
+public class ZebraProjection {
+ private String projectionStr = null;
+
+ private ZebraProjection(String projStr) {
+ projectionStr = projStr;
+ }
+
+ public static ZebraProjection createZebraProjection(String projStr) {
+ return new ZebraProjection(projStr);
+ }
+
+ public String toString() {
+ return projectionStr;
+ }
+}
\ No newline at end of file
Added: hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/ZebraSchema.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/ZebraSchema.java?rev=891828&view=auto
==============================================================================
--- hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/ZebraSchema.java (added)
+++ hadoop/pig/branches/load-store-redesign/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/ZebraSchema.java Thu Dec 17 18:36:28 2009
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.zebra.mapred;
+
+import org.apache.hadoop.zebra.schema.Schema;
+
+/**
+ * A wrapper class for Schema.
+ */
+public class ZebraSchema {
+ private String schemaStr = null;
+
+ private ZebraSchema(String str) {
+ String normalizedStr = Schema.normalize(str);
+ schemaStr = normalizedStr;
+ }
+
+ public static ZebraSchema createZebraSchema(String str) {
+ return new ZebraSchema(str);
+ }
+
+ public String toString() {
+ return schemaStr;
+ }
+}