You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@pig.apache.org by Jonathan Coveney <jc...@gmail.com> on 2013/01/17 22:02:56 UTC
Review Request: Add BigInteger and BigDecimal to Pig
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/9012/
-----------------------------------------------------------
Review request for pig, Alan Gates and Mathias Herberts.
Description
-------
This patch adds big integer and big decimal support to Pig. It could use more tests, something I'd appreciate feedback on (but I wanted to make sure the core implementation is good)
This addresses bug PIG-2764.
https://issues.apache.org/jira/browse/PIG-2764
Diffs
-----
.gitignore cc62d7d
src/org/apache/pig/LoadCaster.java 574769b
src/org/apache/pig/PigWarning.java 5de075f
src/org/apache/pig/StoreCaster.java 5fe48de
src/org/apache/pig/backend/hadoop/BigDecimalWritable.java PRE-CREATION
src/org/apache/pig/backend/hadoop/BigIntegerWritable.java PRE-CREATION
src/org/apache/pig/backend/hadoop/HDataType.java 84a56b8
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java 96fba6b
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBigDecimalRawComparator.java PRE-CREATION
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBigIntegerRawComparator.java PRE-CREATION
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/partitioners/WeightedRangePartitioner.java 9749339
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PhysicalOperator.java f40eb43
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Add.java c84b767
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ConstantExpression.java db3840f
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Divide.java 4656c28
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/EqualToExpr.java 6683beb
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ExpressionOperator.java 2806336
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/GTOrEqualToExpr.java d64a080
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/GreaterThanExpr.java 704d0b8
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/LTOrEqualToExpr.java 9dc929e
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/LessThanExpr.java 0320698
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Mod.java 6819185
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Multiply.java 7b57bed
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/NotEqualToExpr.java 79a4461
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POBinCond.java 08544d5
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POCast.java e8c2f2c
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POIsNull.java f20b839
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/PONegative.java c076ae7
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POProject.java 8887133
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserComparisonFunc.java 479eb83
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java 3c7e741
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Subtract.java 79d4c73
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POForEach.java bf2ba08
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POLocalRearrange.java ddb25f1
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPartialAgg.java aa11409
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPreCombinerLocalRearrange.java 52401eb
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POSort.java ad33e7b
src/org/apache/pig/backend/hadoop/hbase/HBaseBinaryConverter.java 60a5899
src/org/apache/pig/backend/hadoop/hbase/HBaseStorage.java a6f4ea6
src/org/apache/pig/builtin/ABS.java 8a7c631
src/org/apache/pig/builtin/BigDecimalAbs.java PRE-CREATION
src/org/apache/pig/builtin/BigIntegerAbs.java PRE-CREATION
src/org/apache/pig/builtin/BinStorage.java 38b4492
src/org/apache/pig/builtin/TextLoader.java d5bcf02
src/org/apache/pig/builtin/Utf8StorageConverter.java da12ed6
src/org/apache/pig/data/BinInterSedes.java e851d8b
src/org/apache/pig/data/DataReaderWriter.java 37a162a
src/org/apache/pig/data/DataType.java e4c7b98
src/org/apache/pig/data/DefaultTuple.java e182b06
src/org/apache/pig/data/SizeUtil.java 90e5d94
src/org/apache/pig/data/TypeAwareTuple.java cf78d0a
src/org/apache/pig/impl/io/NullableBigDecimalWritable.java PRE-CREATION
src/org/apache/pig/impl/io/NullableBigIntegerWritable.java PRE-CREATION
src/org/apache/pig/impl/logicalLayer/schema/SchemaUtil.java c257ada
src/org/apache/pig/impl/util/CastUtils.java 309130a
src/org/apache/pig/impl/util/NumValCarrier.java af519fd
src/org/apache/pig/impl/util/StorageUtil.java 087651a
src/org/apache/pig/newplan/logical/rules/ConstExpEvaluator.java f968294
src/org/apache/pig/newplan/logical/visitor/TypeCheckingExpVisitor.java d7a1370
src/org/apache/pig/parser/AliasMasker.g 8cb1b30
src/org/apache/pig/parser/AstPrinter.g c6f7ff8
src/org/apache/pig/parser/AstValidator.g 8646078
src/org/apache/pig/parser/LogicalPlanBuilder.java 699f7a6
src/org/apache/pig/parser/LogicalPlanGenerator.g 9b9c099
src/org/apache/pig/parser/QueryLexer.g f201916
src/org/apache/pig/parser/QueryParser.g 642884e
src/org/apache/pig/pen/AugmentBaseDataVisitor.java 57dcb14
test/e2e/pig/udfs/java/org/apache/pig/test/udf/storefunc/PigPerformanceLoader.java 94c28be
test/org/apache/pig/test/TestAdd.java de9e90d
test/org/apache/pig/test/TestBestFitCast.java e3172d2
test/org/apache/pig/test/TestPOCast.java 72787b3
Diff: https://reviews.apache.org/r/9012/diff/
Testing
-------
Thanks,
Jonathan Coveney
Re: Review Request: Add BigInteger and BigDecimal to Pig
Posted by Mathias Herberts <Ma...@gmail.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/9012/#review15469
-----------------------------------------------------------
src/org/apache/pig/backend/hadoop/HDataType.java
<https://reviews.apache.org/r/9012/#comment33407>
Missing 'break;'
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POCast.java
<https://reviews.apache.org/r/9012/#comment33400>
Please use braces after if for clarity.
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POCast.java
<https://reviews.apache.org/r/9012/#comment33401>
Please use braces after if for clarity.
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POCast.java
<https://reviews.apache.org/r/9012/#comment33402>
Please use braces after if for clarity.
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POCast.java
<https://reviews.apache.org/r/9012/#comment33403>
Please use braces after if for clarity.
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POCast.java
<https://reviews.apache.org/r/9012/#comment33404>
Please use braces after if for clarity.
src/org/apache/pig/backend/hadoop/hbase/HBaseBinaryConverter.java
<https://reviews.apache.org/r/9012/#comment33406>
Why not use BigInteger(byte[]) or the String rep? Same comment for BigDecimal and the String rep.
src/org/apache/pig/builtin/TextLoader.java
<https://reviews.apache.org/r/9012/#comment33379>
Message should be 'conversion to BigInteger'
src/org/apache/pig/builtin/TextLoader.java
<https://reviews.apache.org/r/9012/#comment33381>
Message should be 'conversion to BigDecimal'
src/org/apache/pig/builtin/Utf8StorageConverter.java
<https://reviews.apache.org/r/9012/#comment33384>
Shouldn't the charset be specified in the calls to getBytes for both BigInteger and BigDecimal?
src/org/apache/pig/data/DataType.java
<https://reviews.apache.org/r/9012/#comment33386>
Indentation is messy here.
src/org/apache/pig/data/DataType.java
<https://reviews.apache.org/r/9012/#comment33387>
Messy indent.
src/org/apache/pig/data/DataType.java
<https://reviews.apache.org/r/9012/#comment33388>
Messy indent.
src/org/apache/pig/data/DataType.java
<https://reviews.apache.org/r/9012/#comment33391>
Should read 'BigDecimal' here and in next message.
src/org/apache/pig/data/DataType.java
<https://reviews.apache.org/r/9012/#comment33392>
Messy indent
src/org/apache/pig/data/DefaultTuple.java
<https://reviews.apache.org/r/9012/#comment33394>
I don't understand those two lines!
src/org/apache/pig/data/SizeUtil.java
<https://reviews.apache.org/r/9012/#comment33395>
I thought BigDecimal and BigInteger did not have the same size, cf http://javamoods.blogspot.fr/2009/03/how-big-is-bigdecimal.html
src/org/apache/pig/impl/util/StorageUtil.java
<https://reviews.apache.org/r/9012/#comment33397>
Shouldn't the charset be specified?
src/org/apache/pig/parser/LogicalPlanBuilder.java
<https://reviews.apache.org/r/9012/#comment33408>
Since suffixes are 'BI' and 'BD', we should strip the last two characters of the string, not only the last one.
- Mathias Herberts
On Jan. 17, 2013, 9:02 p.m., Jonathan Coveney wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/9012/
> -----------------------------------------------------------
>
> (Updated Jan. 17, 2013, 9:02 p.m.)
>
>
> Review request for pig, Alan Gates and Mathias Herberts.
>
>
> Description
> -------
>
> This patch adds big integer and big decimal support to Pig. It could use more tests, something I'd appreciate feedback on (but I wanted to make sure the core implementation is good)
>
>
> This addresses bug PIG-2764.
> https://issues.apache.org/jira/browse/PIG-2764
>
>
> Diffs
> -----
>
> .gitignore cc62d7d
> src/org/apache/pig/LoadCaster.java 574769b
> src/org/apache/pig/PigWarning.java 5de075f
> src/org/apache/pig/StoreCaster.java 5fe48de
> src/org/apache/pig/backend/hadoop/BigDecimalWritable.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/BigIntegerWritable.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/HDataType.java 84a56b8
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java 96fba6b
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBigDecimalRawComparator.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBigIntegerRawComparator.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/partitioners/WeightedRangePartitioner.java 9749339
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PhysicalOperator.java f40eb43
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Add.java c84b767
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ConstantExpression.java db3840f
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Divide.java 4656c28
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/EqualToExpr.java 6683beb
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ExpressionOperator.java 2806336
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/GTOrEqualToExpr.java d64a080
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/GreaterThanExpr.java 704d0b8
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/LTOrEqualToExpr.java 9dc929e
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/LessThanExpr.java 0320698
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Mod.java 6819185
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Multiply.java 7b57bed
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/NotEqualToExpr.java 79a4461
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POBinCond.java 08544d5
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POCast.java e8c2f2c
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POIsNull.java f20b839
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/PONegative.java c076ae7
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POProject.java 8887133
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserComparisonFunc.java 479eb83
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java 3c7e741
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Subtract.java 79d4c73
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POForEach.java bf2ba08
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POLocalRearrange.java ddb25f1
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPartialAgg.java aa11409
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPreCombinerLocalRearrange.java 52401eb
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POSort.java ad33e7b
> src/org/apache/pig/backend/hadoop/hbase/HBaseBinaryConverter.java 60a5899
> src/org/apache/pig/backend/hadoop/hbase/HBaseStorage.java a6f4ea6
> src/org/apache/pig/builtin/ABS.java 8a7c631
> src/org/apache/pig/builtin/BigDecimalAbs.java PRE-CREATION
> src/org/apache/pig/builtin/BigIntegerAbs.java PRE-CREATION
> src/org/apache/pig/builtin/BinStorage.java 38b4492
> src/org/apache/pig/builtin/TextLoader.java d5bcf02
> src/org/apache/pig/builtin/Utf8StorageConverter.java da12ed6
> src/org/apache/pig/data/BinInterSedes.java e851d8b
> src/org/apache/pig/data/DataReaderWriter.java 37a162a
> src/org/apache/pig/data/DataType.java e4c7b98
> src/org/apache/pig/data/DefaultTuple.java e182b06
> src/org/apache/pig/data/SizeUtil.java 90e5d94
> src/org/apache/pig/data/TypeAwareTuple.java cf78d0a
> src/org/apache/pig/impl/io/NullableBigDecimalWritable.java PRE-CREATION
> src/org/apache/pig/impl/io/NullableBigIntegerWritable.java PRE-CREATION
> src/org/apache/pig/impl/logicalLayer/schema/SchemaUtil.java c257ada
> src/org/apache/pig/impl/util/CastUtils.java 309130a
> src/org/apache/pig/impl/util/NumValCarrier.java af519fd
> src/org/apache/pig/impl/util/StorageUtil.java 087651a
> src/org/apache/pig/newplan/logical/rules/ConstExpEvaluator.java f968294
> src/org/apache/pig/newplan/logical/visitor/TypeCheckingExpVisitor.java d7a1370
> src/org/apache/pig/parser/AliasMasker.g 8cb1b30
> src/org/apache/pig/parser/AstPrinter.g c6f7ff8
> src/org/apache/pig/parser/AstValidator.g 8646078
> src/org/apache/pig/parser/LogicalPlanBuilder.java 699f7a6
> src/org/apache/pig/parser/LogicalPlanGenerator.g 9b9c099
> src/org/apache/pig/parser/QueryLexer.g f201916
> src/org/apache/pig/parser/QueryParser.g 642884e
> src/org/apache/pig/pen/AugmentBaseDataVisitor.java 57dcb14
> test/e2e/pig/udfs/java/org/apache/pig/test/udf/storefunc/PigPerformanceLoader.java 94c28be
> test/org/apache/pig/test/TestAdd.java de9e90d
> test/org/apache/pig/test/TestBestFitCast.java e3172d2
> test/org/apache/pig/test/TestPOCast.java 72787b3
>
> Diff: https://reviews.apache.org/r/9012/diff/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Jonathan Coveney
>
>
Re: Review Request: Add BigInteger and BigDecimal to Pig
Posted by Jonathan Coveney <jc...@gmail.com>.
> On Jan. 19, 2013, 7:29 p.m., Mathias Herberts wrote:
> > src/org/apache/pig/data/DefaultTuple.java, line 359
> > <https://reviews.apache.org/r/9012/diff/1-2/?file=249804#file249804line359>
> >
> > Since BigDecimal have a scale which is an int, nothing prevents a BigDecimal to have a scale which won't fit on a short, and thus whose string representation might also be longer than a short.
> >
> > Is I understand this code correctly (thanks to your explanation), if type if CHARARRAY, field length is encoded on a short? This will be troublesome if scale > 0x7ffff.
This is a bit annoying/confusing, but there are two internal "CHARARRAYS," either a CHARARRAY or a BIGCHARRAY. So if it's bigger than a short, it will be an int (ie a BIGCHARARRAY) and it should deserialized correctly. Literally all DataType.BIGINTEGER in this case signals is "what comes after is a string," which we then serialized and deserialize in accordance with the rest of pig.
- Jonathan
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/9012/#review15522
-----------------------------------------------------------
On Jan. 18, 2013, 10:11 p.m., Jonathan Coveney wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/9012/
> -----------------------------------------------------------
>
> (Updated Jan. 18, 2013, 10:11 p.m.)
>
>
> Review request for pig, Alan Gates and Mathias Herberts.
>
>
> Description
> -------
>
> This patch adds big integer and big decimal support to Pig. It could use more tests, something I'd appreciate feedback on (but I wanted to make sure the core implementation is good)
>
>
> This addresses bug PIG-2764.
> https://issues.apache.org/jira/browse/PIG-2764
>
>
> Diffs
> -----
>
> .gitignore cc62d7d
> src/org/apache/pig/LoadCaster.java 574769b
> src/org/apache/pig/PigWarning.java 5de075f
> src/org/apache/pig/StoreCaster.java 5fe48de
> src/org/apache/pig/backend/hadoop/BigDecimalWritable.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/BigIntegerWritable.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/HDataType.java 84a56b8
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java 96fba6b
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBigDecimalRawComparator.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBigIntegerRawComparator.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/partitioners/WeightedRangePartitioner.java 9749339
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PhysicalOperator.java f40eb43
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Add.java c84b767
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ConstantExpression.java db3840f
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Divide.java 4656c28
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/EqualToExpr.java 6683beb
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ExpressionOperator.java 2806336
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/GTOrEqualToExpr.java d64a080
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/GreaterThanExpr.java 704d0b8
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/LTOrEqualToExpr.java 9dc929e
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/LessThanExpr.java 0320698
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Mod.java 6819185
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Multiply.java 7b57bed
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/NotEqualToExpr.java 79a4461
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POBinCond.java 08544d5
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POCast.java e8c2f2c
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POIsNull.java f20b839
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/PONegative.java c076ae7
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POProject.java 8887133
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserComparisonFunc.java 479eb83
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java 3c7e741
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Subtract.java 79d4c73
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POForEach.java bf2ba08
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POLocalRearrange.java ddb25f1
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPartialAgg.java aa11409
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPreCombinerLocalRearrange.java 52401eb
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POSort.java ad33e7b
> src/org/apache/pig/backend/hadoop/hbase/HBaseBinaryConverter.java 60a5899
> src/org/apache/pig/backend/hadoop/hbase/HBaseStorage.java a6f4ea6
> src/org/apache/pig/builtin/ABS.java 8a7c631
> src/org/apache/pig/builtin/BigDecimalAbs.java PRE-CREATION
> src/org/apache/pig/builtin/BigIntegerAbs.java PRE-CREATION
> src/org/apache/pig/builtin/BinStorage.java 38b4492
> src/org/apache/pig/builtin/TextLoader.java d5bcf02
> src/org/apache/pig/builtin/Utf8StorageConverter.java da12ed6
> src/org/apache/pig/data/BinInterSedes.java e851d8b
> src/org/apache/pig/data/DataReaderWriter.java 37a162a
> src/org/apache/pig/data/DataType.java e4c7b98
> src/org/apache/pig/data/DefaultTuple.java e182b06
> src/org/apache/pig/data/SizeUtil.java 90e5d94
> src/org/apache/pig/data/TypeAwareTuple.java cf78d0a
> src/org/apache/pig/impl/io/NullableBigDecimalWritable.java PRE-CREATION
> src/org/apache/pig/impl/io/NullableBigIntegerWritable.java PRE-CREATION
> src/org/apache/pig/impl/logicalLayer/schema/SchemaUtil.java c257ada
> src/org/apache/pig/impl/util/CastUtils.java 309130a
> src/org/apache/pig/impl/util/NumValCarrier.java af519fd
> src/org/apache/pig/impl/util/StorageUtil.java 087651a
> src/org/apache/pig/newplan/logical/rules/ConstExpEvaluator.java f968294
> src/org/apache/pig/newplan/logical/visitor/TypeCheckingExpVisitor.java d7a1370
> src/org/apache/pig/parser/AliasMasker.g 8cb1b30
> src/org/apache/pig/parser/AstPrinter.g c6f7ff8
> src/org/apache/pig/parser/AstValidator.g 8646078
> src/org/apache/pig/parser/LogicalPlanBuilder.java 699f7a6
> src/org/apache/pig/parser/LogicalPlanGenerator.g 9b9c099
> src/org/apache/pig/parser/QueryLexer.g f201916
> src/org/apache/pig/parser/QueryParser.g 642884e
> src/org/apache/pig/pen/AugmentBaseDataVisitor.java 57dcb14
> test/e2e/pig/udfs/java/org/apache/pig/test/udf/storefunc/PigPerformanceLoader.java 94c28be
> test/org/apache/pig/test/TestAdd.java de9e90d
> test/org/apache/pig/test/TestBestFitCast.java e3172d2
> test/org/apache/pig/test/TestPOCast.java 72787b3
>
> Diff: https://reviews.apache.org/r/9012/diff/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Jonathan Coveney
>
>
Re: Review Request: Add BigInteger and BigDecimal to Pig
Posted by Mathias Herberts <Ma...@gmail.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/9012/#review15522
-----------------------------------------------------------
src/org/apache/pig/data/DefaultTuple.java
<https://reviews.apache.org/r/9012/#comment33505>
Missing 'e' in 'byts'.
src/org/apache/pig/data/DefaultTuple.java
<https://reviews.apache.org/r/9012/#comment33509>
Since BigDecimal have a scale which is an int, nothing prevents a BigDecimal to have a scale which won't fit on a short, and thus whose string representation might also be longer than a short.
Is I understand this code correctly (thanks to your explanation), if type if CHARARRAY, field length is encoded on a short? This will be troublesome if scale > 0x7ffff.
- Mathias Herberts
On Jan. 18, 2013, 10:11 p.m., Jonathan Coveney wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/9012/
> -----------------------------------------------------------
>
> (Updated Jan. 18, 2013, 10:11 p.m.)
>
>
> Review request for pig, Alan Gates and Mathias Herberts.
>
>
> Description
> -------
>
> This patch adds big integer and big decimal support to Pig. It could use more tests, something I'd appreciate feedback on (but I wanted to make sure the core implementation is good)
>
>
> This addresses bug PIG-2764.
> https://issues.apache.org/jira/browse/PIG-2764
>
>
> Diffs
> -----
>
> .gitignore cc62d7d
> src/org/apache/pig/LoadCaster.java 574769b
> src/org/apache/pig/PigWarning.java 5de075f
> src/org/apache/pig/StoreCaster.java 5fe48de
> src/org/apache/pig/backend/hadoop/BigDecimalWritable.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/BigIntegerWritable.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/HDataType.java 84a56b8
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java 96fba6b
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBigDecimalRawComparator.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBigIntegerRawComparator.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/partitioners/WeightedRangePartitioner.java 9749339
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PhysicalOperator.java f40eb43
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Add.java c84b767
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ConstantExpression.java db3840f
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Divide.java 4656c28
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/EqualToExpr.java 6683beb
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ExpressionOperator.java 2806336
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/GTOrEqualToExpr.java d64a080
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/GreaterThanExpr.java 704d0b8
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/LTOrEqualToExpr.java 9dc929e
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/LessThanExpr.java 0320698
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Mod.java 6819185
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Multiply.java 7b57bed
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/NotEqualToExpr.java 79a4461
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POBinCond.java 08544d5
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POCast.java e8c2f2c
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POIsNull.java f20b839
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/PONegative.java c076ae7
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POProject.java 8887133
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserComparisonFunc.java 479eb83
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java 3c7e741
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Subtract.java 79d4c73
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POForEach.java bf2ba08
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POLocalRearrange.java ddb25f1
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPartialAgg.java aa11409
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPreCombinerLocalRearrange.java 52401eb
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POSort.java ad33e7b
> src/org/apache/pig/backend/hadoop/hbase/HBaseBinaryConverter.java 60a5899
> src/org/apache/pig/backend/hadoop/hbase/HBaseStorage.java a6f4ea6
> src/org/apache/pig/builtin/ABS.java 8a7c631
> src/org/apache/pig/builtin/BigDecimalAbs.java PRE-CREATION
> src/org/apache/pig/builtin/BigIntegerAbs.java PRE-CREATION
> src/org/apache/pig/builtin/BinStorage.java 38b4492
> src/org/apache/pig/builtin/TextLoader.java d5bcf02
> src/org/apache/pig/builtin/Utf8StorageConverter.java da12ed6
> src/org/apache/pig/data/BinInterSedes.java e851d8b
> src/org/apache/pig/data/DataReaderWriter.java 37a162a
> src/org/apache/pig/data/DataType.java e4c7b98
> src/org/apache/pig/data/DefaultTuple.java e182b06
> src/org/apache/pig/data/SizeUtil.java 90e5d94
> src/org/apache/pig/data/TypeAwareTuple.java cf78d0a
> src/org/apache/pig/impl/io/NullableBigDecimalWritable.java PRE-CREATION
> src/org/apache/pig/impl/io/NullableBigIntegerWritable.java PRE-CREATION
> src/org/apache/pig/impl/logicalLayer/schema/SchemaUtil.java c257ada
> src/org/apache/pig/impl/util/CastUtils.java 309130a
> src/org/apache/pig/impl/util/NumValCarrier.java af519fd
> src/org/apache/pig/impl/util/StorageUtil.java 087651a
> src/org/apache/pig/newplan/logical/rules/ConstExpEvaluator.java f968294
> src/org/apache/pig/newplan/logical/visitor/TypeCheckingExpVisitor.java d7a1370
> src/org/apache/pig/parser/AliasMasker.g 8cb1b30
> src/org/apache/pig/parser/AstPrinter.g c6f7ff8
> src/org/apache/pig/parser/AstValidator.g 8646078
> src/org/apache/pig/parser/LogicalPlanBuilder.java 699f7a6
> src/org/apache/pig/parser/LogicalPlanGenerator.g 9b9c099
> src/org/apache/pig/parser/QueryLexer.g f201916
> src/org/apache/pig/parser/QueryParser.g 642884e
> src/org/apache/pig/pen/AugmentBaseDataVisitor.java 57dcb14
> test/e2e/pig/udfs/java/org/apache/pig/test/udf/storefunc/PigPerformanceLoader.java 94c28be
> test/org/apache/pig/test/TestAdd.java de9e90d
> test/org/apache/pig/test/TestBestFitCast.java e3172d2
> test/org/apache/pig/test/TestPOCast.java 72787b3
>
> Diff: https://reviews.apache.org/r/9012/diff/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Jonathan Coveney
>
>
Re: Review Request: Add BigInteger and BigDecimal to Pig
Posted by Mathias Herberts <Ma...@gmail.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/9012/#review15588
-----------------------------------------------------------
Ship it!
LGTM
- Mathias Herberts
On Jan. 22, 2013, 10:05 p.m., Jonathan Coveney wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/9012/
> -----------------------------------------------------------
>
> (Updated Jan. 22, 2013, 10:05 p.m.)
>
>
> Review request for pig, Alan Gates and Mathias Herberts.
>
>
> Description
> -------
>
> This patch adds big integer and big decimal support to Pig. It could use more tests, something I'd appreciate feedback on (but I wanted to make sure the core implementation is good)
>
>
> This addresses bug PIG-2764.
> https://issues.apache.org/jira/browse/PIG-2764
>
>
> Diffs
> -----
>
> .gitignore cc62d7d
> src/org/apache/pig/LoadCaster.java 574769b
> src/org/apache/pig/PigWarning.java 5de075f
> src/org/apache/pig/StoreCaster.java 5fe48de
> src/org/apache/pig/backend/hadoop/BigDecimalWritable.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/BigIntegerWritable.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/HDataType.java 84a56b8
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java 96fba6b
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBigDecimalRawComparator.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBigIntegerRawComparator.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/partitioners/WeightedRangePartitioner.java 9749339
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PhysicalOperator.java f40eb43
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Add.java c84b767
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ConstantExpression.java db3840f
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Divide.java 4656c28
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/EqualToExpr.java 6683beb
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ExpressionOperator.java 2806336
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/GTOrEqualToExpr.java d64a080
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/GreaterThanExpr.java 704d0b8
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/LTOrEqualToExpr.java 9dc929e
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/LessThanExpr.java 0320698
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Mod.java 6819185
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Multiply.java 7b57bed
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/NotEqualToExpr.java 79a4461
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POBinCond.java 08544d5
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POCast.java e8c2f2c
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POIsNull.java f20b839
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/PONegative.java c076ae7
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POProject.java 8887133
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserComparisonFunc.java 479eb83
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java 3c7e741
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Subtract.java 79d4c73
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POForEach.java bf2ba08
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POLocalRearrange.java ddb25f1
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPartialAgg.java aa11409
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPreCombinerLocalRearrange.java 52401eb
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POSort.java ad33e7b
> src/org/apache/pig/backend/hadoop/hbase/HBaseBinaryConverter.java 60a5899
> src/org/apache/pig/backend/hadoop/hbase/HBaseStorage.java a6f4ea6
> src/org/apache/pig/builtin/ABS.java 8a7c631
> src/org/apache/pig/builtin/BigDecimalAbs.java PRE-CREATION
> src/org/apache/pig/builtin/BigIntegerAbs.java PRE-CREATION
> src/org/apache/pig/builtin/BinStorage.java 38b4492
> src/org/apache/pig/builtin/TextLoader.java d5bcf02
> src/org/apache/pig/builtin/Utf8StorageConverter.java da12ed6
> src/org/apache/pig/data/BinInterSedes.java e851d8b
> src/org/apache/pig/data/DataReaderWriter.java 37a162a
> src/org/apache/pig/data/DataType.java e4c7b98
> src/org/apache/pig/data/DefaultTuple.java e182b06
> src/org/apache/pig/data/SizeUtil.java 90e5d94
> src/org/apache/pig/data/TypeAwareTuple.java cf78d0a
> src/org/apache/pig/impl/io/NullableBigDecimalWritable.java PRE-CREATION
> src/org/apache/pig/impl/io/NullableBigIntegerWritable.java PRE-CREATION
> src/org/apache/pig/impl/logicalLayer/schema/SchemaUtil.java c257ada
> src/org/apache/pig/impl/util/CastUtils.java 309130a
> src/org/apache/pig/impl/util/NumValCarrier.java af519fd
> src/org/apache/pig/impl/util/StorageUtil.java 087651a
> src/org/apache/pig/newplan/logical/rules/ConstExpEvaluator.java f968294
> src/org/apache/pig/newplan/logical/visitor/TypeCheckingExpVisitor.java d7a1370
> src/org/apache/pig/parser/AliasMasker.g 8cb1b30
> src/org/apache/pig/parser/AstPrinter.g c6f7ff8
> src/org/apache/pig/parser/AstValidator.g 8646078
> src/org/apache/pig/parser/LogicalPlanBuilder.java 699f7a6
> src/org/apache/pig/parser/LogicalPlanGenerator.g 9b9c099
> src/org/apache/pig/parser/QueryLexer.g f201916
> src/org/apache/pig/parser/QueryParser.g 642884e
> src/org/apache/pig/pen/AugmentBaseDataVisitor.java 57dcb14
> test/e2e/pig/udfs/java/org/apache/pig/test/udf/storefunc/PigPerformanceLoader.java 94c28be
> test/org/apache/pig/test/TestAdd.java de9e90d
> test/org/apache/pig/test/TestBestFitCast.java e3172d2
> test/org/apache/pig/test/TestPOCast.java 72787b3
>
> Diff: https://reviews.apache.org/r/9012/diff/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Jonathan Coveney
>
>
Re: Review Request: Add BigInteger and BigDecimal to Pig
Posted by Mathias Herberts <Ma...@gmail.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/9012/#review15587
-----------------------------------------------------------
Ship it!
LGTM
- Mathias Herberts
On Jan. 22, 2013, 10:05 p.m., Jonathan Coveney wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/9012/
> -----------------------------------------------------------
>
> (Updated Jan. 22, 2013, 10:05 p.m.)
>
>
> Review request for pig, Alan Gates and Mathias Herberts.
>
>
> Description
> -------
>
> This patch adds big integer and big decimal support to Pig. It could use more tests, something I'd appreciate feedback on (but I wanted to make sure the core implementation is good)
>
>
> This addresses bug PIG-2764.
> https://issues.apache.org/jira/browse/PIG-2764
>
>
> Diffs
> -----
>
> .gitignore cc62d7d
> src/org/apache/pig/LoadCaster.java 574769b
> src/org/apache/pig/PigWarning.java 5de075f
> src/org/apache/pig/StoreCaster.java 5fe48de
> src/org/apache/pig/backend/hadoop/BigDecimalWritable.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/BigIntegerWritable.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/HDataType.java 84a56b8
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java 96fba6b
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBigDecimalRawComparator.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBigIntegerRawComparator.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/partitioners/WeightedRangePartitioner.java 9749339
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PhysicalOperator.java f40eb43
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Add.java c84b767
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ConstantExpression.java db3840f
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Divide.java 4656c28
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/EqualToExpr.java 6683beb
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ExpressionOperator.java 2806336
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/GTOrEqualToExpr.java d64a080
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/GreaterThanExpr.java 704d0b8
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/LTOrEqualToExpr.java 9dc929e
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/LessThanExpr.java 0320698
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Mod.java 6819185
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Multiply.java 7b57bed
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/NotEqualToExpr.java 79a4461
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POBinCond.java 08544d5
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POCast.java e8c2f2c
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POIsNull.java f20b839
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/PONegative.java c076ae7
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POProject.java 8887133
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserComparisonFunc.java 479eb83
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java 3c7e741
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Subtract.java 79d4c73
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POForEach.java bf2ba08
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POLocalRearrange.java ddb25f1
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPartialAgg.java aa11409
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPreCombinerLocalRearrange.java 52401eb
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POSort.java ad33e7b
> src/org/apache/pig/backend/hadoop/hbase/HBaseBinaryConverter.java 60a5899
> src/org/apache/pig/backend/hadoop/hbase/HBaseStorage.java a6f4ea6
> src/org/apache/pig/builtin/ABS.java 8a7c631
> src/org/apache/pig/builtin/BigDecimalAbs.java PRE-CREATION
> src/org/apache/pig/builtin/BigIntegerAbs.java PRE-CREATION
> src/org/apache/pig/builtin/BinStorage.java 38b4492
> src/org/apache/pig/builtin/TextLoader.java d5bcf02
> src/org/apache/pig/builtin/Utf8StorageConverter.java da12ed6
> src/org/apache/pig/data/BinInterSedes.java e851d8b
> src/org/apache/pig/data/DataReaderWriter.java 37a162a
> src/org/apache/pig/data/DataType.java e4c7b98
> src/org/apache/pig/data/DefaultTuple.java e182b06
> src/org/apache/pig/data/SizeUtil.java 90e5d94
> src/org/apache/pig/data/TypeAwareTuple.java cf78d0a
> src/org/apache/pig/impl/io/NullableBigDecimalWritable.java PRE-CREATION
> src/org/apache/pig/impl/io/NullableBigIntegerWritable.java PRE-CREATION
> src/org/apache/pig/impl/logicalLayer/schema/SchemaUtil.java c257ada
> src/org/apache/pig/impl/util/CastUtils.java 309130a
> src/org/apache/pig/impl/util/NumValCarrier.java af519fd
> src/org/apache/pig/impl/util/StorageUtil.java 087651a
> src/org/apache/pig/newplan/logical/rules/ConstExpEvaluator.java f968294
> src/org/apache/pig/newplan/logical/visitor/TypeCheckingExpVisitor.java d7a1370
> src/org/apache/pig/parser/AliasMasker.g 8cb1b30
> src/org/apache/pig/parser/AstPrinter.g c6f7ff8
> src/org/apache/pig/parser/AstValidator.g 8646078
> src/org/apache/pig/parser/LogicalPlanBuilder.java 699f7a6
> src/org/apache/pig/parser/LogicalPlanGenerator.g 9b9c099
> src/org/apache/pig/parser/QueryLexer.g f201916
> src/org/apache/pig/parser/QueryParser.g 642884e
> src/org/apache/pig/pen/AugmentBaseDataVisitor.java 57dcb14
> test/e2e/pig/udfs/java/org/apache/pig/test/udf/storefunc/PigPerformanceLoader.java 94c28be
> test/org/apache/pig/test/TestAdd.java de9e90d
> test/org/apache/pig/test/TestBestFitCast.java e3172d2
> test/org/apache/pig/test/TestPOCast.java 72787b3
>
> Diff: https://reviews.apache.org/r/9012/diff/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Jonathan Coveney
>
>
Re: Review Request: Add BigInteger and BigDecimal to Pig
Posted by Russell Jurney <ru...@gmail.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/9012/#review15524
-----------------------------------------------------------
src/org/apache/pig/backend/hadoop/BigDecimalWritable.java:70: spelling error in TODO big/bit
There seem to be lots of LINT changes - not keeping tabs for newline only lines. Not sure if that is right? Plz check LINT.
- Russell Jurney
On Jan. 22, 2013, 10:05 p.m., Jonathan Coveney wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/9012/
> -----------------------------------------------------------
>
> (Updated Jan. 22, 2013, 10:05 p.m.)
>
>
> Review request for pig, Alan Gates and Mathias Herberts.
>
>
> Description
> -------
>
> This patch adds big integer and big decimal support to Pig. It could use more tests, something I'd appreciate feedback on (but I wanted to make sure the core implementation is good)
>
>
> This addresses bug PIG-2764.
> https://issues.apache.org/jira/browse/PIG-2764
>
>
> Diffs
> -----
>
> .gitignore cc62d7d
> src/org/apache/pig/LoadCaster.java 574769b
> src/org/apache/pig/PigWarning.java 5de075f
> src/org/apache/pig/StoreCaster.java 5fe48de
> src/org/apache/pig/backend/hadoop/BigDecimalWritable.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/BigIntegerWritable.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/HDataType.java 84a56b8
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java 96fba6b
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBigDecimalRawComparator.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBigIntegerRawComparator.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/partitioners/WeightedRangePartitioner.java 9749339
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PhysicalOperator.java f40eb43
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Add.java c84b767
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ConstantExpression.java db3840f
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Divide.java 4656c28
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/EqualToExpr.java 6683beb
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ExpressionOperator.java 2806336
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/GTOrEqualToExpr.java d64a080
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/GreaterThanExpr.java 704d0b8
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/LTOrEqualToExpr.java 9dc929e
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/LessThanExpr.java 0320698
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Mod.java 6819185
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Multiply.java 7b57bed
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/NotEqualToExpr.java 79a4461
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POBinCond.java 08544d5
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POCast.java e8c2f2c
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POIsNull.java f20b839
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/PONegative.java c076ae7
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POProject.java 8887133
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserComparisonFunc.java 479eb83
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java 3c7e741
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Subtract.java 79d4c73
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POForEach.java bf2ba08
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POLocalRearrange.java ddb25f1
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPartialAgg.java aa11409
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPreCombinerLocalRearrange.java 52401eb
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POSort.java ad33e7b
> src/org/apache/pig/backend/hadoop/hbase/HBaseBinaryConverter.java 60a5899
> src/org/apache/pig/backend/hadoop/hbase/HBaseStorage.java a6f4ea6
> src/org/apache/pig/builtin/ABS.java 8a7c631
> src/org/apache/pig/builtin/BigDecimalAbs.java PRE-CREATION
> src/org/apache/pig/builtin/BigIntegerAbs.java PRE-CREATION
> src/org/apache/pig/builtin/BinStorage.java 38b4492
> src/org/apache/pig/builtin/TextLoader.java d5bcf02
> src/org/apache/pig/builtin/Utf8StorageConverter.java da12ed6
> src/org/apache/pig/data/BinInterSedes.java e851d8b
> src/org/apache/pig/data/DataReaderWriter.java 37a162a
> src/org/apache/pig/data/DataType.java e4c7b98
> src/org/apache/pig/data/DefaultTuple.java e182b06
> src/org/apache/pig/data/SizeUtil.java 90e5d94
> src/org/apache/pig/data/TypeAwareTuple.java cf78d0a
> src/org/apache/pig/impl/io/NullableBigDecimalWritable.java PRE-CREATION
> src/org/apache/pig/impl/io/NullableBigIntegerWritable.java PRE-CREATION
> src/org/apache/pig/impl/logicalLayer/schema/SchemaUtil.java c257ada
> src/org/apache/pig/impl/util/CastUtils.java 309130a
> src/org/apache/pig/impl/util/NumValCarrier.java af519fd
> src/org/apache/pig/impl/util/StorageUtil.java 087651a
> src/org/apache/pig/newplan/logical/rules/ConstExpEvaluator.java f968294
> src/org/apache/pig/newplan/logical/visitor/TypeCheckingExpVisitor.java d7a1370
> src/org/apache/pig/parser/AliasMasker.g 8cb1b30
> src/org/apache/pig/parser/AstPrinter.g c6f7ff8
> src/org/apache/pig/parser/AstValidator.g 8646078
> src/org/apache/pig/parser/LogicalPlanBuilder.java 699f7a6
> src/org/apache/pig/parser/LogicalPlanGenerator.g 9b9c099
> src/org/apache/pig/parser/QueryLexer.g f201916
> src/org/apache/pig/parser/QueryParser.g 642884e
> src/org/apache/pig/pen/AugmentBaseDataVisitor.java 57dcb14
> test/e2e/pig/udfs/java/org/apache/pig/test/udf/storefunc/PigPerformanceLoader.java 94c28be
> test/org/apache/pig/test/TestAdd.java de9e90d
> test/org/apache/pig/test/TestBestFitCast.java e3172d2
> test/org/apache/pig/test/TestPOCast.java 72787b3
>
> Diff: https://reviews.apache.org/r/9012/diff/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Jonathan Coveney
>
>
Re: Review Request: Add BigInteger and BigDecimal to Pig
Posted by Jonathan Coveney <jc...@gmail.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/9012/
-----------------------------------------------------------
(Updated Jan. 22, 2013, 10:05 p.m.)
Review request for pig, Alan Gates and Mathias Herberts.
Description
-------
This patch adds big integer and big decimal support to Pig. It could use more tests, something I'd appreciate feedback on (but I wanted to make sure the core implementation is good)
This addresses bug PIG-2764.
https://issues.apache.org/jira/browse/PIG-2764
Diffs (updated)
-----
.gitignore cc62d7d
src/org/apache/pig/LoadCaster.java 574769b
src/org/apache/pig/PigWarning.java 5de075f
src/org/apache/pig/StoreCaster.java 5fe48de
src/org/apache/pig/backend/hadoop/BigDecimalWritable.java PRE-CREATION
src/org/apache/pig/backend/hadoop/BigIntegerWritable.java PRE-CREATION
src/org/apache/pig/backend/hadoop/HDataType.java 84a56b8
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java 96fba6b
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBigDecimalRawComparator.java PRE-CREATION
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBigIntegerRawComparator.java PRE-CREATION
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/partitioners/WeightedRangePartitioner.java 9749339
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PhysicalOperator.java f40eb43
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Add.java c84b767
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ConstantExpression.java db3840f
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Divide.java 4656c28
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/EqualToExpr.java 6683beb
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ExpressionOperator.java 2806336
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/GTOrEqualToExpr.java d64a080
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/GreaterThanExpr.java 704d0b8
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/LTOrEqualToExpr.java 9dc929e
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/LessThanExpr.java 0320698
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Mod.java 6819185
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Multiply.java 7b57bed
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/NotEqualToExpr.java 79a4461
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POBinCond.java 08544d5
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POCast.java e8c2f2c
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POIsNull.java f20b839
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/PONegative.java c076ae7
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POProject.java 8887133
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserComparisonFunc.java 479eb83
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java 3c7e741
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Subtract.java 79d4c73
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POForEach.java bf2ba08
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POLocalRearrange.java ddb25f1
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPartialAgg.java aa11409
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPreCombinerLocalRearrange.java 52401eb
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POSort.java ad33e7b
src/org/apache/pig/backend/hadoop/hbase/HBaseBinaryConverter.java 60a5899
src/org/apache/pig/backend/hadoop/hbase/HBaseStorage.java a6f4ea6
src/org/apache/pig/builtin/ABS.java 8a7c631
src/org/apache/pig/builtin/BigDecimalAbs.java PRE-CREATION
src/org/apache/pig/builtin/BigIntegerAbs.java PRE-CREATION
src/org/apache/pig/builtin/BinStorage.java 38b4492
src/org/apache/pig/builtin/TextLoader.java d5bcf02
src/org/apache/pig/builtin/Utf8StorageConverter.java da12ed6
src/org/apache/pig/data/BinInterSedes.java e851d8b
src/org/apache/pig/data/DataReaderWriter.java 37a162a
src/org/apache/pig/data/DataType.java e4c7b98
src/org/apache/pig/data/DefaultTuple.java e182b06
src/org/apache/pig/data/SizeUtil.java 90e5d94
src/org/apache/pig/data/TypeAwareTuple.java cf78d0a
src/org/apache/pig/impl/io/NullableBigDecimalWritable.java PRE-CREATION
src/org/apache/pig/impl/io/NullableBigIntegerWritable.java PRE-CREATION
src/org/apache/pig/impl/logicalLayer/schema/SchemaUtil.java c257ada
src/org/apache/pig/impl/util/CastUtils.java 309130a
src/org/apache/pig/impl/util/NumValCarrier.java af519fd
src/org/apache/pig/impl/util/StorageUtil.java 087651a
src/org/apache/pig/newplan/logical/rules/ConstExpEvaluator.java f968294
src/org/apache/pig/newplan/logical/visitor/TypeCheckingExpVisitor.java d7a1370
src/org/apache/pig/parser/AliasMasker.g 8cb1b30
src/org/apache/pig/parser/AstPrinter.g c6f7ff8
src/org/apache/pig/parser/AstValidator.g 8646078
src/org/apache/pig/parser/LogicalPlanBuilder.java 699f7a6
src/org/apache/pig/parser/LogicalPlanGenerator.g 9b9c099
src/org/apache/pig/parser/QueryLexer.g f201916
src/org/apache/pig/parser/QueryParser.g 642884e
src/org/apache/pig/pen/AugmentBaseDataVisitor.java 57dcb14
test/e2e/pig/udfs/java/org/apache/pig/test/udf/storefunc/PigPerformanceLoader.java 94c28be
test/org/apache/pig/test/TestAdd.java de9e90d
test/org/apache/pig/test/TestBestFitCast.java e3172d2
test/org/apache/pig/test/TestPOCast.java 72787b3
Diff: https://reviews.apache.org/r/9012/diff/
Testing
-------
Thanks,
Jonathan Coveney
Re: Review Request: Add BigInteger and BigDecimal to Pig
Posted by Jonathan Coveney <jc...@gmail.com>.
> On Jan. 21, 2013, 6:27 p.m., Alan Gates wrote:
> > src/org/apache/pig/backend/hadoop/hbase/HBaseBinaryConverter.java, line 192
> > <https://reviews.apache.org/r/9012/diff/2/?file=250198#file250198line192>
> >
> > Is this because HBase doesn't have a standard representation for this?
Yes, that's how I understood it
> On Jan. 21, 2013, 6:27 p.m., Alan Gates wrote:
> > src/org/apache/pig/builtin/BinStorage.java, line 143
> > <https://reviews.apache.org/r/9012/diff/2/?file=250203#file250203line143>
> >
> > Why not implement these? You have read and write functions in the BytesWritable implementations.
It's specifically a load caster that has nothing implemented
> On Jan. 21, 2013, 6:27 p.m., Alan Gates wrote:
> > src/org/apache/pig/builtin/TextLoader.java, line 249
> > <https://reviews.apache.org/r/9012/diff/2/?file=250204#file250204line249>
> >
> > Again, why not implement these? There are a string to bigint and bigdecimal functions.
In this case it is because of how the TextLoader works. The whole point of the textloader is that you don't try to cast anything, and you just load each line as text. Note that the only implemented one is bytesToCharArray
> On Jan. 21, 2013, 6:27 p.m., Alan Gates wrote:
> > src/org/apache/pig/data/BinInterSedes.java, line 908
> > <https://reviews.apache.org/r/9012/diff/2/?file=250206#file250206line908>
> >
> > I wonder if there's a way to avoid conversions to and from and strings here. That can hardly be efficient. I don't think it's something we need to change now but down the road maybe something to think about.
I wholeheartedly agree. I think we should make a JIRA for this. I didn't sweat it since raging efficiency isn't what this is about anyway, but we definitely could do something much smarter. It would not be hard.
> On Jan. 21, 2013, 6:27 p.m., Alan Gates wrote:
> > src/org/apache/pig/data/TypeAwareTuple.java, line 39
> > <https://reviews.apache.org/r/9012/diff/2/?file=250211#file250211line39>
> >
> > Seems like the second argument here should be a BigInteger, not a boolean. Same comment for the next line.
Oh wow lol. Copy-paste strikes. Good catch.
- Jonathan
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/9012/#review15544
-----------------------------------------------------------
On Jan. 18, 2013, 10:11 p.m., Jonathan Coveney wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/9012/
> -----------------------------------------------------------
>
> (Updated Jan. 18, 2013, 10:11 p.m.)
>
>
> Review request for pig, Alan Gates and Mathias Herberts.
>
>
> Description
> -------
>
> This patch adds big integer and big decimal support to Pig. It could use more tests, something I'd appreciate feedback on (but I wanted to make sure the core implementation is good)
>
>
> This addresses bug PIG-2764.
> https://issues.apache.org/jira/browse/PIG-2764
>
>
> Diffs
> -----
>
> .gitignore cc62d7d
> src/org/apache/pig/LoadCaster.java 574769b
> src/org/apache/pig/PigWarning.java 5de075f
> src/org/apache/pig/StoreCaster.java 5fe48de
> src/org/apache/pig/backend/hadoop/BigDecimalWritable.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/BigIntegerWritable.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/HDataType.java 84a56b8
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java 96fba6b
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBigDecimalRawComparator.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBigIntegerRawComparator.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/partitioners/WeightedRangePartitioner.java 9749339
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PhysicalOperator.java f40eb43
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Add.java c84b767
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ConstantExpression.java db3840f
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Divide.java 4656c28
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/EqualToExpr.java 6683beb
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ExpressionOperator.java 2806336
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/GTOrEqualToExpr.java d64a080
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/GreaterThanExpr.java 704d0b8
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/LTOrEqualToExpr.java 9dc929e
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/LessThanExpr.java 0320698
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Mod.java 6819185
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Multiply.java 7b57bed
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/NotEqualToExpr.java 79a4461
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POBinCond.java 08544d5
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POCast.java e8c2f2c
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POIsNull.java f20b839
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/PONegative.java c076ae7
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POProject.java 8887133
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserComparisonFunc.java 479eb83
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java 3c7e741
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Subtract.java 79d4c73
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POForEach.java bf2ba08
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POLocalRearrange.java ddb25f1
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPartialAgg.java aa11409
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPreCombinerLocalRearrange.java 52401eb
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POSort.java ad33e7b
> src/org/apache/pig/backend/hadoop/hbase/HBaseBinaryConverter.java 60a5899
> src/org/apache/pig/backend/hadoop/hbase/HBaseStorage.java a6f4ea6
> src/org/apache/pig/builtin/ABS.java 8a7c631
> src/org/apache/pig/builtin/BigDecimalAbs.java PRE-CREATION
> src/org/apache/pig/builtin/BigIntegerAbs.java PRE-CREATION
> src/org/apache/pig/builtin/BinStorage.java 38b4492
> src/org/apache/pig/builtin/TextLoader.java d5bcf02
> src/org/apache/pig/builtin/Utf8StorageConverter.java da12ed6
> src/org/apache/pig/data/BinInterSedes.java e851d8b
> src/org/apache/pig/data/DataReaderWriter.java 37a162a
> src/org/apache/pig/data/DataType.java e4c7b98
> src/org/apache/pig/data/DefaultTuple.java e182b06
> src/org/apache/pig/data/SizeUtil.java 90e5d94
> src/org/apache/pig/data/TypeAwareTuple.java cf78d0a
> src/org/apache/pig/impl/io/NullableBigDecimalWritable.java PRE-CREATION
> src/org/apache/pig/impl/io/NullableBigIntegerWritable.java PRE-CREATION
> src/org/apache/pig/impl/logicalLayer/schema/SchemaUtil.java c257ada
> src/org/apache/pig/impl/util/CastUtils.java 309130a
> src/org/apache/pig/impl/util/NumValCarrier.java af519fd
> src/org/apache/pig/impl/util/StorageUtil.java 087651a
> src/org/apache/pig/newplan/logical/rules/ConstExpEvaluator.java f968294
> src/org/apache/pig/newplan/logical/visitor/TypeCheckingExpVisitor.java d7a1370
> src/org/apache/pig/parser/AliasMasker.g 8cb1b30
> src/org/apache/pig/parser/AstPrinter.g c6f7ff8
> src/org/apache/pig/parser/AstValidator.g 8646078
> src/org/apache/pig/parser/LogicalPlanBuilder.java 699f7a6
> src/org/apache/pig/parser/LogicalPlanGenerator.g 9b9c099
> src/org/apache/pig/parser/QueryLexer.g f201916
> src/org/apache/pig/parser/QueryParser.g 642884e
> src/org/apache/pig/pen/AugmentBaseDataVisitor.java 57dcb14
> test/e2e/pig/udfs/java/org/apache/pig/test/udf/storefunc/PigPerformanceLoader.java 94c28be
> test/org/apache/pig/test/TestAdd.java de9e90d
> test/org/apache/pig/test/TestBestFitCast.java e3172d2
> test/org/apache/pig/test/TestPOCast.java 72787b3
>
> Diff: https://reviews.apache.org/r/9012/diff/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Jonathan Coveney
>
>
Re: Review Request: Add BigInteger and BigDecimal to Pig
Posted by Alan Gates <ga...@hortonworks.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/9012/#review15544
-----------------------------------------------------------
Looks good. The only comment that I think needs to be addressed before checking it in is the one on TypeAwareTuple.java. After that I'm +1 on checking this in. The rest are questions or JIRAs can be filed for later.
I also think we should file a JIRA to add e2e tests for these two types and make it a blocker for 0.12. I know with the datetime stuff we found a bunch of new issues when we added those tests.
As a general question, I'm trying to figure out the utility of BigInteger. I totally get BigDecimal, since people want fixed precision calculations. BigInteger would be useful if you want integer semantics for numbers larger than 2^63, but I can't think of a use case for that off the top of my head. I'm not saying we shouldn't do it, I just wonder if it will be used.
src/org/apache/pig/backend/hadoop/hbase/HBaseBinaryConverter.java
<https://reviews.apache.org/r/9012/#comment33547>
Is this because HBase doesn't have a standard representation for this?
src/org/apache/pig/builtin/BinStorage.java
<https://reviews.apache.org/r/9012/#comment33548>
Why not implement these? You have read and write functions in the BytesWritable implementations.
src/org/apache/pig/builtin/TextLoader.java
<https://reviews.apache.org/r/9012/#comment33549>
Again, why not implement these? There are a string to bigint and bigdecimal functions.
src/org/apache/pig/data/BinInterSedes.java
<https://reviews.apache.org/r/9012/#comment33550>
I wonder if there's a way to avoid conversions to and from and strings here. That can hardly be efficient. I don't think it's something we need to change now but down the road maybe something to think about.
src/org/apache/pig/data/TypeAwareTuple.java
<https://reviews.apache.org/r/9012/#comment33551>
Seems like the second argument here should be a BigInteger, not a boolean. Same comment for the next line.
- Alan Gates
On Jan. 18, 2013, 10:11 p.m., Jonathan Coveney wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/9012/
> -----------------------------------------------------------
>
> (Updated Jan. 18, 2013, 10:11 p.m.)
>
>
> Review request for pig, Alan Gates and Mathias Herberts.
>
>
> Description
> -------
>
> This patch adds big integer and big decimal support to Pig. It could use more tests, something I'd appreciate feedback on (but I wanted to make sure the core implementation is good)
>
>
> This addresses bug PIG-2764.
> https://issues.apache.org/jira/browse/PIG-2764
>
>
> Diffs
> -----
>
> .gitignore cc62d7d
> src/org/apache/pig/LoadCaster.java 574769b
> src/org/apache/pig/PigWarning.java 5de075f
> src/org/apache/pig/StoreCaster.java 5fe48de
> src/org/apache/pig/backend/hadoop/BigDecimalWritable.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/BigIntegerWritable.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/HDataType.java 84a56b8
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java 96fba6b
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBigDecimalRawComparator.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBigIntegerRawComparator.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/partitioners/WeightedRangePartitioner.java 9749339
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PhysicalOperator.java f40eb43
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Add.java c84b767
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ConstantExpression.java db3840f
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Divide.java 4656c28
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/EqualToExpr.java 6683beb
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ExpressionOperator.java 2806336
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/GTOrEqualToExpr.java d64a080
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/GreaterThanExpr.java 704d0b8
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/LTOrEqualToExpr.java 9dc929e
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/LessThanExpr.java 0320698
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Mod.java 6819185
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Multiply.java 7b57bed
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/NotEqualToExpr.java 79a4461
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POBinCond.java 08544d5
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POCast.java e8c2f2c
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POIsNull.java f20b839
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/PONegative.java c076ae7
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POProject.java 8887133
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserComparisonFunc.java 479eb83
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java 3c7e741
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Subtract.java 79d4c73
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POForEach.java bf2ba08
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POLocalRearrange.java ddb25f1
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPartialAgg.java aa11409
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPreCombinerLocalRearrange.java 52401eb
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POSort.java ad33e7b
> src/org/apache/pig/backend/hadoop/hbase/HBaseBinaryConverter.java 60a5899
> src/org/apache/pig/backend/hadoop/hbase/HBaseStorage.java a6f4ea6
> src/org/apache/pig/builtin/ABS.java 8a7c631
> src/org/apache/pig/builtin/BigDecimalAbs.java PRE-CREATION
> src/org/apache/pig/builtin/BigIntegerAbs.java PRE-CREATION
> src/org/apache/pig/builtin/BinStorage.java 38b4492
> src/org/apache/pig/builtin/TextLoader.java d5bcf02
> src/org/apache/pig/builtin/Utf8StorageConverter.java da12ed6
> src/org/apache/pig/data/BinInterSedes.java e851d8b
> src/org/apache/pig/data/DataReaderWriter.java 37a162a
> src/org/apache/pig/data/DataType.java e4c7b98
> src/org/apache/pig/data/DefaultTuple.java e182b06
> src/org/apache/pig/data/SizeUtil.java 90e5d94
> src/org/apache/pig/data/TypeAwareTuple.java cf78d0a
> src/org/apache/pig/impl/io/NullableBigDecimalWritable.java PRE-CREATION
> src/org/apache/pig/impl/io/NullableBigIntegerWritable.java PRE-CREATION
> src/org/apache/pig/impl/logicalLayer/schema/SchemaUtil.java c257ada
> src/org/apache/pig/impl/util/CastUtils.java 309130a
> src/org/apache/pig/impl/util/NumValCarrier.java af519fd
> src/org/apache/pig/impl/util/StorageUtil.java 087651a
> src/org/apache/pig/newplan/logical/rules/ConstExpEvaluator.java f968294
> src/org/apache/pig/newplan/logical/visitor/TypeCheckingExpVisitor.java d7a1370
> src/org/apache/pig/parser/AliasMasker.g 8cb1b30
> src/org/apache/pig/parser/AstPrinter.g c6f7ff8
> src/org/apache/pig/parser/AstValidator.g 8646078
> src/org/apache/pig/parser/LogicalPlanBuilder.java 699f7a6
> src/org/apache/pig/parser/LogicalPlanGenerator.g 9b9c099
> src/org/apache/pig/parser/QueryLexer.g f201916
> src/org/apache/pig/parser/QueryParser.g 642884e
> src/org/apache/pig/pen/AugmentBaseDataVisitor.java 57dcb14
> test/e2e/pig/udfs/java/org/apache/pig/test/udf/storefunc/PigPerformanceLoader.java 94c28be
> test/org/apache/pig/test/TestAdd.java de9e90d
> test/org/apache/pig/test/TestBestFitCast.java e3172d2
> test/org/apache/pig/test/TestPOCast.java 72787b3
>
> Diff: https://reviews.apache.org/r/9012/diff/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Jonathan Coveney
>
>
Re: Review Request: Add BigInteger and BigDecimal to Pig
Posted by Cheolsoo Park <ch...@cloudera.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/9012/#review15532
-----------------------------------------------------------
src/org/apache/pig/backend/hadoop/BigDecimalWritable.java
<https://reviews.apache.org/r/9012/#comment33533>
I believe this is a typo: 'b1' => 'b2'.
src/org/apache/pig/backend/hadoop/BigDecimalWritable.java
<https://reviews.apache.org/r/9012/#comment33536>
bye => byte
src/org/apache/pig/backend/hadoop/BigIntegerWritable.java
<https://reviews.apache.org/r/9012/#comment33534>
I believe this is a typo: 'b1' => 'b2'.
src/org/apache/pig/backend/hadoop/BigIntegerWritable.java
<https://reviews.apache.org/r/9012/#comment33537>
bye => byte
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POCast.java
<https://reviews.apache.org/r/9012/#comment33538>
Shouldn't it be bytesToBigDecimal?
src/org/apache/pig/data/DataType.java
<https://reviews.apache.org/r/9012/#comment33539>
Typo: BigInteger => BigDecimal.
- Cheolsoo Park
On Jan. 18, 2013, 10:11 p.m., Jonathan Coveney wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/9012/
> -----------------------------------------------------------
>
> (Updated Jan. 18, 2013, 10:11 p.m.)
>
>
> Review request for pig, Alan Gates and Mathias Herberts.
>
>
> Description
> -------
>
> This patch adds big integer and big decimal support to Pig. It could use more tests, something I'd appreciate feedback on (but I wanted to make sure the core implementation is good)
>
>
> This addresses bug PIG-2764.
> https://issues.apache.org/jira/browse/PIG-2764
>
>
> Diffs
> -----
>
> .gitignore cc62d7d
> src/org/apache/pig/LoadCaster.java 574769b
> src/org/apache/pig/PigWarning.java 5de075f
> src/org/apache/pig/StoreCaster.java 5fe48de
> src/org/apache/pig/backend/hadoop/BigDecimalWritable.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/BigIntegerWritable.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/HDataType.java 84a56b8
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java 96fba6b
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBigDecimalRawComparator.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBigIntegerRawComparator.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/partitioners/WeightedRangePartitioner.java 9749339
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PhysicalOperator.java f40eb43
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Add.java c84b767
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ConstantExpression.java db3840f
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Divide.java 4656c28
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/EqualToExpr.java 6683beb
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ExpressionOperator.java 2806336
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/GTOrEqualToExpr.java d64a080
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/GreaterThanExpr.java 704d0b8
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/LTOrEqualToExpr.java 9dc929e
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/LessThanExpr.java 0320698
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Mod.java 6819185
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Multiply.java 7b57bed
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/NotEqualToExpr.java 79a4461
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POBinCond.java 08544d5
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POCast.java e8c2f2c
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POIsNull.java f20b839
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/PONegative.java c076ae7
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POProject.java 8887133
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserComparisonFunc.java 479eb83
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java 3c7e741
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Subtract.java 79d4c73
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POForEach.java bf2ba08
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POLocalRearrange.java ddb25f1
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPartialAgg.java aa11409
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPreCombinerLocalRearrange.java 52401eb
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POSort.java ad33e7b
> src/org/apache/pig/backend/hadoop/hbase/HBaseBinaryConverter.java 60a5899
> src/org/apache/pig/backend/hadoop/hbase/HBaseStorage.java a6f4ea6
> src/org/apache/pig/builtin/ABS.java 8a7c631
> src/org/apache/pig/builtin/BigDecimalAbs.java PRE-CREATION
> src/org/apache/pig/builtin/BigIntegerAbs.java PRE-CREATION
> src/org/apache/pig/builtin/BinStorage.java 38b4492
> src/org/apache/pig/builtin/TextLoader.java d5bcf02
> src/org/apache/pig/builtin/Utf8StorageConverter.java da12ed6
> src/org/apache/pig/data/BinInterSedes.java e851d8b
> src/org/apache/pig/data/DataReaderWriter.java 37a162a
> src/org/apache/pig/data/DataType.java e4c7b98
> src/org/apache/pig/data/DefaultTuple.java e182b06
> src/org/apache/pig/data/SizeUtil.java 90e5d94
> src/org/apache/pig/data/TypeAwareTuple.java cf78d0a
> src/org/apache/pig/impl/io/NullableBigDecimalWritable.java PRE-CREATION
> src/org/apache/pig/impl/io/NullableBigIntegerWritable.java PRE-CREATION
> src/org/apache/pig/impl/logicalLayer/schema/SchemaUtil.java c257ada
> src/org/apache/pig/impl/util/CastUtils.java 309130a
> src/org/apache/pig/impl/util/NumValCarrier.java af519fd
> src/org/apache/pig/impl/util/StorageUtil.java 087651a
> src/org/apache/pig/newplan/logical/rules/ConstExpEvaluator.java f968294
> src/org/apache/pig/newplan/logical/visitor/TypeCheckingExpVisitor.java d7a1370
> src/org/apache/pig/parser/AliasMasker.g 8cb1b30
> src/org/apache/pig/parser/AstPrinter.g c6f7ff8
> src/org/apache/pig/parser/AstValidator.g 8646078
> src/org/apache/pig/parser/LogicalPlanBuilder.java 699f7a6
> src/org/apache/pig/parser/LogicalPlanGenerator.g 9b9c099
> src/org/apache/pig/parser/QueryLexer.g f201916
> src/org/apache/pig/parser/QueryParser.g 642884e
> src/org/apache/pig/pen/AugmentBaseDataVisitor.java 57dcb14
> test/e2e/pig/udfs/java/org/apache/pig/test/udf/storefunc/PigPerformanceLoader.java 94c28be
> test/org/apache/pig/test/TestAdd.java de9e90d
> test/org/apache/pig/test/TestBestFitCast.java e3172d2
> test/org/apache/pig/test/TestPOCast.java 72787b3
>
> Diff: https://reviews.apache.org/r/9012/diff/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Jonathan Coveney
>
>
Re: Review Request: Add BigInteger and BigDecimal to Pig
Posted by Jonathan Coveney <jc...@gmail.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/9012/
-----------------------------------------------------------
(Updated Jan. 18, 2013, 10:11 p.m.)
Review request for pig, Alan Gates and Mathias Herberts.
Description
-------
This patch adds big integer and big decimal support to Pig. It could use more tests, something I'd appreciate feedback on (but I wanted to make sure the core implementation is good)
This addresses bug PIG-2764.
https://issues.apache.org/jira/browse/PIG-2764
Diffs (updated)
-----
.gitignore cc62d7d
src/org/apache/pig/LoadCaster.java 574769b
src/org/apache/pig/PigWarning.java 5de075f
src/org/apache/pig/StoreCaster.java 5fe48de
src/org/apache/pig/backend/hadoop/BigDecimalWritable.java PRE-CREATION
src/org/apache/pig/backend/hadoop/BigIntegerWritable.java PRE-CREATION
src/org/apache/pig/backend/hadoop/HDataType.java 84a56b8
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java 96fba6b
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBigDecimalRawComparator.java PRE-CREATION
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBigIntegerRawComparator.java PRE-CREATION
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/partitioners/WeightedRangePartitioner.java 9749339
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PhysicalOperator.java f40eb43
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Add.java c84b767
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ConstantExpression.java db3840f
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Divide.java 4656c28
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/EqualToExpr.java 6683beb
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ExpressionOperator.java 2806336
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/GTOrEqualToExpr.java d64a080
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/GreaterThanExpr.java 704d0b8
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/LTOrEqualToExpr.java 9dc929e
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/LessThanExpr.java 0320698
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Mod.java 6819185
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Multiply.java 7b57bed
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/NotEqualToExpr.java 79a4461
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POBinCond.java 08544d5
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POCast.java e8c2f2c
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POIsNull.java f20b839
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/PONegative.java c076ae7
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POProject.java 8887133
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserComparisonFunc.java 479eb83
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java 3c7e741
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Subtract.java 79d4c73
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POForEach.java bf2ba08
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POLocalRearrange.java ddb25f1
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPartialAgg.java aa11409
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPreCombinerLocalRearrange.java 52401eb
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POSort.java ad33e7b
src/org/apache/pig/backend/hadoop/hbase/HBaseBinaryConverter.java 60a5899
src/org/apache/pig/backend/hadoop/hbase/HBaseStorage.java a6f4ea6
src/org/apache/pig/builtin/ABS.java 8a7c631
src/org/apache/pig/builtin/BigDecimalAbs.java PRE-CREATION
src/org/apache/pig/builtin/BigIntegerAbs.java PRE-CREATION
src/org/apache/pig/builtin/BinStorage.java 38b4492
src/org/apache/pig/builtin/TextLoader.java d5bcf02
src/org/apache/pig/builtin/Utf8StorageConverter.java da12ed6
src/org/apache/pig/data/BinInterSedes.java e851d8b
src/org/apache/pig/data/DataReaderWriter.java 37a162a
src/org/apache/pig/data/DataType.java e4c7b98
src/org/apache/pig/data/DefaultTuple.java e182b06
src/org/apache/pig/data/SizeUtil.java 90e5d94
src/org/apache/pig/data/TypeAwareTuple.java cf78d0a
src/org/apache/pig/impl/io/NullableBigDecimalWritable.java PRE-CREATION
src/org/apache/pig/impl/io/NullableBigIntegerWritable.java PRE-CREATION
src/org/apache/pig/impl/logicalLayer/schema/SchemaUtil.java c257ada
src/org/apache/pig/impl/util/CastUtils.java 309130a
src/org/apache/pig/impl/util/NumValCarrier.java af519fd
src/org/apache/pig/impl/util/StorageUtil.java 087651a
src/org/apache/pig/newplan/logical/rules/ConstExpEvaluator.java f968294
src/org/apache/pig/newplan/logical/visitor/TypeCheckingExpVisitor.java d7a1370
src/org/apache/pig/parser/AliasMasker.g 8cb1b30
src/org/apache/pig/parser/AstPrinter.g c6f7ff8
src/org/apache/pig/parser/AstValidator.g 8646078
src/org/apache/pig/parser/LogicalPlanBuilder.java 699f7a6
src/org/apache/pig/parser/LogicalPlanGenerator.g 9b9c099
src/org/apache/pig/parser/QueryLexer.g f201916
src/org/apache/pig/parser/QueryParser.g 642884e
src/org/apache/pig/pen/AugmentBaseDataVisitor.java 57dcb14
test/e2e/pig/udfs/java/org/apache/pig/test/udf/storefunc/PigPerformanceLoader.java 94c28be
test/org/apache/pig/test/TestAdd.java de9e90d
test/org/apache/pig/test/TestBestFitCast.java e3172d2
test/org/apache/pig/test/TestPOCast.java 72787b3
Diff: https://reviews.apache.org/r/9012/diff/
Testing
-------
Thanks,
Jonathan Coveney
Re: Review Request: Add BigInteger and BigDecimal to Pig
Posted by Jonathan Coveney <jc...@gmail.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/9012/#review15496
-----------------------------------------------------------
src/org/apache/pig/backend/hadoop/hbase/HBaseBinaryConverter.java
<https://reviews.apache.org/r/9012/#comment33457>
I'm not sure what the contract of HBaseBinaryConverter is, which is why I didn't implement it. Theoretically, any Pig type can be serialized to Bytes since we do this in BinInterSedes, but here they were very particular to only use HBase's serialization, leading me to believe we should only convert bytes in the case where they can be transformed into the HBase analogue. We should find someone who knows more about this code, or dig around to see how it is used.
src/org/apache/pig/builtin/Utf8StorageConverter.java
<https://reviews.apache.org/r/9012/#comment33458>
None of the other calls in this class specified the charset, so I did not either. Is there a reason why in this case specifically we need to, given we did bi => string => bytes?
src/org/apache/pig/data/DataType.java
<https://reviews.apache.org/r/9012/#comment33459>
Odd. I think this is because of the preexisting presence of tabs in the file, which I will remove.
src/org/apache/pig/data/DefaultTuple.java
<https://reviews.apache.org/r/9012/#comment33460>
This is janky, but to understand what is going on you need to look at DataReaderWriter to see how BigInteger and BigDecimal are being serialized in this specific case (which is not BinInterSedes, and in fact, I don't think this code is called anywhere...but still, it should be correct. We should probably unify all the serialization under BinInterSedes if possible, but that's a JIRA for another day). So basically, in this case we serialize as a String, and then decode. I'm going to update it to serialize the bytes directly...no reason not to.
src/org/apache/pig/data/SizeUtil.java
<https://reviews.apache.org/r/9012/#comment33464>
Thus the TODO :S I completely punted on figuring out how big they are. I will try and tackle that now...thanks for the link, it will definitely help a lot
src/org/apache/pig/impl/util/StorageUtil.java
<https://reviews.apache.org/r/9012/#comment33469>
Same comment here as earlier. It may need to be, but I'm unsure, and if it does, why doesn't it need to be in the other cases?
- Jonathan Coveney
On Jan. 17, 2013, 9:02 p.m., Jonathan Coveney wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/9012/
> -----------------------------------------------------------
>
> (Updated Jan. 17, 2013, 9:02 p.m.)
>
>
> Review request for pig, Alan Gates and Mathias Herberts.
>
>
> Description
> -------
>
> This patch adds big integer and big decimal support to Pig. It could use more tests, something I'd appreciate feedback on (but I wanted to make sure the core implementation is good)
>
>
> This addresses bug PIG-2764.
> https://issues.apache.org/jira/browse/PIG-2764
>
>
> Diffs
> -----
>
> .gitignore cc62d7d
> src/org/apache/pig/LoadCaster.java 574769b
> src/org/apache/pig/PigWarning.java 5de075f
> src/org/apache/pig/StoreCaster.java 5fe48de
> src/org/apache/pig/backend/hadoop/BigDecimalWritable.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/BigIntegerWritable.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/HDataType.java 84a56b8
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java 96fba6b
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBigDecimalRawComparator.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBigIntegerRawComparator.java PRE-CREATION
> src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/partitioners/WeightedRangePartitioner.java 9749339
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PhysicalOperator.java f40eb43
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Add.java c84b767
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ConstantExpression.java db3840f
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Divide.java 4656c28
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/EqualToExpr.java 6683beb
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ExpressionOperator.java 2806336
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/GTOrEqualToExpr.java d64a080
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/GreaterThanExpr.java 704d0b8
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/LTOrEqualToExpr.java 9dc929e
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/LessThanExpr.java 0320698
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Mod.java 6819185
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Multiply.java 7b57bed
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/NotEqualToExpr.java 79a4461
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POBinCond.java 08544d5
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POCast.java e8c2f2c
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POIsNull.java f20b839
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/PONegative.java c076ae7
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POProject.java 8887133
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserComparisonFunc.java 479eb83
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java 3c7e741
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/Subtract.java 79d4c73
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POForEach.java bf2ba08
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POLocalRearrange.java ddb25f1
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPartialAgg.java aa11409
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPreCombinerLocalRearrange.java 52401eb
> src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POSort.java ad33e7b
> src/org/apache/pig/backend/hadoop/hbase/HBaseBinaryConverter.java 60a5899
> src/org/apache/pig/backend/hadoop/hbase/HBaseStorage.java a6f4ea6
> src/org/apache/pig/builtin/ABS.java 8a7c631
> src/org/apache/pig/builtin/BigDecimalAbs.java PRE-CREATION
> src/org/apache/pig/builtin/BigIntegerAbs.java PRE-CREATION
> src/org/apache/pig/builtin/BinStorage.java 38b4492
> src/org/apache/pig/builtin/TextLoader.java d5bcf02
> src/org/apache/pig/builtin/Utf8StorageConverter.java da12ed6
> src/org/apache/pig/data/BinInterSedes.java e851d8b
> src/org/apache/pig/data/DataReaderWriter.java 37a162a
> src/org/apache/pig/data/DataType.java e4c7b98
> src/org/apache/pig/data/DefaultTuple.java e182b06
> src/org/apache/pig/data/SizeUtil.java 90e5d94
> src/org/apache/pig/data/TypeAwareTuple.java cf78d0a
> src/org/apache/pig/impl/io/NullableBigDecimalWritable.java PRE-CREATION
> src/org/apache/pig/impl/io/NullableBigIntegerWritable.java PRE-CREATION
> src/org/apache/pig/impl/logicalLayer/schema/SchemaUtil.java c257ada
> src/org/apache/pig/impl/util/CastUtils.java 309130a
> src/org/apache/pig/impl/util/NumValCarrier.java af519fd
> src/org/apache/pig/impl/util/StorageUtil.java 087651a
> src/org/apache/pig/newplan/logical/rules/ConstExpEvaluator.java f968294
> src/org/apache/pig/newplan/logical/visitor/TypeCheckingExpVisitor.java d7a1370
> src/org/apache/pig/parser/AliasMasker.g 8cb1b30
> src/org/apache/pig/parser/AstPrinter.g c6f7ff8
> src/org/apache/pig/parser/AstValidator.g 8646078
> src/org/apache/pig/parser/LogicalPlanBuilder.java 699f7a6
> src/org/apache/pig/parser/LogicalPlanGenerator.g 9b9c099
> src/org/apache/pig/parser/QueryLexer.g f201916
> src/org/apache/pig/parser/QueryParser.g 642884e
> src/org/apache/pig/pen/AugmentBaseDataVisitor.java 57dcb14
> test/e2e/pig/udfs/java/org/apache/pig/test/udf/storefunc/PigPerformanceLoader.java 94c28be
> test/org/apache/pig/test/TestAdd.java de9e90d
> test/org/apache/pig/test/TestBestFitCast.java e3172d2
> test/org/apache/pig/test/TestPOCast.java 72787b3
>
> Diff: https://reviews.apache.org/r/9012/diff/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Jonathan Coveney
>
>