You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by "Cathy Daw (JIRA)" <ji...@apache.org> on 2012/07/23 20:52:35 UTC
[jira] [Created] (CASSANDRA-4459) pig driver casts ints as
bytearray
Cathy Daw created CASSANDRA-4459:
------------------------------------
Summary: pig driver casts ints as bytearray
Key: CASSANDRA-4459
URL: https://issues.apache.org/jira/browse/CASSANDRA-4459
Project: Cassandra
Issue Type: Bug
Environment: C* 1.1.2 embedded in DSE
Reporter: Cathy Daw
Assignee: Brandon Williams
we seem to be auto-mapping C* int columns to bytearray in Pig, and farther down I can't seem to find a way to cast that to int and do an average.
{code}
grunt> cassandra_users = LOAD 'cassandra://cqldb/users' USING CassandraStorage();
grunt> dump cassandra_users;
(bobhatter,(act,22),(fname,bob),(gender,m),(highSchool,Cal High),(lname,hatter),(sat,500),(state,CA),{})
(alicesmith,(act,27),(fname,alice),(gender,f),(highSchool,Tuscon High),(lname,smith),(sat,650),(state,AZ),{})
// notice sat and act columns are bytearray values
grunt> describe cassandra_users;
cassandra_users: {key: chararray,act: (name: chararray,value: bytearray),fname: (name: chararray,value: chararray),
gender: (name: chararray,value: chararray),highSchool: (name: chararray,value: chararray),lname: (name: chararray,value: chararray),
sat: (name: chararray,value: bytearray),state: (name: chararray,value: chararray),columns: {(name: chararray,value: chararray)}}
grunt> users_by_state = GROUP cassandra_users BY state;
grunt> dump users_by_state;
((state,AX),{(aoakley,(highSchool,Phoenix High),(lname,Oakley),state,(act,22),(sat,500),(gender,m),(fname,Anne),{})})
((state,AZ),{(gjames,(highSchool,Tuscon High),(lname,James),state,(act,24),(sat,650),(gender,f),(fname,Geronomo),{})})
((state,CA),{(philton,(highSchool,Beverly High),(lname,Hilton),state,(act,37),(sat,220),(gender,m),(fname,Paris),{}),(jbrown,(highSchool,Cal High),(lname,Brown),state,(act,20),(sat,700),(gender,m),(fname,Jerry),{})})
// Error - use explicit cast
grunt> user_avg = FOREACH users_by_state GENERATE cassandra_users.state, AVG(cassandra_users.sat);
grunt> dump user_avg;
2012-07-22 17:15:04,361 [main] ERROR org.apache.pig.tools.grunt.Grunt - ERROR 1045: Could not infer the matching function for org.apache.pig.builtin.AVG as multiple or none of them fit. Please use an explicit cast.
// Unable to cast as int
grunt> user_avg = FOREACH users_by_state GENERATE cassandra_users.state, AVG((int)cassandra_users.sat);
grunt> dump user_avg;
2012-07-22 17:07:39,217 [main] ERROR org.apache.pig.tools.grunt.Grunt - ERROR 1052: Cannot cast bag with schema sat: bag({name: chararray,value: bytearray}) to int
{code}
*Seed data in CQL*
{code}
CREATE KEYSPACE cqldb with
strategy_class = 'org.apache.cassandra.locator.SimpleStrategy'
and strategy_options:replication_factor=3;
use cqldb;
CREATE COLUMNFAMILY users (
KEY text PRIMARY KEY,
fname text, lname text, gender varchar,
act int, sat int, highSchool text, state varchar);
insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
values (gjames, Geronomo, James, f, 24, 650, 'Tuscon High', 'AZ');
insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
values (aoakley, Anne, Oakley, m , 22, 500, 'Phoenix High', 'AX');
insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
values (jbrown, Jerry, Brown, m , 20, 700, 'Cal High', 'CA');
insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
values (philton, Paris, Hilton, m , 37, 220, 'Beverly High', 'CA');
select * from users;
{code}
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators: https://issues.apache.org/jira/secure/ContactAdministrators!default.jspa
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Updated] (CASSANDRA-4459) pig driver casts ints as
bytearray
Posted by "Brandon Williams (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/CASSANDRA-4459?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Brandon Williams updated CASSANDRA-4459:
----------------------------------------
Attachment: 4459.txt
We actually have tests in examples/pig/test that explicitly cover this, but we populate that data with the cli which uses IntegerType, but cql uses Int32Type. Trivial patch to cast both to pig's integer type.
> pig driver casts ints as bytearray
> ----------------------------------
>
> Key: CASSANDRA-4459
> URL: https://issues.apache.org/jira/browse/CASSANDRA-4459
> Project: Cassandra
> Issue Type: Bug
> Environment: C* 1.1.2 embedded in DSE
> Reporter: Cathy Daw
> Assignee: Brandon Williams
> Fix For: 1.1.3
>
> Attachments: 4459.txt
>
>
> we seem to be auto-mapping C* int columns to bytearray in Pig, and farther down I can't seem to find a way to cast that to int and do an average.
> {code}
> grunt> cassandra_users = LOAD 'cassandra://cqldb/users' USING CassandraStorage();
> grunt> dump cassandra_users;
> (bobhatter,(act,22),(fname,bob),(gender,m),(highSchool,Cal High),(lname,hatter),(sat,500),(state,CA),{})
> (alicesmith,(act,27),(fname,alice),(gender,f),(highSchool,Tuscon High),(lname,smith),(sat,650),(state,AZ),{})
>
> // notice sat and act columns are bytearray values
> grunt> describe cassandra_users;
> cassandra_users: {key: chararray,act: (name: chararray,value: bytearray),fname: (name: chararray,value: chararray),
> gender: (name: chararray,value: chararray),highSchool: (name: chararray,value: chararray),lname: (name: chararray,value: chararray),
> sat: (name: chararray,value: bytearray),state: (name: chararray,value: chararray),columns: {(name: chararray,value: chararray)}}
> grunt> users_by_state = GROUP cassandra_users BY state;
> grunt> dump users_by_state;
> ((state,AX),{(aoakley,(highSchool,Phoenix High),(lname,Oakley),state,(act,22),(sat,500),(gender,m),(fname,Anne),{})})
> ((state,AZ),{(gjames,(highSchool,Tuscon High),(lname,James),state,(act,24),(sat,650),(gender,f),(fname,Geronomo),{})})
> ((state,CA),{(philton,(highSchool,Beverly High),(lname,Hilton),state,(act,37),(sat,220),(gender,m),(fname,Paris),{}),(jbrown,(highSchool,Cal High),(lname,Brown),state,(act,20),(sat,700),(gender,m),(fname,Jerry),{})})
> // Error - use explicit cast
> grunt> user_avg = FOREACH users_by_state GENERATE cassandra_users.state, AVG(cassandra_users.sat);
> grunt> dump user_avg;
> 2012-07-22 17:15:04,361 [main] ERROR org.apache.pig.tools.grunt.Grunt - ERROR 1045: Could not infer the matching function for org.apache.pig.builtin.AVG as multiple or none of them fit. Please use an explicit cast.
> // Unable to cast as int
> grunt> user_avg = FOREACH users_by_state GENERATE cassandra_users.state, AVG((int)cassandra_users.sat);
> grunt> dump user_avg;
> 2012-07-22 17:07:39,217 [main] ERROR org.apache.pig.tools.grunt.Grunt - ERROR 1052: Cannot cast bag with schema sat: bag({name: chararray,value: bytearray}) to int
> {code}
> *Seed data in CQL*
> {code}
> CREATE KEYSPACE cqldb with
> strategy_class = 'org.apache.cassandra.locator.SimpleStrategy'
> and strategy_options:replication_factor=3;
> use cqldb;
> CREATE COLUMNFAMILY users (
> KEY text PRIMARY KEY,
> fname text, lname text, gender varchar,
> act int, sat int, highSchool text, state varchar);
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (gjames, Geronomo, James, f, 24, 650, 'Tuscon High', 'AZ');
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (aoakley, Anne, Oakley, m , 22, 500, 'Phoenix High', 'AX');
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (jbrown, Jerry, Brown, m , 20, 700, 'Cal High', 'CA');
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (philton, Paris, Hilton, m , 37, 220, 'Beverly High', 'CA');
> select * from users;
> {code}
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators: https://issues.apache.org/jira/secure/ContactAdministrators!default.jspa
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Commented] (CASSANDRA-4459) pig driver casts ints as
bytearray
Posted by "Jonathan Ellis (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/CASSANDRA-4459?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13421510#comment-13421510 ]
Jonathan Ellis commented on CASSANDRA-4459:
-------------------------------------------
That's reasonable, but we should still use int32type for cli population :)
> pig driver casts ints as bytearray
> ----------------------------------
>
> Key: CASSANDRA-4459
> URL: https://issues.apache.org/jira/browse/CASSANDRA-4459
> Project: Cassandra
> Issue Type: Bug
> Environment: C* 1.1.2 embedded in DSE
> Reporter: Cathy Daw
> Assignee: Brandon Williams
> Fix For: 1.1.3
>
> Attachments: 4459.txt
>
>
> we seem to be auto-mapping C* int columns to bytearray in Pig, and farther down I can't seem to find a way to cast that to int and do an average.
> {code}
> grunt> cassandra_users = LOAD 'cassandra://cqldb/users' USING CassandraStorage();
> grunt> dump cassandra_users;
> (bobhatter,(act,22),(fname,bob),(gender,m),(highSchool,Cal High),(lname,hatter),(sat,500),(state,CA),{})
> (alicesmith,(act,27),(fname,alice),(gender,f),(highSchool,Tuscon High),(lname,smith),(sat,650),(state,AZ),{})
>
> // notice sat and act columns are bytearray values
> grunt> describe cassandra_users;
> cassandra_users: {key: chararray,act: (name: chararray,value: bytearray),fname: (name: chararray,value: chararray),
> gender: (name: chararray,value: chararray),highSchool: (name: chararray,value: chararray),lname: (name: chararray,value: chararray),
> sat: (name: chararray,value: bytearray),state: (name: chararray,value: chararray),columns: {(name: chararray,value: chararray)}}
> grunt> users_by_state = GROUP cassandra_users BY state;
> grunt> dump users_by_state;
> ((state,AX),{(aoakley,(highSchool,Phoenix High),(lname,Oakley),state,(act,22),(sat,500),(gender,m),(fname,Anne),{})})
> ((state,AZ),{(gjames,(highSchool,Tuscon High),(lname,James),state,(act,24),(sat,650),(gender,f),(fname,Geronomo),{})})
> ((state,CA),{(philton,(highSchool,Beverly High),(lname,Hilton),state,(act,37),(sat,220),(gender,m),(fname,Paris),{}),(jbrown,(highSchool,Cal High),(lname,Brown),state,(act,20),(sat,700),(gender,m),(fname,Jerry),{})})
> // Error - use explicit cast
> grunt> user_avg = FOREACH users_by_state GENERATE cassandra_users.state, AVG(cassandra_users.sat);
> grunt> dump user_avg;
> 2012-07-22 17:15:04,361 [main] ERROR org.apache.pig.tools.grunt.Grunt - ERROR 1045: Could not infer the matching function for org.apache.pig.builtin.AVG as multiple or none of them fit. Please use an explicit cast.
> // Unable to cast as int
> grunt> user_avg = FOREACH users_by_state GENERATE cassandra_users.state, AVG((int)cassandra_users.sat);
> grunt> dump user_avg;
> 2012-07-22 17:07:39,217 [main] ERROR org.apache.pig.tools.grunt.Grunt - ERROR 1052: Cannot cast bag with schema sat: bag({name: chararray,value: bytearray}) to int
> {code}
> *Seed data in CQL*
> {code}
> CREATE KEYSPACE cqldb with
> strategy_class = 'org.apache.cassandra.locator.SimpleStrategy'
> and strategy_options:replication_factor=3;
> use cqldb;
> CREATE COLUMNFAMILY users (
> KEY text PRIMARY KEY,
> fname text, lname text, gender varchar,
> act int, sat int, highSchool text, state varchar);
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (gjames, Geronomo, James, f, 24, 650, 'Tuscon High', 'AZ');
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (aoakley, Anne, Oakley, m , 22, 500, 'Phoenix High', 'AX');
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (jbrown, Jerry, Brown, m , 20, 700, 'Cal High', 'CA');
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (philton, Paris, Hilton, m , 37, 220, 'Beverly High', 'CA');
> select * from users;
> {code}
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators: https://issues.apache.org/jira/secure/ContactAdministrators!default.jspa
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Commented] (CASSANDRA-4459) pig driver casts ints as
bytearray
Posted by "Pavel Yaskevich (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/CASSANDRA-4459?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13421338#comment-13421338 ]
Pavel Yaskevich commented on CASSANDRA-4459:
--------------------------------------------
I agree with Jonathan on this, IntegerType could be larger than int32.
> pig driver casts ints as bytearray
> ----------------------------------
>
> Key: CASSANDRA-4459
> URL: https://issues.apache.org/jira/browse/CASSANDRA-4459
> Project: Cassandra
> Issue Type: Bug
> Environment: C* 1.1.2 embedded in DSE
> Reporter: Cathy Daw
> Assignee: Brandon Williams
> Fix For: 1.1.3
>
> Attachments: 4459.txt
>
>
> we seem to be auto-mapping C* int columns to bytearray in Pig, and farther down I can't seem to find a way to cast that to int and do an average.
> {code}
> grunt> cassandra_users = LOAD 'cassandra://cqldb/users' USING CassandraStorage();
> grunt> dump cassandra_users;
> (bobhatter,(act,22),(fname,bob),(gender,m),(highSchool,Cal High),(lname,hatter),(sat,500),(state,CA),{})
> (alicesmith,(act,27),(fname,alice),(gender,f),(highSchool,Tuscon High),(lname,smith),(sat,650),(state,AZ),{})
>
> // notice sat and act columns are bytearray values
> grunt> describe cassandra_users;
> cassandra_users: {key: chararray,act: (name: chararray,value: bytearray),fname: (name: chararray,value: chararray),
> gender: (name: chararray,value: chararray),highSchool: (name: chararray,value: chararray),lname: (name: chararray,value: chararray),
> sat: (name: chararray,value: bytearray),state: (name: chararray,value: chararray),columns: {(name: chararray,value: chararray)}}
> grunt> users_by_state = GROUP cassandra_users BY state;
> grunt> dump users_by_state;
> ((state,AX),{(aoakley,(highSchool,Phoenix High),(lname,Oakley),state,(act,22),(sat,500),(gender,m),(fname,Anne),{})})
> ((state,AZ),{(gjames,(highSchool,Tuscon High),(lname,James),state,(act,24),(sat,650),(gender,f),(fname,Geronomo),{})})
> ((state,CA),{(philton,(highSchool,Beverly High),(lname,Hilton),state,(act,37),(sat,220),(gender,m),(fname,Paris),{}),(jbrown,(highSchool,Cal High),(lname,Brown),state,(act,20),(sat,700),(gender,m),(fname,Jerry),{})})
> // Error - use explicit cast
> grunt> user_avg = FOREACH users_by_state GENERATE cassandra_users.state, AVG(cassandra_users.sat);
> grunt> dump user_avg;
> 2012-07-22 17:15:04,361 [main] ERROR org.apache.pig.tools.grunt.Grunt - ERROR 1045: Could not infer the matching function for org.apache.pig.builtin.AVG as multiple or none of them fit. Please use an explicit cast.
> // Unable to cast as int
> grunt> user_avg = FOREACH users_by_state GENERATE cassandra_users.state, AVG((int)cassandra_users.sat);
> grunt> dump user_avg;
> 2012-07-22 17:07:39,217 [main] ERROR org.apache.pig.tools.grunt.Grunt - ERROR 1052: Cannot cast bag with schema sat: bag({name: chararray,value: bytearray}) to int
> {code}
> *Seed data in CQL*
> {code}
> CREATE KEYSPACE cqldb with
> strategy_class = 'org.apache.cassandra.locator.SimpleStrategy'
> and strategy_options:replication_factor=3;
> use cqldb;
> CREATE COLUMNFAMILY users (
> KEY text PRIMARY KEY,
> fname text, lname text, gender varchar,
> act int, sat int, highSchool text, state varchar);
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (gjames, Geronomo, James, f, 24, 650, 'Tuscon High', 'AZ');
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (aoakley, Anne, Oakley, m , 22, 500, 'Phoenix High', 'AX');
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (jbrown, Jerry, Brown, m , 20, 700, 'Cal High', 'CA');
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (philton, Paris, Hilton, m , 37, 220, 'Beverly High', 'CA');
> select * from users;
> {code}
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators: https://issues.apache.org/jira/secure/ContactAdministrators!default.jspa
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Comment Edited] (CASSANDRA-4459) pig driver casts ints as
bytearray
Posted by "Brandon Williams (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/CASSANDRA-4459?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13421508#comment-13421508 ]
Brandon Williams edited comment on CASSANDRA-4459 at 7/24/12 4:10 PM:
----------------------------------------------------------------------
bq. casting IntegerType to pig's [32bit] int sounds broken to me.
I agree, but the conundrum we're in now is, I'm almost certain someone is relying on the current behavior to work and always using ints under 2**31, so changing it now would break things for them, and really the only danger is exceeding that limit, which I suspect people who create int columns never intend to do (or they'd make them longs.)
So, I propose that when pig has a BigInteger, we switch to that, allowing a smooth transition (unless you're exceeding 2**31 already, which to my knowledge no one is.)
was (Author: brandon.williams):
bq. casting IntegerType to pig's [32bit] int sounds broken to me.
I agree, but the conundrum we're in now is, I'm almost certain someone is relying on the current behavior to work and always using ints under 2**31, so changing it now would break things for them, and really the only danger is exceeding that limit, which I suspect people who create int columns never intend to do (or they'd make them longs.)
So, I propose that when pig has a BigInteger, we use switch to that, allowing a smooth transition (unless you're exceeding 2**31 already, which to my knowledge no one is.)
> pig driver casts ints as bytearray
> ----------------------------------
>
> Key: CASSANDRA-4459
> URL: https://issues.apache.org/jira/browse/CASSANDRA-4459
> Project: Cassandra
> Issue Type: Bug
> Environment: C* 1.1.2 embedded in DSE
> Reporter: Cathy Daw
> Assignee: Brandon Williams
> Fix For: 1.1.3
>
> Attachments: 4459.txt
>
>
> we seem to be auto-mapping C* int columns to bytearray in Pig, and farther down I can't seem to find a way to cast that to int and do an average.
> {code}
> grunt> cassandra_users = LOAD 'cassandra://cqldb/users' USING CassandraStorage();
> grunt> dump cassandra_users;
> (bobhatter,(act,22),(fname,bob),(gender,m),(highSchool,Cal High),(lname,hatter),(sat,500),(state,CA),{})
> (alicesmith,(act,27),(fname,alice),(gender,f),(highSchool,Tuscon High),(lname,smith),(sat,650),(state,AZ),{})
>
> // notice sat and act columns are bytearray values
> grunt> describe cassandra_users;
> cassandra_users: {key: chararray,act: (name: chararray,value: bytearray),fname: (name: chararray,value: chararray),
> gender: (name: chararray,value: chararray),highSchool: (name: chararray,value: chararray),lname: (name: chararray,value: chararray),
> sat: (name: chararray,value: bytearray),state: (name: chararray,value: chararray),columns: {(name: chararray,value: chararray)}}
> grunt> users_by_state = GROUP cassandra_users BY state;
> grunt> dump users_by_state;
> ((state,AX),{(aoakley,(highSchool,Phoenix High),(lname,Oakley),state,(act,22),(sat,500),(gender,m),(fname,Anne),{})})
> ((state,AZ),{(gjames,(highSchool,Tuscon High),(lname,James),state,(act,24),(sat,650),(gender,f),(fname,Geronomo),{})})
> ((state,CA),{(philton,(highSchool,Beverly High),(lname,Hilton),state,(act,37),(sat,220),(gender,m),(fname,Paris),{}),(jbrown,(highSchool,Cal High),(lname,Brown),state,(act,20),(sat,700),(gender,m),(fname,Jerry),{})})
> // Error - use explicit cast
> grunt> user_avg = FOREACH users_by_state GENERATE cassandra_users.state, AVG(cassandra_users.sat);
> grunt> dump user_avg;
> 2012-07-22 17:15:04,361 [main] ERROR org.apache.pig.tools.grunt.Grunt - ERROR 1045: Could not infer the matching function for org.apache.pig.builtin.AVG as multiple or none of them fit. Please use an explicit cast.
> // Unable to cast as int
> grunt> user_avg = FOREACH users_by_state GENERATE cassandra_users.state, AVG((int)cassandra_users.sat);
> grunt> dump user_avg;
> 2012-07-22 17:07:39,217 [main] ERROR org.apache.pig.tools.grunt.Grunt - ERROR 1052: Cannot cast bag with schema sat: bag({name: chararray,value: bytearray}) to int
> {code}
> *Seed data in CQL*
> {code}
> CREATE KEYSPACE cqldb with
> strategy_class = 'org.apache.cassandra.locator.SimpleStrategy'
> and strategy_options:replication_factor=3;
> use cqldb;
> CREATE COLUMNFAMILY users (
> KEY text PRIMARY KEY,
> fname text, lname text, gender varchar,
> act int, sat int, highSchool text, state varchar);
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (gjames, Geronomo, James, f, 24, 650, 'Tuscon High', 'AZ');
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (aoakley, Anne, Oakley, m , 22, 500, 'Phoenix High', 'AX');
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (jbrown, Jerry, Brown, m , 20, 700, 'Cal High', 'CA');
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (philton, Paris, Hilton, m , 37, 220, 'Beverly High', 'CA');
> select * from users;
> {code}
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators: https://issues.apache.org/jira/secure/ContactAdministrators!default.jspa
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Commented] (CASSANDRA-4459) pig driver casts ints as
bytearray
Posted by "Jonathan Ellis (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/CASSANDRA-4459?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13422395#comment-13422395 ]
Jonathan Ellis commented on CASSANDRA-4459:
-------------------------------------------
+1
> pig driver casts ints as bytearray
> ----------------------------------
>
> Key: CASSANDRA-4459
> URL: https://issues.apache.org/jira/browse/CASSANDRA-4459
> Project: Cassandra
> Issue Type: Bug
> Environment: C* 1.1.2 embedded in DSE
> Reporter: Cathy Daw
> Assignee: Brandon Williams
> Fix For: 1.1.3
>
> Attachments: 4459-v2.txt, 4459.txt
>
>
> we seem to be auto-mapping C* int columns to bytearray in Pig, and farther down I can't seem to find a way to cast that to int and do an average.
> {code}
> grunt> cassandra_users = LOAD 'cassandra://cqldb/users' USING CassandraStorage();
> grunt> dump cassandra_users;
> (bobhatter,(act,22),(fname,bob),(gender,m),(highSchool,Cal High),(lname,hatter),(sat,500),(state,CA),{})
> (alicesmith,(act,27),(fname,alice),(gender,f),(highSchool,Tuscon High),(lname,smith),(sat,650),(state,AZ),{})
>
> // notice sat and act columns are bytearray values
> grunt> describe cassandra_users;
> cassandra_users: {key: chararray,act: (name: chararray,value: bytearray),fname: (name: chararray,value: chararray),
> gender: (name: chararray,value: chararray),highSchool: (name: chararray,value: chararray),lname: (name: chararray,value: chararray),
> sat: (name: chararray,value: bytearray),state: (name: chararray,value: chararray),columns: {(name: chararray,value: chararray)}}
> grunt> users_by_state = GROUP cassandra_users BY state;
> grunt> dump users_by_state;
> ((state,AX),{(aoakley,(highSchool,Phoenix High),(lname,Oakley),state,(act,22),(sat,500),(gender,m),(fname,Anne),{})})
> ((state,AZ),{(gjames,(highSchool,Tuscon High),(lname,James),state,(act,24),(sat,650),(gender,f),(fname,Geronomo),{})})
> ((state,CA),{(philton,(highSchool,Beverly High),(lname,Hilton),state,(act,37),(sat,220),(gender,m),(fname,Paris),{}),(jbrown,(highSchool,Cal High),(lname,Brown),state,(act,20),(sat,700),(gender,m),(fname,Jerry),{})})
> // Error - use explicit cast
> grunt> user_avg = FOREACH users_by_state GENERATE cassandra_users.state, AVG(cassandra_users.sat);
> grunt> dump user_avg;
> 2012-07-22 17:15:04,361 [main] ERROR org.apache.pig.tools.grunt.Grunt - ERROR 1045: Could not infer the matching function for org.apache.pig.builtin.AVG as multiple or none of them fit. Please use an explicit cast.
> // Unable to cast as int
> grunt> user_avg = FOREACH users_by_state GENERATE cassandra_users.state, AVG((int)cassandra_users.sat);
> grunt> dump user_avg;
> 2012-07-22 17:07:39,217 [main] ERROR org.apache.pig.tools.grunt.Grunt - ERROR 1052: Cannot cast bag with schema sat: bag({name: chararray,value: bytearray}) to int
> {code}
> *Seed data in CQL*
> {code}
> CREATE KEYSPACE cqldb with
> strategy_class = 'org.apache.cassandra.locator.SimpleStrategy'
> and strategy_options:replication_factor=3;
> use cqldb;
> CREATE COLUMNFAMILY users (
> KEY text PRIMARY KEY,
> fname text, lname text, gender varchar,
> act int, sat int, highSchool text, state varchar);
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (gjames, Geronomo, James, f, 24, 650, 'Tuscon High', 'AZ');
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (aoakley, Anne, Oakley, m , 22, 500, 'Phoenix High', 'AX');
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (jbrown, Jerry, Brown, m , 20, 700, 'Cal High', 'CA');
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (philton, Paris, Hilton, m , 37, 220, 'Beverly High', 'CA');
> select * from users;
> {code}
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators: https://issues.apache.org/jira/secure/ContactAdministrators!default.jspa
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Commented] (CASSANDRA-4459) pig driver casts ints as
bytearray
Posted by "Jonathan Ellis (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/CASSANDRA-4459?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13420909#comment-13420909 ]
Jonathan Ellis commented on CASSANDRA-4459:
-------------------------------------------
casting IntegerType to pig's [32bit] int sounds broken to me. shouldn't we fix the population script to use int32 as well?
> pig driver casts ints as bytearray
> ----------------------------------
>
> Key: CASSANDRA-4459
> URL: https://issues.apache.org/jira/browse/CASSANDRA-4459
> Project: Cassandra
> Issue Type: Bug
> Environment: C* 1.1.2 embedded in DSE
> Reporter: Cathy Daw
> Assignee: Brandon Williams
> Fix For: 1.1.3
>
> Attachments: 4459.txt
>
>
> we seem to be auto-mapping C* int columns to bytearray in Pig, and farther down I can't seem to find a way to cast that to int and do an average.
> {code}
> grunt> cassandra_users = LOAD 'cassandra://cqldb/users' USING CassandraStorage();
> grunt> dump cassandra_users;
> (bobhatter,(act,22),(fname,bob),(gender,m),(highSchool,Cal High),(lname,hatter),(sat,500),(state,CA),{})
> (alicesmith,(act,27),(fname,alice),(gender,f),(highSchool,Tuscon High),(lname,smith),(sat,650),(state,AZ),{})
>
> // notice sat and act columns are bytearray values
> grunt> describe cassandra_users;
> cassandra_users: {key: chararray,act: (name: chararray,value: bytearray),fname: (name: chararray,value: chararray),
> gender: (name: chararray,value: chararray),highSchool: (name: chararray,value: chararray),lname: (name: chararray,value: chararray),
> sat: (name: chararray,value: bytearray),state: (name: chararray,value: chararray),columns: {(name: chararray,value: chararray)}}
> grunt> users_by_state = GROUP cassandra_users BY state;
> grunt> dump users_by_state;
> ((state,AX),{(aoakley,(highSchool,Phoenix High),(lname,Oakley),state,(act,22),(sat,500),(gender,m),(fname,Anne),{})})
> ((state,AZ),{(gjames,(highSchool,Tuscon High),(lname,James),state,(act,24),(sat,650),(gender,f),(fname,Geronomo),{})})
> ((state,CA),{(philton,(highSchool,Beverly High),(lname,Hilton),state,(act,37),(sat,220),(gender,m),(fname,Paris),{}),(jbrown,(highSchool,Cal High),(lname,Brown),state,(act,20),(sat,700),(gender,m),(fname,Jerry),{})})
> // Error - use explicit cast
> grunt> user_avg = FOREACH users_by_state GENERATE cassandra_users.state, AVG(cassandra_users.sat);
> grunt> dump user_avg;
> 2012-07-22 17:15:04,361 [main] ERROR org.apache.pig.tools.grunt.Grunt - ERROR 1045: Could not infer the matching function for org.apache.pig.builtin.AVG as multiple or none of them fit. Please use an explicit cast.
> // Unable to cast as int
> grunt> user_avg = FOREACH users_by_state GENERATE cassandra_users.state, AVG((int)cassandra_users.sat);
> grunt> dump user_avg;
> 2012-07-22 17:07:39,217 [main] ERROR org.apache.pig.tools.grunt.Grunt - ERROR 1052: Cannot cast bag with schema sat: bag({name: chararray,value: bytearray}) to int
> {code}
> *Seed data in CQL*
> {code}
> CREATE KEYSPACE cqldb with
> strategy_class = 'org.apache.cassandra.locator.SimpleStrategy'
> and strategy_options:replication_factor=3;
> use cqldb;
> CREATE COLUMNFAMILY users (
> KEY text PRIMARY KEY,
> fname text, lname text, gender varchar,
> act int, sat int, highSchool text, state varchar);
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (gjames, Geronomo, James, f, 24, 650, 'Tuscon High', 'AZ');
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (aoakley, Anne, Oakley, m , 22, 500, 'Phoenix High', 'AX');
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (jbrown, Jerry, Brown, m , 20, 700, 'Cal High', 'CA');
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (philton, Paris, Hilton, m , 37, 220, 'Beverly High', 'CA');
> select * from users;
> {code}
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators: https://issues.apache.org/jira/secure/ContactAdministrators!default.jspa
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Updated] (CASSANDRA-4459) pig driver casts ints as
bytearray
Posted by "Brandon Williams (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/CASSANDRA-4459?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Brandon Williams updated CASSANDRA-4459:
----------------------------------------
Attachment: 4459-v2.txt
Update with a comment explaining that IntegerType is wrong, but we're doing it anyway. Also switched all the IntegerTypes to Int32Types in the tests, which pass. I don't see any point in explicitly testing IntegerType as well until pig has a BigInteger.
> pig driver casts ints as bytearray
> ----------------------------------
>
> Key: CASSANDRA-4459
> URL: https://issues.apache.org/jira/browse/CASSANDRA-4459
> Project: Cassandra
> Issue Type: Bug
> Environment: C* 1.1.2 embedded in DSE
> Reporter: Cathy Daw
> Assignee: Brandon Williams
> Fix For: 1.1.3
>
> Attachments: 4459-v2.txt, 4459.txt
>
>
> we seem to be auto-mapping C* int columns to bytearray in Pig, and farther down I can't seem to find a way to cast that to int and do an average.
> {code}
> grunt> cassandra_users = LOAD 'cassandra://cqldb/users' USING CassandraStorage();
> grunt> dump cassandra_users;
> (bobhatter,(act,22),(fname,bob),(gender,m),(highSchool,Cal High),(lname,hatter),(sat,500),(state,CA),{})
> (alicesmith,(act,27),(fname,alice),(gender,f),(highSchool,Tuscon High),(lname,smith),(sat,650),(state,AZ),{})
>
> // notice sat and act columns are bytearray values
> grunt> describe cassandra_users;
> cassandra_users: {key: chararray,act: (name: chararray,value: bytearray),fname: (name: chararray,value: chararray),
> gender: (name: chararray,value: chararray),highSchool: (name: chararray,value: chararray),lname: (name: chararray,value: chararray),
> sat: (name: chararray,value: bytearray),state: (name: chararray,value: chararray),columns: {(name: chararray,value: chararray)}}
> grunt> users_by_state = GROUP cassandra_users BY state;
> grunt> dump users_by_state;
> ((state,AX),{(aoakley,(highSchool,Phoenix High),(lname,Oakley),state,(act,22),(sat,500),(gender,m),(fname,Anne),{})})
> ((state,AZ),{(gjames,(highSchool,Tuscon High),(lname,James),state,(act,24),(sat,650),(gender,f),(fname,Geronomo),{})})
> ((state,CA),{(philton,(highSchool,Beverly High),(lname,Hilton),state,(act,37),(sat,220),(gender,m),(fname,Paris),{}),(jbrown,(highSchool,Cal High),(lname,Brown),state,(act,20),(sat,700),(gender,m),(fname,Jerry),{})})
> // Error - use explicit cast
> grunt> user_avg = FOREACH users_by_state GENERATE cassandra_users.state, AVG(cassandra_users.sat);
> grunt> dump user_avg;
> 2012-07-22 17:15:04,361 [main] ERROR org.apache.pig.tools.grunt.Grunt - ERROR 1045: Could not infer the matching function for org.apache.pig.builtin.AVG as multiple or none of them fit. Please use an explicit cast.
> // Unable to cast as int
> grunt> user_avg = FOREACH users_by_state GENERATE cassandra_users.state, AVG((int)cassandra_users.sat);
> grunt> dump user_avg;
> 2012-07-22 17:07:39,217 [main] ERROR org.apache.pig.tools.grunt.Grunt - ERROR 1052: Cannot cast bag with schema sat: bag({name: chararray,value: bytearray}) to int
> {code}
> *Seed data in CQL*
> {code}
> CREATE KEYSPACE cqldb with
> strategy_class = 'org.apache.cassandra.locator.SimpleStrategy'
> and strategy_options:replication_factor=3;
> use cqldb;
> CREATE COLUMNFAMILY users (
> KEY text PRIMARY KEY,
> fname text, lname text, gender varchar,
> act int, sat int, highSchool text, state varchar);
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (gjames, Geronomo, James, f, 24, 650, 'Tuscon High', 'AZ');
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (aoakley, Anne, Oakley, m , 22, 500, 'Phoenix High', 'AX');
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (jbrown, Jerry, Brown, m , 20, 700, 'Cal High', 'CA');
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (philton, Paris, Hilton, m , 37, 220, 'Beverly High', 'CA');
> select * from users;
> {code}
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators: https://issues.apache.org/jira/secure/ContactAdministrators!default.jspa
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Commented] (CASSANDRA-4459) pig driver casts ints as
bytearray
Posted by "Pavel Yaskevich (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/CASSANDRA-4459?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13422400#comment-13422400 ]
Pavel Yaskevich commented on CASSANDRA-4459:
--------------------------------------------
+1
> pig driver casts ints as bytearray
> ----------------------------------
>
> Key: CASSANDRA-4459
> URL: https://issues.apache.org/jira/browse/CASSANDRA-4459
> Project: Cassandra
> Issue Type: Bug
> Environment: C* 1.1.2 embedded in DSE
> Reporter: Cathy Daw
> Assignee: Brandon Williams
> Fix For: 1.1.3
>
> Attachments: 4459-v2.txt, 4459.txt
>
>
> we seem to be auto-mapping C* int columns to bytearray in Pig, and farther down I can't seem to find a way to cast that to int and do an average.
> {code}
> grunt> cassandra_users = LOAD 'cassandra://cqldb/users' USING CassandraStorage();
> grunt> dump cassandra_users;
> (bobhatter,(act,22),(fname,bob),(gender,m),(highSchool,Cal High),(lname,hatter),(sat,500),(state,CA),{})
> (alicesmith,(act,27),(fname,alice),(gender,f),(highSchool,Tuscon High),(lname,smith),(sat,650),(state,AZ),{})
>
> // notice sat and act columns are bytearray values
> grunt> describe cassandra_users;
> cassandra_users: {key: chararray,act: (name: chararray,value: bytearray),fname: (name: chararray,value: chararray),
> gender: (name: chararray,value: chararray),highSchool: (name: chararray,value: chararray),lname: (name: chararray,value: chararray),
> sat: (name: chararray,value: bytearray),state: (name: chararray,value: chararray),columns: {(name: chararray,value: chararray)}}
> grunt> users_by_state = GROUP cassandra_users BY state;
> grunt> dump users_by_state;
> ((state,AX),{(aoakley,(highSchool,Phoenix High),(lname,Oakley),state,(act,22),(sat,500),(gender,m),(fname,Anne),{})})
> ((state,AZ),{(gjames,(highSchool,Tuscon High),(lname,James),state,(act,24),(sat,650),(gender,f),(fname,Geronomo),{})})
> ((state,CA),{(philton,(highSchool,Beverly High),(lname,Hilton),state,(act,37),(sat,220),(gender,m),(fname,Paris),{}),(jbrown,(highSchool,Cal High),(lname,Brown),state,(act,20),(sat,700),(gender,m),(fname,Jerry),{})})
> // Error - use explicit cast
> grunt> user_avg = FOREACH users_by_state GENERATE cassandra_users.state, AVG(cassandra_users.sat);
> grunt> dump user_avg;
> 2012-07-22 17:15:04,361 [main] ERROR org.apache.pig.tools.grunt.Grunt - ERROR 1045: Could not infer the matching function for org.apache.pig.builtin.AVG as multiple or none of them fit. Please use an explicit cast.
> // Unable to cast as int
> grunt> user_avg = FOREACH users_by_state GENERATE cassandra_users.state, AVG((int)cassandra_users.sat);
> grunt> dump user_avg;
> 2012-07-22 17:07:39,217 [main] ERROR org.apache.pig.tools.grunt.Grunt - ERROR 1052: Cannot cast bag with schema sat: bag({name: chararray,value: bytearray}) to int
> {code}
> *Seed data in CQL*
> {code}
> CREATE KEYSPACE cqldb with
> strategy_class = 'org.apache.cassandra.locator.SimpleStrategy'
> and strategy_options:replication_factor=3;
> use cqldb;
> CREATE COLUMNFAMILY users (
> KEY text PRIMARY KEY,
> fname text, lname text, gender varchar,
> act int, sat int, highSchool text, state varchar);
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (gjames, Geronomo, James, f, 24, 650, 'Tuscon High', 'AZ');
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (aoakley, Anne, Oakley, m , 22, 500, 'Phoenix High', 'AX');
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (jbrown, Jerry, Brown, m , 20, 700, 'Cal High', 'CA');
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (philton, Paris, Hilton, m , 37, 220, 'Beverly High', 'CA');
> select * from users;
> {code}
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators: https://issues.apache.org/jira/secure/ContactAdministrators!default.jspa
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Commented] (CASSANDRA-4459) pig driver casts ints as
bytearray
Posted by "Brandon Williams (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/CASSANDRA-4459?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13421508#comment-13421508 ]
Brandon Williams commented on CASSANDRA-4459:
---------------------------------------------
bq. casting IntegerType to pig's [32bit] int sounds broken to me.
I agree, but the conundrum we're in now is, I'm almost certain someone is relying on the current behavior to work and always using ints under 2**31, so changing it now would break things for them, and really the only danger is exceeding that limit, which I suspect people who create int columns never intend to do (or they'd make them longs.)
So, I propose that when pig has a BigInteger, we use switch to that, allowing a smooth transition (unless you're exceeding 2**31 already, which to my knowledge no one is.)
> pig driver casts ints as bytearray
> ----------------------------------
>
> Key: CASSANDRA-4459
> URL: https://issues.apache.org/jira/browse/CASSANDRA-4459
> Project: Cassandra
> Issue Type: Bug
> Environment: C* 1.1.2 embedded in DSE
> Reporter: Cathy Daw
> Assignee: Brandon Williams
> Fix For: 1.1.3
>
> Attachments: 4459.txt
>
>
> we seem to be auto-mapping C* int columns to bytearray in Pig, and farther down I can't seem to find a way to cast that to int and do an average.
> {code}
> grunt> cassandra_users = LOAD 'cassandra://cqldb/users' USING CassandraStorage();
> grunt> dump cassandra_users;
> (bobhatter,(act,22),(fname,bob),(gender,m),(highSchool,Cal High),(lname,hatter),(sat,500),(state,CA),{})
> (alicesmith,(act,27),(fname,alice),(gender,f),(highSchool,Tuscon High),(lname,smith),(sat,650),(state,AZ),{})
>
> // notice sat and act columns are bytearray values
> grunt> describe cassandra_users;
> cassandra_users: {key: chararray,act: (name: chararray,value: bytearray),fname: (name: chararray,value: chararray),
> gender: (name: chararray,value: chararray),highSchool: (name: chararray,value: chararray),lname: (name: chararray,value: chararray),
> sat: (name: chararray,value: bytearray),state: (name: chararray,value: chararray),columns: {(name: chararray,value: chararray)}}
> grunt> users_by_state = GROUP cassandra_users BY state;
> grunt> dump users_by_state;
> ((state,AX),{(aoakley,(highSchool,Phoenix High),(lname,Oakley),state,(act,22),(sat,500),(gender,m),(fname,Anne),{})})
> ((state,AZ),{(gjames,(highSchool,Tuscon High),(lname,James),state,(act,24),(sat,650),(gender,f),(fname,Geronomo),{})})
> ((state,CA),{(philton,(highSchool,Beverly High),(lname,Hilton),state,(act,37),(sat,220),(gender,m),(fname,Paris),{}),(jbrown,(highSchool,Cal High),(lname,Brown),state,(act,20),(sat,700),(gender,m),(fname,Jerry),{})})
> // Error - use explicit cast
> grunt> user_avg = FOREACH users_by_state GENERATE cassandra_users.state, AVG(cassandra_users.sat);
> grunt> dump user_avg;
> 2012-07-22 17:15:04,361 [main] ERROR org.apache.pig.tools.grunt.Grunt - ERROR 1045: Could not infer the matching function for org.apache.pig.builtin.AVG as multiple or none of them fit. Please use an explicit cast.
> // Unable to cast as int
> grunt> user_avg = FOREACH users_by_state GENERATE cassandra_users.state, AVG((int)cassandra_users.sat);
> grunt> dump user_avg;
> 2012-07-22 17:07:39,217 [main] ERROR org.apache.pig.tools.grunt.Grunt - ERROR 1052: Cannot cast bag with schema sat: bag({name: chararray,value: bytearray}) to int
> {code}
> *Seed data in CQL*
> {code}
> CREATE KEYSPACE cqldb with
> strategy_class = 'org.apache.cassandra.locator.SimpleStrategy'
> and strategy_options:replication_factor=3;
> use cqldb;
> CREATE COLUMNFAMILY users (
> KEY text PRIMARY KEY,
> fname text, lname text, gender varchar,
> act int, sat int, highSchool text, state varchar);
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (gjames, Geronomo, James, f, 24, 650, 'Tuscon High', 'AZ');
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (aoakley, Anne, Oakley, m , 22, 500, 'Phoenix High', 'AX');
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (jbrown, Jerry, Brown, m , 20, 700, 'Cal High', 'CA');
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (philton, Paris, Hilton, m , 37, 220, 'Beverly High', 'CA');
> select * from users;
> {code}
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators: https://issues.apache.org/jira/secure/ContactAdministrators!default.jspa
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Commented] (CASSANDRA-4459) pig driver casts ints as
bytearray
Posted by "Jeremy Hanna (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/CASSANDRA-4459?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13421502#comment-13421502 ]
Jeremy Hanna commented on CASSANDRA-4459:
-----------------------------------------
fwiw - see https://issues.apache.org/jira/browse/PIG-2764 for the addition of BigInteger and BigDecimal as built-in pig data types. Also, I'm not sure how much of an issue it is for users to use pig ints for now because I don't know how many users know that the cassandra IntegerType is actually a BigInteger and not just a regular Integer. That's not to say that it's not dangerous to try to put a BigInteger value into an Integer type. It's just that I don't know if it's common knowledge that Cassandra uses a BigInteger underneath.
> pig driver casts ints as bytearray
> ----------------------------------
>
> Key: CASSANDRA-4459
> URL: https://issues.apache.org/jira/browse/CASSANDRA-4459
> Project: Cassandra
> Issue Type: Bug
> Environment: C* 1.1.2 embedded in DSE
> Reporter: Cathy Daw
> Assignee: Brandon Williams
> Fix For: 1.1.3
>
> Attachments: 4459.txt
>
>
> we seem to be auto-mapping C* int columns to bytearray in Pig, and farther down I can't seem to find a way to cast that to int and do an average.
> {code}
> grunt> cassandra_users = LOAD 'cassandra://cqldb/users' USING CassandraStorage();
> grunt> dump cassandra_users;
> (bobhatter,(act,22),(fname,bob),(gender,m),(highSchool,Cal High),(lname,hatter),(sat,500),(state,CA),{})
> (alicesmith,(act,27),(fname,alice),(gender,f),(highSchool,Tuscon High),(lname,smith),(sat,650),(state,AZ),{})
>
> // notice sat and act columns are bytearray values
> grunt> describe cassandra_users;
> cassandra_users: {key: chararray,act: (name: chararray,value: bytearray),fname: (name: chararray,value: chararray),
> gender: (name: chararray,value: chararray),highSchool: (name: chararray,value: chararray),lname: (name: chararray,value: chararray),
> sat: (name: chararray,value: bytearray),state: (name: chararray,value: chararray),columns: {(name: chararray,value: chararray)}}
> grunt> users_by_state = GROUP cassandra_users BY state;
> grunt> dump users_by_state;
> ((state,AX),{(aoakley,(highSchool,Phoenix High),(lname,Oakley),state,(act,22),(sat,500),(gender,m),(fname,Anne),{})})
> ((state,AZ),{(gjames,(highSchool,Tuscon High),(lname,James),state,(act,24),(sat,650),(gender,f),(fname,Geronomo),{})})
> ((state,CA),{(philton,(highSchool,Beverly High),(lname,Hilton),state,(act,37),(sat,220),(gender,m),(fname,Paris),{}),(jbrown,(highSchool,Cal High),(lname,Brown),state,(act,20),(sat,700),(gender,m),(fname,Jerry),{})})
> // Error - use explicit cast
> grunt> user_avg = FOREACH users_by_state GENERATE cassandra_users.state, AVG(cassandra_users.sat);
> grunt> dump user_avg;
> 2012-07-22 17:15:04,361 [main] ERROR org.apache.pig.tools.grunt.Grunt - ERROR 1045: Could not infer the matching function for org.apache.pig.builtin.AVG as multiple or none of them fit. Please use an explicit cast.
> // Unable to cast as int
> grunt> user_avg = FOREACH users_by_state GENERATE cassandra_users.state, AVG((int)cassandra_users.sat);
> grunt> dump user_avg;
> 2012-07-22 17:07:39,217 [main] ERROR org.apache.pig.tools.grunt.Grunt - ERROR 1052: Cannot cast bag with schema sat: bag({name: chararray,value: bytearray}) to int
> {code}
> *Seed data in CQL*
> {code}
> CREATE KEYSPACE cqldb with
> strategy_class = 'org.apache.cassandra.locator.SimpleStrategy'
> and strategy_options:replication_factor=3;
> use cqldb;
> CREATE COLUMNFAMILY users (
> KEY text PRIMARY KEY,
> fname text, lname text, gender varchar,
> act int, sat int, highSchool text, state varchar);
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (gjames, Geronomo, James, f, 24, 650, 'Tuscon High', 'AZ');
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (aoakley, Anne, Oakley, m , 22, 500, 'Phoenix High', 'AX');
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (jbrown, Jerry, Brown, m , 20, 700, 'Cal High', 'CA');
> insert into users (KEY, fname, lname, gender, act, sat, highSchool, state)
> values (philton, Paris, Hilton, m , 37, 220, 'Beverly High', 'CA');
> select * from users;
> {code}
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators: https://issues.apache.org/jira/secure/ContactAdministrators!default.jspa
For more information on JIRA, see: http://www.atlassian.com/software/jira