You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by br...@apache.org on 2013/12/13 18:42:32 UTC
git commit: Pig: fix duplicate schema error Patch by Alex Liu,
reviewed by brandonwilliams for CASSANDRA-6309
Updated Branches:
refs/heads/trunk 611f328f3 -> 343a6472d
Pig: fix duplicate schema error
Patch by Alex Liu, reviewed by brandonwilliams for CASSANDRA-6309
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/343a6472
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/343a6472
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/343a6472
Branch: refs/heads/trunk
Commit: 343a6472d8e26bc846c575c03d0af7d8b66e6dfa
Parents: 611f328
Author: Brandon Williams <br...@apache.org>
Authored: Fri Dec 13 11:40:20 2013 -0600
Committer: Brandon Williams <br...@apache.org>
Committed: Fri Dec 13 11:40:20 2013 -0600
----------------------------------------------------------------------
build.xml | 27 ++-
.../apache/cassandra/db/ConsistencyLevel.java | 2 -
.../hadoop/pig/AbstractCassandraStorage.java | 22 +--
.../apache/cassandra/hadoop/pig/CqlStorage.java | 3 -
.../cassandra/pig/CqlTableDataTypeTest.java | 35 +---
.../org/apache/cassandra/pig/CqlTableTest.java | 9 +-
.../org/apache/cassandra/pig/PigTestBase.java | 7 +-
.../pig/ThriftColumnFamilyDataTypeTest.java | 21 ---
.../cassandra/pig/ThriftColumnFamilyTest.java | 163 ++++++-------------
9 files changed, 105 insertions(+), 184 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/343a6472/build.xml
----------------------------------------------------------------------
diff --git a/build.xml b/build.xml
index 606d2e2..6e579bf 100644
--- a/build.xml
+++ b/build.xml
@@ -413,7 +413,6 @@
<dependency groupId="org.apache.hadoop" artifactId="hadoop-core"/>
<dependency groupId="org.apache.hadoop" artifactId="hadoop-minicluster"/>
<dependency groupId="org.apache.pig" artifactId="pig"/>
-
<dependency groupId="net.java.dev.jna" artifactId="jna"/>
<dependency groupId="com.google.code.findbugs" artifactId="jsr305"/>
</artifact:pom>
@@ -431,6 +430,9 @@
<parent groupId="org.apache.cassandra"
artifactId="cassandra-parent"
version="${version}"/>
+ <dependency groupId="joda-time" artifactId="joda-time" version="2.3" />
+ <dependency groupId="org.slf4j" artifactId="slf4j-log4j12" version="1.7.2"/>
+ <dependency groupId="log4j" artifactId="log4j" version="1.2.16" />
</artifact:pom>
<!-- now the pom's for artifacts being deployed to Maven Central -->
@@ -563,6 +565,25 @@
</copy>
</target>
+ <target name="maven-ant-tasks-retrieve-pig-test" depends="maven-ant-tasks-init">
+ <artifact:dependencies pomRefId="test-deps-pom"
+ filesetId="test-dependency-jars"
+ sourcesFilesetId="test-dependency-sources"
+ cacheDependencyRefs="true"
+ dependencyRefsBuildFile="${build.dir}/test-dependencies.xml">
+ <remoteRepository refid="apache"/>
+ <remoteRepository refid="central"/>
+ <remoteRepository refid="java.net2"/>
+ </artifact:dependencies>
+ <copy todir="${build.dir.lib}/jars">
+ <fileset refid="test-dependency-jars"/>
+ <mapper type="flatten"/>
+ </copy>
+ <copy todir="${build.dir.lib}/sources">
+ <fileset refid="test-dependency-sources"/>
+ <mapper type="flatten"/>
+ </copy>
+ </target>
<!--
Generate thrift code. We have targets to build java because
@@ -995,6 +1016,7 @@
</classpath>
<src path="${test.unit.src}"/>
<src path="${test.long.src}"/>
+ <src path="${test.pig.src}"/>
</javac>
<!-- Non-java resources needed by the test suite -->
@@ -1132,7 +1154,7 @@
</testmacro>
</target>
- <target name="pig-test" depends="build-test" description="Excute Pig tests">
+ <target name="pig-test" depends="build-test,maven-ant-tasks-retrieve-pig-test" description="Excute Pig tests">
<testmacro suitename="pig" inputdir="${test.pig.src}"
timeout="1200000">
</testmacro>
@@ -1248,6 +1270,7 @@
<classpathentry kind="src" path="interface/thrift/gen-java"/>
<classpathentry kind="src" path="test/unit"/>
<classpathentry kind="src" path="test/long"/>
+ <classpathentry kind="src" path="test/pig"/>
<classpathentry kind="src" path="tools/stress/src"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry kind="output" path="build/classes/main"/>
http://git-wip-us.apache.org/repos/asf/cassandra/blob/343a6472/src/java/org/apache/cassandra/db/ConsistencyLevel.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ConsistencyLevel.java b/src/java/org/apache/cassandra/db/ConsistencyLevel.java
index cbb4bb1..0f6aba7 100644
--- a/src/java/org/apache/cassandra/db/ConsistencyLevel.java
+++ b/src/java/org/apache/cassandra/db/ConsistencyLevel.java
@@ -285,9 +285,7 @@ public enum ConsistencyLevel
{
switch (this)
{
- case LOCAL_QUORUM:
case EACH_QUORUM:
- case LOCAL_ONE:
requireNetworkTopologyStrategy(keyspaceName);
break;
case SERIAL:
http://git-wip-us.apache.org/repos/asf/cassandra/blob/343a6472/src/java/org/apache/cassandra/hadoop/pig/AbstractCassandraStorage.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/hadoop/pig/AbstractCassandraStorage.java b/src/java/org/apache/cassandra/hadoop/pig/AbstractCassandraStorage.java
index 9735df5..b5a4c67 100644
--- a/src/java/org/apache/cassandra/hadoop/pig/AbstractCassandraStorage.java
+++ b/src/java/org/apache/cassandra/hadoop/pig/AbstractCassandraStorage.java
@@ -83,7 +83,7 @@ public abstract class AbstractCassandraStorage extends LoadFunc implements Store
public final static String PARTITION_FILTER_SIGNATURE = "cassandra.partition.filter";
- protected static final Logger logger = LoggerFactory.getLogger(AbstractCassandraStorage.class);
+ private static final Logger logger = LoggerFactory.getLogger(AbstractCassandraStorage.class);
protected String username;
protected String password;
@@ -619,19 +619,7 @@ public abstract class AbstractCassandraStorage extends LoadFunc implements Store
cfDef.key_validation_class = ByteBufferUtil.string(cqlRow.columns.get(4).value);
String keyAliases = ByteBufferUtil.string(cqlRow.columns.get(5).value);
List<String> keys = FBUtilities.fromJsonList(keyAliases);
- // classis thrift tables
- if (keys.size() == 0)
- {
- CFMetaData cfm = getCFMetaData(keyspace, column_family, client);
- for (ColumnDefinition def : cfm.partitionKeyColumns())
- {
- String key = def.name.toString();
- String type = def.type.toString();
- logger.debug("name: {}, type: {} ", key, type);
- keys.add(key);
- }
- }
- else
+ if (FBUtilities.fromJsonList(keyAliases).size() > 0)
cql3Table = true;
}
cfDef.column_metadata = getColumnMetadata(client);
@@ -670,7 +658,8 @@ public abstract class AbstractCassandraStorage extends LoadFunc implements Store
{
String query = "SELECT column_name, " +
" validator, " +
- " index_type " +
+ " index_type, " +
+ " type " +
"FROM system.schema_columns " +
"WHERE keyspace_name = '%s' " +
" AND columnfamily_name = '%s'";
@@ -721,6 +710,9 @@ public abstract class AbstractCassandraStorage extends LoadFunc implements Store
{
CqlRow row = iterator.next();
ColumnDef cDef = new ColumnDef();
+ String type = ByteBufferUtil.string(row.getColumns().get(3).value);
+ if (!type.equals("regular"))
+ continue;
cDef.setName(ByteBufferUtil.clone(row.getColumns().get(0).value));
cDef.validation_class = ByteBufferUtil.string(row.getColumns().get(1).value);
ByteBuffer indexType = row.getColumns().get(2).value;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/343a6472/src/java/org/apache/cassandra/hadoop/pig/CqlStorage.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/hadoop/pig/CqlStorage.java b/src/java/org/apache/cassandra/hadoop/pig/CqlStorage.java
index a7793e2..89c1944 100644
--- a/src/java/org/apache/cassandra/hadoop/pig/CqlStorage.java
+++ b/src/java/org/apache/cassandra/hadoop/pig/CqlStorage.java
@@ -22,8 +22,6 @@ import java.nio.ByteBuffer;
import java.nio.charset.CharacterCodingException;
import java.util.*;
-
-import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.db.Column;
@@ -57,7 +55,6 @@ import org.slf4j.LoggerFactory;
public class CqlStorage extends AbstractCassandraStorage
{
private static final Logger logger = LoggerFactory.getLogger(CqlStorage.class);
-
private RecordReader<Map<String, ByteBuffer>, Map<String, ByteBuffer>> reader;
private RecordWriter<Map<String, ByteBuffer>, List<ByteBuffer>> writer;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/343a6472/test/pig/org/apache/cassandra/pig/CqlTableDataTypeTest.java
----------------------------------------------------------------------
diff --git a/test/pig/org/apache/cassandra/pig/CqlTableDataTypeTest.java b/test/pig/org/apache/cassandra/pig/CqlTableDataTypeTest.java
index 1ae9806..2020b0a 100644
--- a/test/pig/org/apache/cassandra/pig/CqlTableDataTypeTest.java
+++ b/test/pig/org/apache/cassandra/pig/CqlTableDataTypeTest.java
@@ -7,37 +7,16 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.cassandra.pig;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
+package org.apache.cassandra.pig;
import java.io.IOException;
import java.nio.charset.CharacterCodingException;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/343a6472/test/pig/org/apache/cassandra/pig/CqlTableTest.java
----------------------------------------------------------------------
diff --git a/test/pig/org/apache/cassandra/pig/CqlTableTest.java b/test/pig/org/apache/cassandra/pig/CqlTableTest.java
index 785d819..e33364f 100644
--- a/test/pig/org/apache/cassandra/pig/CqlTableTest.java
+++ b/test/pig/org/apache/cassandra/pig/CqlTableTest.java
@@ -50,6 +50,7 @@ public class CqlTableTest extends PigTestBase
"INSERT INTO compactcqltable (key1, column1, column2) values ('key1', 100, 10.1)",
"CREATE TABLE test (a int PRIMARY KEY, b int);",
+ "CREATE INDEX test_b on test (b);",
"CREATE TABLE moredata (x int PRIMARY KEY, y int);",
"INSERT INTO test (a,b) VALUES (1,1);",
@@ -158,10 +159,12 @@ public class CqlTableTest extends PigTestBase
//(9,10,Ninen,nomatch)
pig.registerQuery("result= LOAD 'cql://cql3ks/compotable?" + defaultParameters + "' USING CqlStorage();");
Iterator<Tuple> it = pig.openIterator("result");
- if (it.hasNext()) {
- Tuple t = it.next();
- Assert.assertEquals(t.get(3), "match");
+ int count = 0;
+ while (it.hasNext()) {
+ it.next();
+ count ++;
}
+ Assert.assertEquals(count, 9);
}
@Test
http://git-wip-us.apache.org/repos/asf/cassandra/blob/343a6472/test/pig/org/apache/cassandra/pig/PigTestBase.java
----------------------------------------------------------------------
diff --git a/test/pig/org/apache/cassandra/pig/PigTestBase.java b/test/pig/org/apache/cassandra/pig/PigTestBase.java
index ea06b8c..2335023 100644
--- a/test/pig/org/apache/cassandra/pig/PigTestBase.java
+++ b/test/pig/org/apache/cassandra/pig/PigTestBase.java
@@ -67,6 +67,11 @@ public class PigTestBase extends SchemaLoader
protected static PigServer pig;
protected static String defaultParameters= "init_address=localhost&rpc_port=9170&partitioner=org.apache.cassandra.dht.ByteOrderedPartitioner";
+ static
+ {
+ System.setProperty("log4j.configuration", "log4j-junit.properties");
+ }
+
@AfterClass
public static void oneTimeTearDown() throws Exception {
cluster.shutDown();
@@ -85,7 +90,7 @@ public class PigTestBase extends SchemaLoader
protected static Cassandra.Client getClient() throws TTransportException
{
- TTransport tr = new TFramedTransport(new TSocket("localhost", DatabaseDescriptor.getRpcPort()));
+ TTransport tr = new TFramedTransport(new TSocket("localhost", 9170));
TProtocol proto = new TBinaryProtocol(tr);
Cassandra.Client client = new Cassandra.Client(proto);
tr.open();
http://git-wip-us.apache.org/repos/asf/cassandra/blob/343a6472/test/pig/org/apache/cassandra/pig/ThriftColumnFamilyDataTypeTest.java
----------------------------------------------------------------------
diff --git a/test/pig/org/apache/cassandra/pig/ThriftColumnFamilyDataTypeTest.java b/test/pig/org/apache/cassandra/pig/ThriftColumnFamilyDataTypeTest.java
index 7bccc23..cc54620 100644
--- a/test/pig/org/apache/cassandra/pig/ThriftColumnFamilyDataTypeTest.java
+++ b/test/pig/org/apache/cassandra/pig/ThriftColumnFamilyDataTypeTest.java
@@ -17,27 +17,6 @@
* under the License.
*/
package org.apache.cassandra.pig;
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
import java.io.IOException;
import java.nio.charset.CharacterCodingException;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/343a6472/test/pig/org/apache/cassandra/pig/ThriftColumnFamilyTest.java
----------------------------------------------------------------------
diff --git a/test/pig/org/apache/cassandra/pig/ThriftColumnFamilyTest.java b/test/pig/org/apache/cassandra/pig/ThriftColumnFamilyTest.java
index 9369a18..60344d2 100644
--- a/test/pig/org/apache/cassandra/pig/ThriftColumnFamilyTest.java
+++ b/test/pig/org/apache/cassandra/pig/ThriftColumnFamilyTest.java
@@ -24,6 +24,7 @@ import java.nio.charset.CharacterCodingException;
import java.util.Iterator;
import org.apache.cassandra.cli.CliMain;
+import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.thrift.AuthenticationException;
import org.apache.cassandra.thrift.AuthorizationException;
import org.apache.cassandra.thrift.Cassandra;
@@ -182,8 +183,15 @@ public class ThriftColumnFamilyTest extends PigTestBase
"and comparator = LongType;"
};
+ private static String[] deleteCopyOfSomeAppTableData = { "use thriftKs;",
+ "DEL CopyOfSomeApp ['foo']",
+ "DEL CopyOfSomeApp ['bar']",
+ "DEL CopyOfSomeApp ['baz']",
+ "DEL CopyOfSomeApp ['qux']"
+ };
+
@BeforeClass
- public static void setup() throws TTransportException, IOException,
+ public static void setup() throws TTransportException, IOException, InterruptedException, ConfigurationException,
AuthenticationException, AuthorizationException, InvalidRequestException, UnavailableException, TimedOutException, TException, NotFoundException, CharacterCodingException, ClassNotFoundException, NoSuchFieldException, IllegalAccessException, InstantiationException
{
startCassandra();
@@ -192,7 +200,7 @@ public class ThriftColumnFamilyTest extends PigTestBase
}
@Test
- public void testCqlStorage() throws IOException
+ public void testCqlStorage() throws IOException, ClassNotFoundException, TException, TimedOutException, NotFoundException, InvalidRequestException, NoSuchFieldException, UnavailableException, IllegalAccessException, InstantiationException, AuthenticationException, AuthorizationException
{
//regular thrift column families
pig.registerQuery("data = load 'cql://thriftKs/SomeApp?" + defaultParameters + "' using CqlStorage();");
@@ -208,9 +216,8 @@ public class ThriftColumnFamilyTest extends PigTestBase
while (it.hasNext()) {
count ++;
Tuple t = it.next();
- if (count == 1)
+ if ("bar".equals(t.get(0)))
{
- Assert.assertEquals(t.get(0), "bar");
Assert.assertEquals(t.get(1), 3.141592653589793d);
Assert.assertEquals(t.get(3), "User Bar");
Assert.assertEquals(t.get(4), 35.0f);
@@ -218,16 +225,16 @@ public class ThriftColumnFamilyTest extends PigTestBase
Assert.assertEquals(t.get(6), 15000L);
Assert.assertEquals(t.get(7), "like");
}
- else if (count == 2)
+ else if ("baz".equals(t.get(0)))
{
- Assert.assertEquals(t.get(0), "baz");
Assert.assertEquals(t.get(1), 1.61803399d);
Assert.assertEquals(t.get(3), "User Baz");
Assert.assertEquals(t.get(4), 95.3f);
Assert.assertEquals(t.get(5), 3);
Assert.assertEquals(t.get(6), 512000L);
Assert.assertEquals(t.get(7), "dislike");
- }else if (count == 3)
+ }
+ else if ("foo".equals(t.get(0)))
{
Assert.assertEquals(t.get(0), "foo");
Assert.assertEquals(t.get(1), 2.718281828459045d);
@@ -237,7 +244,7 @@ public class ThriftColumnFamilyTest extends PigTestBase
Assert.assertEquals(t.get(6), 125000L);
Assert.assertEquals(t.get(7), "like");
}
- else if (count == 4)
+ else if ("qux".equals(t.get(0)))
{
Assert.assertEquals(t.get(0), "qux");
Assert.assertEquals(t.get(1), 0.660161815846869d);
@@ -262,18 +269,10 @@ public class ThriftColumnFamilyTest extends PigTestBase
while (it.hasNext()) {
count ++;
Tuple t = it.next();
- if (count == 1)
- {
- Assert.assertEquals(t.get(0), "chuck");
- Assert.assertEquals(t.get(1), "fist");
+ if ("chuck".equals(t.get(0)) && "fist".equals(t.get(1)))
Assert.assertEquals(t.get(2), 1L);
- }
- else if (count == 2)
- {
- Assert.assertEquals(t.get(0), "chuck");
- Assert.assertEquals(t.get(1), "kick");
+ else if ("chuck".equals(t.get(0)) && "kick".equals(t.get(1)))
Assert.assertEquals(t.get(2), 3L);
- }
}
Assert.assertEquals(count, 2);
@@ -291,40 +290,20 @@ public class ThriftColumnFamilyTest extends PigTestBase
while (it.hasNext()) {
count ++;
Tuple t = it.next();
- if (count == 1)
- {
- Assert.assertEquals(t.get(0), "kick");
- Assert.assertEquals(t.get(1), "bruce");
- Assert.assertEquals(t.get(2), "bruce");
+ if ("kick".equals(t.get(0)) && "bruce".equals(t.get(1)) && "bruce".equals(t.get(2)))
Assert.assertEquals(t.get(3), "watch it, mate");
- }
- else if (count == 2)
- {
- Assert.assertEquals(t.get(0), "kick");
- Assert.assertEquals(t.get(1), "bruce");
- Assert.assertEquals(t.get(2), "lee");
+ else if ("kick".equals(t.get(0)) && "bruce".equals(t.get(1)) && "lee".equals(t.get(2)))
Assert.assertEquals(t.get(3), "oww");
- }
- else if (count == 3)
- {
- Assert.assertEquals(t.get(0), "punch");
- Assert.assertEquals(t.get(1), "bruce");
- Assert.assertEquals(t.get(2), "bruce");
+ else if ("punch".equals(t.get(0)) && "bruce".equals(t.get(1)) && "bruce".equals(t.get(2)))
Assert.assertEquals(t.get(3), "hunh?");
- }
- else if (count == 4)
- {
- Assert.assertEquals(t.get(0), "punch");
- Assert.assertEquals(t.get(1), "bruce");
- Assert.assertEquals(t.get(2), "lee");
+ else if ("punch".equals(t.get(0)) && "bruce".equals(t.get(1)) && "lee".equals(t.get(2)))
Assert.assertEquals(t.get(3), "ouch");
- }
}
Assert.assertEquals(count, 4);
}
@Test
- public void testCassandraStorageSchema() throws IOException
+ public void testCassandraStorageSchema() throws IOException, ClassNotFoundException, TException, TimedOutException, NotFoundException, InvalidRequestException, NoSuchFieldException, UnavailableException, IllegalAccessException, InstantiationException
{
//results: (qux,(atomic_weight,0.660161815846869),(created,1335890877),(name,User Qux),(percent,64.7),
//(rating,2),(score,12000),(vote_type,dislike),{(extra1,extra1),
@@ -377,9 +356,8 @@ public class ThriftColumnFamilyTest extends PigTestBase
}
@Test
- public void testCassandraStorageFullCopy() throws IOException, TException, TimedOutException, NotFoundException, InvalidRequestException, NoSuchFieldException, UnavailableException, IllegalAccessException, InstantiationException, AuthenticationException, AuthorizationException
+ public void testCassandraStorageFullCopy() throws IOException, ClassNotFoundException, TException, TimedOutException, NotFoundException, InvalidRequestException, NoSuchFieldException, UnavailableException, IllegalAccessException, InstantiationException, AuthenticationException, AuthorizationException
{
- createColumnFamily("thriftKs", "CopyOfSomeApp", statements[3]);
pig.setBatchOn();
pig.registerQuery("rows = LOAD 'cassandra://thriftKs/SomeApp?" + defaultParameters + "' USING CassandraStorage();");
//full copy
@@ -391,9 +369,9 @@ public class ThriftColumnFamilyTest extends PigTestBase
}
@Test
- public void testCassandraStorageSigleTupleCopy() throws IOException, TException, TimedOutException, NotFoundException, InvalidRequestException, NoSuchFieldException, UnavailableException, IllegalAccessException, InstantiationException, AuthenticationException, AuthorizationException
+ public void testCassandraStorageSigleTupleCopy() throws IOException, ClassNotFoundException, TException, TimedOutException, NotFoundException, InvalidRequestException, NoSuchFieldException, UnavailableException, IllegalAccessException, InstantiationException, AuthenticationException, AuthorizationException
{
- createColumnFamily("thriftKs", "CopyOfSomeApp", statements[3]);
+ executeCliStatements(deleteCopyOfSomeAppTableData);
pig.setBatchOn();
pig.registerQuery("rows = LOAD 'cassandra://thriftKs/SomeApp?" + defaultParameters + "' USING CassandraStorage();");
//sigle tuple
@@ -425,9 +403,9 @@ public class ThriftColumnFamilyTest extends PigTestBase
}
@Test
- public void testCassandraStorageBagOnlyCopy() throws IOException, TException, TimedOutException, NotFoundException, InvalidRequestException, NoSuchFieldException, UnavailableException, IllegalAccessException, InstantiationException, AuthenticationException, AuthorizationException
+ public void testCassandraStorageBagOnlyCopy() throws IOException, ClassNotFoundException, TException, TimedOutException, NotFoundException, InvalidRequestException, NoSuchFieldException, UnavailableException, IllegalAccessException, InstantiationException, AuthenticationException, AuthorizationException
{
- createColumnFamily("thriftKs", "CopyOfSomeApp", statements[3]);
+ executeCliStatements(deleteCopyOfSomeAppTableData);
pig.setBatchOn();
pig.registerQuery("rows = LOAD 'cassandra://thriftKs/SomeApp?" + defaultParameters + "' USING CassandraStorage();");
//bag only
@@ -469,9 +447,9 @@ public class ThriftColumnFamilyTest extends PigTestBase
}
@Test
- public void testCassandraStorageFilter() throws IOException, TException, TimedOutException, NotFoundException, InvalidRequestException, NoSuchFieldException, UnavailableException, IllegalAccessException, InstantiationException, AuthenticationException, AuthorizationException
+ public void testCassandraStorageFilter() throws IOException, ClassNotFoundException, TException, TimedOutException, NotFoundException, InvalidRequestException, NoSuchFieldException, UnavailableException, IllegalAccessException, InstantiationException, AuthenticationException, AuthorizationException
{
- createColumnFamily("thriftKs", "CopyOfSomeApp", statements[3]);
+ executeCliStatements(deleteCopyOfSomeAppTableData);
pig.setBatchOn();
pig.registerQuery("rows = LOAD 'cassandra://thriftKs/SomeApp?" + defaultParameters + "' USING CassandraStorage();");
@@ -504,7 +482,7 @@ public class ThriftColumnFamilyTest extends PigTestBase
if (value != null)
Assert.fail();
- createColumnFamily("thriftKs", "CopyOfSomeApp", statements[3]);
+ executeCliStatements(deleteCopyOfSomeAppTableData);
pig.setBatchOn();
pig.registerQuery("rows = LOAD 'cassandra://thriftKs/SomeApp?" + defaultParameters + "' USING CassandraStorage();");
pig.registerQuery("dislikes_extras = FILTER rows by vote_type.value eq 'dislike' AND COUNT(columns) > 0;");
@@ -537,7 +515,7 @@ public class ThriftColumnFamilyTest extends PigTestBase
}
@Test
- public void testCassandraStorageJoin() throws IOException
+ public void testCassandraStorageJoin() throws IOException, ClassNotFoundException, TException, TimedOutException, NotFoundException, InvalidRequestException, NoSuchFieldException, UnavailableException, IllegalAccessException, InstantiationException, AuthenticationException, AuthorizationException
{
//test key types with a join
pig.registerQuery("U8 = load 'cassandra://thriftKs/U8?" + defaultParameters + "' using CassandraStorage();");
@@ -589,7 +567,7 @@ public class ThriftColumnFamilyTest extends PigTestBase
}
@Test
- public void testCassandraStorageCounterCF() throws IOException
+ public void testCassandraStorageCounterCF() throws IOException, ClassNotFoundException, TException, TimedOutException, NotFoundException, InvalidRequestException, NoSuchFieldException, UnavailableException, IllegalAccessException, InstantiationException, AuthenticationException, AuthorizationException
{
pig.registerQuery("rows = LOAD 'cassandra://thriftKs/SomeApp?" + defaultParameters + "' USING CassandraStorage();");
@@ -606,7 +584,7 @@ public class ThriftColumnFamilyTest extends PigTestBase
}
@Test
- public void testCassandraStorageCompositeColumnCF() throws IOException
+ public void testCassandraStorageCompositeColumnCF() throws IOException, ClassNotFoundException, TException, TimedOutException, NotFoundException, InvalidRequestException, NoSuchFieldException, UnavailableException, IllegalAccessException, InstantiationException, AuthenticationException, AuthorizationException
{
//Test CompositeType
pig.registerQuery("compo = load 'cassandra://thriftKs/Compo?" + defaultParameters + "' using CassandraStorage();");
@@ -620,16 +598,12 @@ public class ThriftColumnFamilyTest extends PigTestBase
while (it.hasNext()) {
count ++;
Tuple t = it.next();
- if (count == 1)
- Assert.assertEquals(t.get(0), "kick");
- else
- Assert.assertEquals(t.get(0), "punch");
Tuple t1 = (Tuple) t.get(1);
Assert.assertEquals(t1.get(0), "bruce");
Assert.assertEquals(t1.get(1), "lee");
- if (count == 1)
+ if ("kick".equals(t.get(0)))
Assert.assertEquals(t.get(2), "oww");
- else
+ else if ("kick".equals(t.get(0)))
Assert.assertEquals(t.get(2), "ouch");
}
Assert.assertEquals(count, 2);
@@ -665,37 +639,21 @@ public class ThriftColumnFamilyTest extends PigTestBase
count ++;
Tuple t1 = iter.next();
Tuple inner = (Tuple) t1.get(0);
- if (count == 1)
- {
- Assert.assertEquals(inner.get(0), 1L);
- Assert.assertEquals(inner.get(1), 0L);
+ if ((Long) inner.get(0) == 1L && (Long) inner.get(1) == 0L)
Assert.assertEquals(t1.get(1), "z");
- }
- else if (count == 2)
- {
- Assert.assertEquals(inner.get(0), 1L);
- Assert.assertEquals(inner.get(1), 30L);
+ else if ((Long) inner.get(0) == 1L && (Long) inner.get(1) == 30L)
Assert.assertEquals(t1.get(1), "zzzz");
- }
- else if (count == 3)
- {
- Assert.assertEquals(inner.get(0), 2L);
- Assert.assertEquals(inner.get(1), 30L);
+ else if ((Long) inner.get(0) == 2L && (Long) inner.get(1) == 30L)
Assert.assertEquals(t1.get(1), "daddy?");
- }
- else if (count == 4)
- {
- Assert.assertEquals(inner.get(0), 6L);
- Assert.assertEquals(inner.get(1), 30L);
+ else if ((Long) inner.get(0) == 6L && (Long) inner.get(1) == 30L)
Assert.assertEquals(t1.get(1), "coffee...");
- }
}
Assert.assertEquals(count, 4);
}
}
@Test
- public void testCassandraStorageCompositeKeyCF() throws IOException
+ public void testCassandraStorageCompositeKeyCF() throws IOException, ClassNotFoundException, TException, TimedOutException, NotFoundException, InvalidRequestException, NoSuchFieldException, UnavailableException, IllegalAccessException, InstantiationException, AuthenticationException, AuthorizationException
{
//Test CompositeKey
pig.registerQuery("compokeys = load 'cassandra://thriftKs/CompoKey?" + defaultParameters + "' using CassandraStorage();");
@@ -730,11 +688,9 @@ public class ThriftColumnFamilyTest extends PigTestBase
while (it.hasNext()) {
Tuple t = it.next();
count ++;
- if (count == 1)
+ Tuple key = (Tuple) t.get(0);
+ if ("clock".equals(key.get(0)) && (Long) key.get(1) == 10L)
{
- Tuple key = (Tuple) t.get(0);
- Assert.assertEquals(key.get(0), "clock");
- Assert.assertEquals(key.get(1), 10L);
DataBag columns = (DataBag) t.get(1);
Iterator<Tuple> iter = columns.iterator();
if (iter.hasNext())
@@ -744,46 +700,37 @@ public class ThriftColumnFamilyTest extends PigTestBase
Assert.assertEquals(t1.get(1), "z");
}
}
- else if (count == 2)
+ else if ("clock".equals(key.get(0)) && (Long) key.get(1) == 40L)
{
- Tuple key = (Tuple) t.get(0);
- Assert.assertEquals(key.get(0), "clock");
- Assert.assertEquals(key.get(1), 20L);
DataBag columns = (DataBag) t.get(1);
Iterator<Tuple> iter = columns.iterator();
if (iter.hasNext())
{
Tuple t1 = iter.next();
- Assert.assertEquals(t1.get(0), 1L);
- Assert.assertEquals(t1.get(1), "zzzz");
+ Assert.assertEquals(t1.get(0), 6L);
+ Assert.assertEquals(t1.get(1), "coffee...");
}
}
- else if (count == 3)
+ else if ("clock".equals(key.get(0)) && (Long) key.get(1) == 20L)
{
- Tuple key = (Tuple) t.get(0);
- Assert.assertEquals(key.get(0), "clock");
- Assert.assertEquals(key.get(1), 30L);
DataBag columns = (DataBag) t.get(1);
Iterator<Tuple> iter = columns.iterator();
if (iter.hasNext())
{
Tuple t1 = iter.next();
- Assert.assertEquals(t1.get(0), 2L);
- Assert.assertEquals(t1.get(1), "daddy?");
+ Assert.assertEquals(t1.get(0), 1L);
+ Assert.assertEquals(t1.get(1), "zzzz");
}
}
- else if (count == 4)
+ else if ("clock".equals(key.get(0)) && (Long) key.get(1) == 30L)
{
- Tuple key = (Tuple) t.get(0);
- Assert.assertEquals(key.get(0), "clock");
- Assert.assertEquals(key.get(1), 40L);
DataBag columns = (DataBag) t.get(1);
Iterator<Tuple> iter = columns.iterator();
if (iter.hasNext())
{
Tuple t1 = iter.next();
- Assert.assertEquals(t1.get(0), 6L);
- Assert.assertEquals(t1.get(1), "coffee...");
+ Assert.assertEquals(t1.get(0), 2L);
+ Assert.assertEquals(t1.get(1), "daddy?");
}
}
}
@@ -797,7 +744,6 @@ public class ThriftColumnFamilyTest extends PigTestBase
client.set_keyspace(ks);
ByteBuffer key_user_id = ByteBufferUtil.bytes(key);
-
ColumnPath cp = new ColumnPath(cf);
cp.column = ByteBufferUtil.bytes(colName);
@@ -806,17 +752,16 @@ public class ThriftColumnFamilyTest extends PigTestBase
return parseType(validator).getString(got.getColumn().value);
}
- private void createColumnFamily(String ks, String cf, String statement) throws CharacterCodingException, TException, TimedOutException, NotFoundException, InvalidRequestException, NoSuchFieldException, UnavailableException, IllegalAccessException, InstantiationException
+ private void executeCliStatements(String[] statements) throws CharacterCodingException, ClassNotFoundException, TException, TimedOutException, NotFoundException, InvalidRequestException, NoSuchFieldException, UnavailableException, IllegalAccessException, InstantiationException
{
CliMain.connect("127.0.0.1", 9170);
try
{
- CliMain.processStatement("use " + ks + ";");
- CliMain.processStatement("drop column family " + cf + ";");
+ for (String stmt : statements)
+ CliMain.processStatement(stmt);
}
catch (Exception e)
{
}
- CliMain.processStatement(statement);
}
}