You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by al...@apache.org on 2015/10/16 14:28:25 UTC
[1/5] cassandra git commit: Deprecate Pig support
Repository: cassandra
Updated Branches:
refs/heads/cassandra-3.0 e1fb18a00 -> 56cfc6ea3
Deprecate Pig support
patch by Aleksey Yeschenko; reviewed by Sylvain Lebresne for
CASSANDRA-10542
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/def58031
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/def58031
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/def58031
Branch: refs/heads/cassandra-3.0
Commit: def58031d54fef239a4051201d097349da05ad50
Parents: 94a7d06
Author: Aleksey Yeschenko <al...@apache.org>
Authored: Fri Oct 16 13:16:49 2015 +0100
Committer: Aleksey Yeschenko <al...@apache.org>
Committed: Fri Oct 16 13:16:49 2015 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
NEWS.txt | 5 +++++
src/java/org/apache/cassandra/hadoop/pig/CqlNativeStorage.java | 2 +-
src/java/org/apache/cassandra/hadoop/pig/StorageHelper.java | 1 +
4 files changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/def58031/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 9a0baaa..632e262 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.2.4
+ * Deprecate Pig support (CASSANDRA-10542)
* Reduce contention getting instances of CompositeType (CASSANDRA-10433)
Merged from 2.1:
* (cqlsh) Distinguish negative and positive infinity in output (CASSANDRA-10523)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/def58031/NEWS.txt
----------------------------------------------------------------------
diff --git a/NEWS.txt b/NEWS.txt
index a235a1d..4fa61c0 100644
--- a/NEWS.txt
+++ b/NEWS.txt
@@ -16,6 +16,11 @@ using the provided 'sstableupgrade' tool.
2.2.4
=====
+Deprecation
+-----------
+ - Pig support has been deprecated, and will be removed in 3.0.
+ Please see CASSANDRA-10542 for more details.
+
Operations
----------
- Switching racks is no longer an allowed operation on a node which has
http://git-wip-us.apache.org/repos/asf/cassandra/blob/def58031/src/java/org/apache/cassandra/hadoop/pig/CqlNativeStorage.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/hadoop/pig/CqlNativeStorage.java b/src/java/org/apache/cassandra/hadoop/pig/CqlNativeStorage.java
index bdf99a0..ba0a37d 100644
--- a/src/java/org/apache/cassandra/hadoop/pig/CqlNativeStorage.java
+++ b/src/java/org/apache/cassandra/hadoop/pig/CqlNativeStorage.java
@@ -61,7 +61,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.external.biz.base64Coder.Base64Coder;
-
+@Deprecated
public class CqlNativeStorage extends LoadFunc implements StoreFuncInterface, LoadMetadata
{
protected String DEFAULT_INPUT_FORMAT;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/def58031/src/java/org/apache/cassandra/hadoop/pig/StorageHelper.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/hadoop/pig/StorageHelper.java b/src/java/org/apache/cassandra/hadoop/pig/StorageHelper.java
index 66836b2..d700cb7 100644
--- a/src/java/org/apache/cassandra/hadoop/pig/StorageHelper.java
+++ b/src/java/org/apache/cassandra/hadoop/pig/StorageHelper.java
@@ -16,6 +16,7 @@ import org.apache.pig.data.DataByteArray;
import org.apache.pig.data.DataType;
import org.apache.pig.data.Tuple;
+@Deprecated
public class StorageHelper
{
// system environment variables that can be set to configure connection info:
[5/5] cassandra git commit: Remove Pig support
Posted by al...@apache.org.
Remove Pig support
patch by Aleksey Yeschenko; reviewed by Sylvain Lebresne for
CASSANDRA-10542
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/56cfc6ea
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/56cfc6ea
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/56cfc6ea
Branch: refs/heads/cassandra-3.0
Commit: 56cfc6ea35d1410f2f5a8ae711ae33342f286d79
Parents: 56d8351
Author: Aleksey Yeschenko <al...@apache.org>
Authored: Fri Oct 16 13:23:20 2015 +0100
Committer: Aleksey Yeschenko <al...@apache.org>
Committed: Fri Oct 16 13:23:20 2015 +0100
----------------------------------------------------------------------
.rat-excludes | 1 -
CHANGES.txt | 1 +
NEWS.txt | 2 +-
build.xml | 67 +-
examples/pig/README.txt | 118 --
examples/pig/bin/pig_cassandra | 46 -
examples/pig/example-populate-cql.txt | 24 -
examples/pig/example-script-cql.pig | 11 -
examples/pig/example-script.pig | 9 -
examples/pig/libdata.csv | 200 ----
examples/pig/test/populate-cli.txt | 134 ---
examples/pig/test/populate-cql.txt | 33 -
examples/pig/test/test_cql_storage.pig | 14 -
examples/pig/test/test_storage.pig | 85 --
.../cassandra/hadoop/pig/CqlNativeStorage.java | 1082 ------------------
.../cassandra/hadoop/pig/StorageHelper.java | 122 --
test/conf/cassandra_pig.yaml | 42 -
.../cassandra/pig/CqlRecordReaderTest.java | 150 ---
.../cassandra/pig/CqlTableDataTypeTest.java | 479 --------
.../org/apache/cassandra/pig/CqlTableTest.java | 303 -----
.../org/apache/cassandra/pig/PigTestBase.java | 117 --
.../pig/ThriftColumnFamilyDataTypeTest.java | 150 ---
.../cassandra/pig/ThriftColumnFamilyTest.java | 617 ----------
test/pig/org/apache/pig/test/MiniCluster.java | 86 --
.../org/apache/pig/test/MiniGenericCluster.java | 122 --
.../apache/pig/test/WindowsLocalFileSystem.java | 62 -
26 files changed, 11 insertions(+), 4066 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/56cfc6ea/.rat-excludes
----------------------------------------------------------------------
diff --git a/.rat-excludes b/.rat-excludes
index d95b499..6df36a5 100644
--- a/.rat-excludes
+++ b/.rat-excludes
@@ -24,7 +24,6 @@ test/data/serialization/*/*
doc/**
build.properties.default
test/data/**
-examples/pig/**
examples/triggers/build.xml
examples/triggers/conf/*
examples/hadoop_word_count/conf/log4j.properties
http://git-wip-us.apache.org/repos/asf/cassandra/blob/56cfc6ea/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index dcacc69..6bdaa04 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
3.0-rc2
+ * Remove Pig support (CASSANDRA-10542)
* Fix LogFile throws Exception when assertion is disabled (CASSANDRA-10522)
* Revert CASSANDRA-7486, make CMS default GC, move GC config to
conf/jvm.options (CASSANDRA-10403)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/56cfc6ea/NEWS.txt
----------------------------------------------------------------------
diff --git a/NEWS.txt b/NEWS.txt
index a18ce6b..48a0733 100644
--- a/NEWS.txt
+++ b/NEWS.txt
@@ -84,7 +84,7 @@ Upgrading
(schema versions outputted by `nodetool describecluster` are all the same).
- Schema metadata is now stored in the new `system_schema` keyspace, and
legacy `system.schema_*` tables are now gone; see CASSANDRA-6717 for details.
- - Pig's CassandraStorage has been removed. Use CqlNativeStorage instead.
+ - Pig's support has been removed.
- Hadoop BulkOutputFormat and BulkRecordWriter have been removed; use
CqlBulkOutputFormat and CqlBulkRecordWriter instead.
- Hadoop ColumnFamilyInputFormat and ColumnFamilyOutputFormat have been removed;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/56cfc6ea/build.xml
----------------------------------------------------------------------
diff --git a/build.xml b/build.xml
index 87f98cc..e039bd2 100644
--- a/build.xml
+++ b/build.xml
@@ -63,7 +63,6 @@
<property name="test.long.src" value="${test.dir}/long"/>
<property name="test.burn.src" value="${test.dir}/burn"/>
<property name="test.microbench.src" value="${test.dir}/microbench"/>
- <property name="test.pig.src" value="${test.dir}/pig"/>
<property name="dist.dir" value="${build.dir}/dist"/>
<property name="tmp.dir" value="${java.io.tmpdir}"/>
@@ -386,17 +385,12 @@
<dependency groupId="org.apache.hadoop" artifactId="hadoop-minicluster" version="1.0.3">
<exclusion groupId="asm" artifactId="asm"/> <!-- this is the outdated version 3.1 -->
</dependency>
- <dependency groupId="org.apache.pig" artifactId="pig" version="0.12.1">
- <exclusion groupId="org.mortbay.jetty" artifactId="jetty"/>
- <exclusion groupId="hsqldb" artifactId="hsqldb"/> <!-- outdated version -->
- <exclusion groupId="antlr" artifactId="antlr"/> <!-- outdated version -->
- </dependency>
<dependency groupId="net.java.dev.jna" artifactId="jna" version="4.0.0"/>
<dependency groupId="net.sourceforge.cobertura" artifactId="cobertura" version="${cobertura.version}">
<exclusion groupId="xerces" artifactId="xercesImpl"/>
- <exclusion groupId="org.mortbay.jetty" artifactId="jetty"/> <!-- older version, also via hadoop-core + pig -->
- <exclusion groupId="org.mortbay.jetty" artifactId="jetty-util"/> <!-- older version, also via hadoop-core + pig -->
+ <exclusion groupId="org.mortbay.jetty" artifactId="jetty"/> <!-- older version, also via hadoop-core -->
+ <exclusion groupId="org.mortbay.jetty" artifactId="jetty-util"/> <!-- older version, also via hadoop-core -->
<exclusion groupId="org.apache.ant" artifactId="ant"/> <!-- older version 1.8.3 -->
</dependency>
<dependency groupId="org.jacoco" artifactId="org.jacoco.agent" version="${jacoco.version}"/>
@@ -467,7 +461,6 @@
<dependency groupId="org.apache.rat" artifactId="apache-rat"/>
<dependency groupId="org.apache.hadoop" artifactId="hadoop-core"/>
<dependency groupId="org.apache.hadoop" artifactId="hadoop-minicluster"/>
- <dependency groupId="org.apache.pig" artifactId="pig"/>
<dependency groupId="com.google.code.findbugs" artifactId="jsr305"/>
<dependency groupId="org.antlr" artifactId="antlr"/>
<!-- TODO CASSANDRA-9543
@@ -489,15 +482,6 @@
artifactId="cassandra-parent"
version="${version}"/>
<dependency groupId="junit" artifactId="junit"/>
- <dependency groupId="org.apache.pig" artifactId="pig">
- <exclusion groupId="xmlenc" artifactId="xmlenc"/>
- <exclusion groupId="tomcat" artifactId="jasper-runtime"/>
- <exclusion groupId="tomcat" artifactId="jasper-compiler"/>
- <exclusion groupId="org.eclipse.jdt" artifactId="core"/>
- <exclusion groupId="net.sf.kosmosfs" artifactId="kfs"/>
- <exclusion groupId="hsqldb" artifactId="hsqldb"/>
- <exclusion groupId="antlr" artifactId="antlr"/>
- </dependency>
<!-- TODO CASSANDRA-9543
<dependency groupId="com.datastax.cassandra" artifactId="cassandra-driver-core" classifier="shaded"/>
-->
@@ -524,7 +508,6 @@
<parent groupId="org.apache.cassandra"
artifactId="cassandra-parent"
version="${version}"/>
- <!-- do NOT remove this, it breaks pig-test -->
<dependency groupId="joda-time" artifactId="joda-time"/>
</artifact:pom>
@@ -572,7 +555,6 @@
<!-- don't need hadoop classes to run, but if you use the hadoop stuff -->
<dependency groupId="org.apache.hadoop" artifactId="hadoop-core" optional="true"/>
<dependency groupId="org.apache.hadoop" artifactId="hadoop-minicluster" optional="true"/>
- <dependency groupId="org.apache.pig" artifactId="pig" optional="true"/>
<!-- TODO CASSANDRA-9543
<dependency groupId="com.datastax.cassandra" artifactId="cassandra-driver-core" classifier="shaded" optional="true"/>
-->
@@ -694,26 +676,6 @@
</copy>
</target>
- <target name="maven-ant-tasks-retrieve-pig-test" depends="maven-ant-tasks-init">
- <artifact:dependencies pomRefId="test-deps-pom"
- filesetId="test-dependency-jars"
- sourcesFilesetId="test-dependency-sources"
- cacheDependencyRefs="true"
- dependencyRefsBuildFile="${build.dir}/test-dependencies.xml">
- <remoteRepository refid="apache"/>
- <remoteRepository refid="central"/>
- <remoteRepository refid="java.net2"/>
- </artifact:dependencies>
- <copy todir="${build.dir.lib}/jars">
- <fileset refid="test-dependency-jars"/>
- <mapper type="flatten"/>
- </copy>
- <copy todir="${build.dir.lib}/sources">
- <fileset refid="test-dependency-sources"/>
- <mapper type="flatten"/>
- </copy>
- </target>
-
<!--
Generate thrift code. We have targets to build java because
Cassandra depends on it, and python because that is what the system
@@ -1167,7 +1129,6 @@
<src path="${test.unit.src}"/>
<src path="${test.long.src}"/>
<src path="${test.burn.src}"/>
- <src path="${test.pig.src}"/>
<src path="${test.microbench.src}"/>
</javac>
@@ -1282,7 +1243,7 @@
</target>
<target name="testold" depends="build-test" description="Execute unit tests">
- <testmacro inputdir="${test.unit.src}" exclude="**/pig/*.java" timeout="${test.timeout}">
+ <testmacro inputdir="${test.unit.src}" timeout="${test.timeout}">
<jvmarg value="-Dlegacy-sstable-root=${test.data}/legacy-sstables"/>
<jvmarg value="-Dinvalid-legacy-sstable-root=${test.data}/invalid-legacy-sstables"/>
<jvmarg value="-Dmigration-sstable-root=${test.data}/migration-sstables"/>
@@ -1291,9 +1252,7 @@
<jvmarg value="-Djava.io.tmpdir=${tmp.dir}"/>
<jvmarg value="-Dcassandra.skip_sync=true" />
</testmacro>
- <fileset dir="${test.unit.src}">
- <exclude name="**/pig/*.java" />
- </fileset>
+ <fileset dir="${test.unit.src}" />
</target>
<!-- Will not generate a junit report or fail on error since it is called in parallel for test-compression
@@ -1337,7 +1296,7 @@
</macrodef>
<!--
- Run named ant task with jacoco, such as "ant jacoco-run -Dtaskname=pig-test"
+ Run named ant task with jacoco, such as "ant jacoco-run -Dtaskname=test"
the target run must enable the jacoco agent if usejacoco is 'yes' -->
<target name="jacoco-run" description="run named task with jacoco instrumentation">
<condition property="runtask" value="${taskname}" else="test">
@@ -1352,7 +1311,7 @@
ant testsome -Dtest.name=org.apache.cassandra.service.StorageServiceServerTest -Dtest.methods=testRegularMode,testGetAllRangesEmpty
-->
<target name="testsome" depends="build-test" description="Execute specific unit tests" >
- <testmacro inputdir="${test.unit.src}" exclude="**/pig/*.java" timeout="${test.timeout}">
+ <testmacro inputdir="${test.unit.src}" timeout="${test.timeout}">
<test name="${test.name}" methods="${test.methods}"/>
<jvmarg value="-Dlegacy-sstable-root=${test.data}/legacy-sstables"/>
<jvmarg value="-Dinvalid-legacy-sstable-root=${test.data}/invalid-legacy-sstables"/>
@@ -1370,7 +1329,7 @@
<fileset file="${test.conf}/commitlog_compression.yaml"/>
</concat>
<path id="all-test-classes-path">
- <fileset dir="${test.unit.src}" excludes="**/pig/*.java" includes="**/${test.name}.java" />
+ <fileset dir="${test.unit.src}" includes="**/${test.name}.java" />
</path>
<property name="all-test-classes" refid="all-test-classes-path"/>
<testparallel testdelegate="testlist-compression" />
@@ -1489,14 +1448,8 @@
</sequential>
</target>
- <target name="pig-test" depends="build-test,maven-ant-tasks-retrieve-pig-test" description="Excute Pig tests">
- <testmacro inputdir="${test.pig.src}"
- timeout="1200000">
- </testmacro>
- </target>
-
<target name="test-all"
- depends="eclipse-warnings,test,long-test,test-compression,pig-test,test-clientutil-jar"
+ depends="eclipse-warnings,test,long-test,test-compression,test-clientutil-jar"
description="Run all tests except for those under test-burn" />
<!-- Use JaCoCo ant extension without needing externally saved lib -->
@@ -1716,7 +1669,7 @@
<target name="test" depends="build-test" description="Parallel Test Runner">
<path id="all-test-classes-path">
- <fileset dir="${test.unit.src}" excludes="**/pig/*.java" includes="**/${test.name}.java" />
+ <fileset dir="${test.unit.src}" includes="**/${test.name}.java" />
</path>
<property name="all-test-classes" refid="all-test-classes-path"/>
<testparallel testdelegate="testlist"/>
@@ -1756,7 +1709,6 @@
<sourceFolder url="file://$MODULE_DIR$/tools/stress/src" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/test/unit" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/test/long" isTestSource="true" />
- <sourceFolder url="file://$MODULE_DIR$/test/pig" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/test/microbench" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/test/resources" type="java-test-resource" />
<excludeFolder url="file://$MODULE_DIR$/.idea" />
@@ -1964,7 +1916,6 @@
<classpathentry kind="src" path="interface/thrift/gen-java"/>
<classpathentry kind="src" output="build/test/classes" path="test/unit"/>
<classpathentry kind="src" output="build/test/classes" path="test/long"/>
- <classpathentry kind="src" output="build/test/classes" path="test/pig"/>
<classpathentry kind="src" output="build/test/classes" path="test/resources" />
<classpathentry kind="src" path="tools/stress/src"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
http://git-wip-us.apache.org/repos/asf/cassandra/blob/56cfc6ea/examples/pig/README.txt
----------------------------------------------------------------------
diff --git a/examples/pig/README.txt b/examples/pig/README.txt
deleted file mode 100644
index 1553a9f..0000000
--- a/examples/pig/README.txt
+++ /dev/null
@@ -1,118 +0,0 @@
-A Pig storage class that reads all columns from a given ColumnFamily, or writes
-properly formatted results into a ColumnFamily.
-
-Getting Started
-===============
-
-First build and start a Cassandra server with the default
-configuration and set the PIG_HOME and JAVA_HOME environment
-variables to the location of a Pig >= 0.7.0 install and your Java
-install.
-
-If you would like to run using the Hadoop backend, you should
-also set PIG_CONF_DIR to the location of your Hadoop config.
-
-Finally, set the following as environment variables (uppercase,
-underscored), or as Hadoop configuration variables (lowercase, dotted):
-* PIG_INITIAL_ADDRESS or cassandra.thrift.address : initial address to connect to
-* PIG_RPC_PORT or cassandra.thrift.port : the port thrift is listening on
-* PIG_PARTITIONER or cassandra.partitioner.class : cluster partitioner
-
-For example, against a local node with the default settings, you'd use:
-export PIG_INITIAL_ADDRESS=localhost
-export PIG_RPC_PORT=9160
-export PIG_PARTITIONER=org.apache.cassandra.dht.Murmur3Partitioner
-
-These properties can be overridden with the following if you use different clusters
-for input and output:
-* PIG_INPUT_INITIAL_ADDRESS : initial address to connect to for reading
-* PIG_INPUT_RPC_PORT : the port thrift is listening on for reading
-* PIG_INPUT_PARTITIONER : cluster partitioner for reading
-* PIG_OUTPUT_INITIAL_ADDRESS : initial address to connect to for writing
-* PIG_OUTPUT_RPC_PORT : the port thrift is listening on for writing
-* PIG_OUTPUT_PARTITIONER : cluster partitioner for writing
-
-CassandraStorage
-================
-
-The CassandraStorage class is for any non-CQL3 ColumnFamilies you may have. For CQL3 support, refer to the CqlNativeStorage section.
-
-examples/pig$ bin/pig_cassandra -x local example-script.pig
-
-This will run the test script against your Cassandra instance
-and will assume that there is a MyKeyspace/MyColumnFamily with some
-data in it. It will run in local mode (see pig docs for more info).
-
-If you'd like to get to a 'grunt>' shell prompt, run:
-
-examples/pig$ bin/pig_cassandra -x local
-
-Once the 'grunt>' shell has loaded, try a simple program like the
-following, which will determine the top 50 column names:
-
-grunt> rows = LOAD 'cassandra://MyKeyspace/MyColumnFamily' USING CassandraStorage();
-grunt> cols = FOREACH rows GENERATE flatten(columns);
-grunt> colnames = FOREACH cols GENERATE $0;
-grunt> namegroups = GROUP colnames BY (chararray) $0;
-grunt> namecounts = FOREACH namegroups GENERATE COUNT($1), group;
-grunt> orderednames = ORDER namecounts BY $0 DESC;
-grunt> topnames = LIMIT orderednames 50;
-grunt> dump topnames;
-
-Slices on columns can also be specified:
-grunt> rows = LOAD 'cassandra://MyKeyspace/MyColumnFamily?slice_start=C2&slice_end=C4&limit=1&reversed=true' USING CassandraStorage();
-
-Binary values for slice_start and slice_end can be escaped such as '\u0255'
-
-Outputting to Cassandra requires the same format from input, so the simplest example is:
-
-grunt> rows = LOAD 'cassandra://MyKeyspace/MyColumnFamily' USING CassandraStorage();
-grunt> STORE rows into 'cassandra://MyKeyspace/MyColumnFamily' USING CassandraStorage();
-
-Which will copy the ColumnFamily. Note that the destination ColumnFamily must
-already exist for this to work.
-
-See the example in test/ to see how schema is inferred.
-
-Advanced Options for CassandraStorage
-=====================================
-
-The following environment variables default to false but can be set to true to enable them:
-
-PIG_WIDEROW_INPUT: this enables loading of rows with many columns without
- incurring memory pressure. All columns will be in a bag and indexes are not
- supported. This can also be set in the LOAD url by adding
- the 'widerows=true' parameter.
-
-PIG_USE_SECONDARY: this allows easy use of secondary indexes within your
- script, by appending every index to the schema as 'index_$name', allowing
- filtering of loaded rows with a statement like "FILTER rows BY index_color eq
- 'blue'" if you have an index called 'color' defined. This
- can also be set in the LOAD url by adding the
- 'use_secondary=true' parameter.
-
-PIG_INPUT_SPLIT_SIZE: this sets the split size passed to Hadoop, controlling
- the amount of mapper tasks created. This can also be set in the LOAD url by
- adding the 'split_size=X' parameter, where X is an integer amount for the size.
-
-CqlNativeStorage
-================
-
-The CqlNativeStorage class is somewhat similar to CassandraStorage, but it can work with CQL3-defined ColumnFamilies. The main difference is in the URL format:
-
-cql://[username:password@]<keyspace>/<columnfamily>
- [?[page_size=<size>][&columns=<col1,col2>][&output_query=<prepared_statement>]
- [&where_clause=<clause>][&split_size=<size>][&partitioner=<partitioner>][&use_secondary=true|false]
- [&init_address=<host>][&native_port=<native_port>][&core_conns=<core_conns>]
- [&max_conns=<max_conns>][&min_simult_reqs=<min_simult_reqs>][&max_simult_reqs=<max_simult_reqs>]
- [&native_timeout=<native_timeout>][&native_read_timeout=<native_read_timeout>][&rec_buff_size=<rec_buff_size>]
- [&send_buff_size=<send_buff_size>][&solinger=<solinger>][&tcp_nodelay=<tcp_nodelay>][&reuse_address=<reuse_address>]
- [&keep_alive=<keep_alive>][&auth_provider=<auth_provider>][&trust_store_path=<trust_store_path>]
- [&key_store_path=<key_store_path>][&trust_store_password=<trust_store_password>]
- [&key_store_password=<key_store_password>][&cipher_suites=<cipher_suites>][&input_cql=<input_cql>]
- [columns=<columns>][where_clause=<where_clause>]]
-Which in grunt, the simplest example would look like:
-
-grunt> rows = LOAD 'cql://MyKeyspace/MyColumnFamily' USING CqlNativeStorage();
-
-CqlNativeStorage handles wide rows automatically and thus has no separate flag for this.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/56cfc6ea/examples/pig/bin/pig_cassandra
----------------------------------------------------------------------
diff --git a/examples/pig/bin/pig_cassandra b/examples/pig/bin/pig_cassandra
deleted file mode 100755
index 5d98888..0000000
--- a/examples/pig/bin/pig_cassandra
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/bin/sh
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-cwd=`dirname $0`
-cassandra_home="$cwd/../../../"
-
-# general jars.
-for jar in $cassandra_home/lib/*.jar $cassandra_home/build/lib/jars/*.jar $cassandra_home/build/apache-cassandra*.jar; do
- CLASSPATH=$CLASSPATH:$jar
-done
-
-if [ "x$PIG_HOME" = "x" ]; then
- echo "PIG_HOME not set: requires Pig >= 0.7.0" >&2
- exit 1
-fi
-
-# pig jar.
-for jar in $PIG_HOME/*.jar; do
- PIG_JAR=$jar
-done
-echo "Using $PIG_JAR."
-if [ ! -e $PIG_JAR ]; then
- echo "Unable to locate Pig jar" >&2
- exit 1
-fi
-
-CLASSPATH=$CLASSPATH:$PIG_JAR
-
-export PIG_CLASSPATH=$PIG_CLASSPATH:$CLASSPATH
-export PIG_OPTS="$PIG_OPTS -Dudf.import.list=org.apache.cassandra.hadoop.pig"
-$PIG_HOME/bin/pig $*
http://git-wip-us.apache.org/repos/asf/cassandra/blob/56cfc6ea/examples/pig/example-populate-cql.txt
----------------------------------------------------------------------
diff --git a/examples/pig/example-populate-cql.txt b/examples/pig/example-populate-cql.txt
deleted file mode 100644
index 8e23582..0000000
--- a/examples/pig/example-populate-cql.txt
+++ /dev/null
@@ -1,24 +0,0 @@
-CREATE KEYSPACE libdata WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1 };
-USE libdata;
-
-CREATE TABLE libout ("STABR" TEXT, "FSCSKEY" TEXT, "FSCS_SEQ" TEXT,
- "LIBID" TEXT, "LIBNAME" TEXT, "ADDRESS" TEXT, "CITY" TEXT,
- "ZIP" TEXT, "ZIP4" TEXT, "CNTY" TEXT, "PHONE" TEXT, "C_OUT_TY" TEXT,
- "C_MSA" TEXT, "SQ_FEET" INT, "F_SQ_FT" TEXT, "L_NUM_BM" INT,
- "F_BKMOB" TEXT, "HOURS" INT, "F_HOURS" TEXT, "WKS_OPEN" INT,
- "F_WKSOPN" TEXT, "YR_SUB" INT, "STATSTRU" INT, "STATNAME" INT,
- "STATADDR" INT, "LONGITUD" FLOAT, "LATITUDE" FLOAT, "FIPSST" INT,
- "FIPSCO" INT, "FIPSPLAC" INT, "CNTYPOP" INT, "LOCALE" TEXT,
- "CENTRACT" FLOAT, "CENBLOCK" INT, "CDCODE" TEXT, "MAT_CENT" TEXT,
- "MAT_TYPE" INT, "CBSA" INT, "MICROF" TEXT,
- PRIMARY KEY ("FSCSKEY", "FSCS_SEQ"));
-
-COPY libout ("STABR","FSCSKEY","FSCS_SEQ","LIBID","LIBNAME",
- "ADDRESS","CITY","ZIP","ZIP4","CNTY","PHONE","C_OUT_TY",
- "C_MSA","SQ_FEET","F_SQ_FT","L_NUM_BM","F_BKMOB","HOURS",
- "F_HOURS","WKS_OPEN","F_WKSOPN","YR_SUB","STATSTRU","STATNAME",
- "STATADDR","LONGITUD","LATITUDE","FIPSST","FIPSCO","FIPSPLAC",
- "CNTYPOP","LOCALE","CENTRACT","CENBLOCK","CDCODE","MAT_CENT",
- "MAT_TYPE","CBSA","MICROF") FROM 'libdata.csv' WITH HEADER=TRUE;
-
-CREATE TABLE libsqft (year INT, state TEXT, sqft BIGINT, PRIMARY KEY (year, state));
http://git-wip-us.apache.org/repos/asf/cassandra/blob/56cfc6ea/examples/pig/example-script-cql.pig
----------------------------------------------------------------------
diff --git a/examples/pig/example-script-cql.pig b/examples/pig/example-script-cql.pig
deleted file mode 100644
index ef11130..0000000
--- a/examples/pig/example-script-cql.pig
+++ /dev/null
@@ -1,11 +0,0 @@
--- CqlNativeStorage
-libdata = LOAD 'cql://libdata/libout' USING CqlNativeStorage();
-book_by_mail = FILTER libdata BY C_OUT_TY == 'BM';
-
-libdata_buildings = FILTER libdata BY SQ_FEET > 0;
-state_flat = FOREACH libdata_buildings GENERATE STABR AS State,SQ_FEET AS SquareFeet;
-state_grouped = GROUP state_flat BY State;
-state_footage = FOREACH state_grouped GENERATE group AS State, SUM(state_flat.SquareFeet) AS TotalFeet:int;
-
-insert_format= FOREACH state_footage GENERATE TOTUPLE(TOTUPLE('year',2011),TOTUPLE('state',State)),TOTUPLE(TotalFeet);
-STORE insert_format INTO 'cql://libdata/libsqft?output_query=UPDATE%20libdata.libsqft%20SET%20sqft%20%3D%20%3F' USING CqlNativeStorage;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/cassandra/blob/56cfc6ea/examples/pig/example-script.pig
----------------------------------------------------------------------
diff --git a/examples/pig/example-script.pig b/examples/pig/example-script.pig
deleted file mode 100644
index d47fc28..0000000
--- a/examples/pig/example-script.pig
+++ /dev/null
@@ -1,9 +0,0 @@
--- CassandraStorage
-rows = LOAD 'cassandra://MyKeyspace/MyColumnFamily' USING CassandraStorage();
-cols = FOREACH rows GENERATE flatten(columns);
-colnames = FOREACH cols GENERATE $0;
-namegroups = GROUP colnames BY (chararray) $0;
-namecounts = FOREACH namegroups GENERATE COUNT($1), group;
-orderednames = ORDER namecounts BY $0 DESC;
-topnames = LIMIT orderednames 50;
-dump topnames;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/cassandra/blob/56cfc6ea/examples/pig/libdata.csv
----------------------------------------------------------------------
diff --git a/examples/pig/libdata.csv b/examples/pig/libdata.csv
deleted file mode 100644
index f9bd0b6..0000000
--- a/examples/pig/libdata.csv
+++ /dev/null
@@ -1,200 +0,0 @@
-KS,KS0189,002,KS0037,CALDWELL PUBLIC LIBRARY,120 S. MAIN ST.,CALDWELL,67022,1414,SUMNER,6208456879,CE,NO,4500,R_11,0,R_11,1200,R_11,50,R_11,2012,00,00,00,-97.606964,37.03116,20,191,09900,23835,43,9623,3078,2004,0,1,48620,0
-CA,CA0152,004,M748,SOLIZ LIBRARY,2820 JOURDAN ST.,OXNARD,93036,1611,VENTURA,8054854515,BR,CC,3030,R_11,0,R_11,1227,R_11,52,R_11,2012,00,00,00,-119.168424,34.2372,06,111,22370,831126,21,50.02999878,2005,0626,0,1,37100,0
-WV,WV0023,002,22,MORGANTOWN PUBLIC LIBRARY,373 SPRUCE STREET,MORGANTOWN,26505,5564,MONONGALIA,3042917425,CE,NO,24140,R_11,0,R_11,3016,R_11,52,R_11,2012,00,00,00,-79.954276,39.630204,54,061,55756,98613,13,107,1011,5401,0,1,34060,0
-IL,IL0603,002,3067300,VESPASIAN WARNER PUBLIC LIBRARY DISTRICT,310 NORTH QUINCY STREET,CLINTON,61727,1300,DE WITT,2179355174,CE,NO,36000,R_11,0,R_11,3172,R_11,-1,U_11,2012,00,00,00,-88.961745,40.154608,17,039,15001,16528,32,9717,2032,1713,0,1,14010,0
-NC,NC0032,017,C-FORSYTH-Y,YWCA BEST CHOICE CENTER,1031 HIGHLAND AVE,WINSTON-SALEM,27101,3108,FORSYTH,3367220597,BR,CC,750,R_11,0,R_11,312,R_11,52,R_11,2012,00,00,00,-80.233147,36.108588,37,067,75000,354454,12,6,2018,3712,0,1,49180,0
-NY,NY0578,002,7200446540,TUXEDO PARK LIBRARY,227 ROUTE 17,TUXEDO PARK,10987,4405,ORANGE,8453512207,CE,NC,9000,R_11,0,R_11,3094,R_11,52,R_11,2012,00,00,00,-74.18461,41.193192,36,071,00000,374438,41,149,3005,3618,0,5,35620,0
-IL,IL0508,002,3063300,STEELEVILLE AREA PUBLIC LIBRARY DISTRICT,625 SOUTH SPARTA STREET,STEELEVILLE,62288,2147,RANDOLPH,6189659732,CE,NO,4998,R_11,0,R_11,2630,R_11,-1,U_11,2012,00,00,00,-89.658669,38.003021,17,157,72468,33262,32,9511,4060,1712,0,1,0,.
-NY,NY0601,002,7200592060,SUNSHINE HALL FREE LIBRARY,14 PROCTOR ROAD,ELDRED,12732,0157,SULLIVAN,8455576258,CE,NO,2300,R_11,0,R_11,1250,R_11,52,R_11,2012,00,00,00,-74.882528,41.526061,36,105,00000,77079,42,9524,1021,3619,0,1,0,.
-FL,FL0001,012,FL0001-012,REDDICK PUBLIC LIBRARY,15150 N.W. GAINESVILLE ROAD,REDDICK,32686,3221,MARION,3524382566,BR,NC,2496,R_11,0,R_11,1228,R_11,52,R_11,2012,00,00,00,-82.197543,29.369123,12,083,59675,332472,42,2,2056,1205,0,1,36100,0
-NC,NC0010,009,R-HYCONEECHEE-CC,CARRBORO CYBRARY,100 N GREENSBORO ST,CARRBORO,27510,2016,ORANGE,9199187387,BR,NC,1060,R_11,0,R_11,1560,R_11,52,R_11,2012,00,00,15,-79.072256,35.910375,37,135,10620,135418,21,107.0500031,3021,3704,0,1,20500,0
-WV,WV0077,004,076B,COAL RIVER,494 JOHN SLACK CIRCLE,RACINE,25165,M,BOONE,3048378437,BR,NO,1200,R_11,0,R_11,1820,R_11,52,R_11,2012,00,00,00,-81.656444,38.145905,54,005,00000,24459,41,9582,3074,5403,X,14,16620,0
-TX,TX0279,002,360,SHEPHERD PUBLIC LIBRARY,30 N LIBERTY ST,SHEPHERD,77371,2460,SAN JACINTO,9366283515,CE,NC,2668,R_11,0,R_11,1626,R_11,52,R_11,2012,00,00,00,-94.997618,30.498398,48,407,67424,26876,42,2001.01001,1011,4808,0,1,0,.
-MI,MI0261,002,MI0261-002,SHIAWASSEE DISTRICT LIBRARY,502 WEST MAIN STREET,OWOSSO,48867,2687,SHIAWASSEE,9897255134,CE,NO,8400,R_11,0,R_11,2868,R_11,52,R_11,2012,00,00,00,-84.177198,42.997558,26,155,61940,69934,32,306,1007,2604,0,1,37020,1
-NE,NE0210,002,235T,SPENCER TOWNSHIP LIBRARY,110 WEST MAIN STREET,SPENCER,68777,0189,BOYD,4025891331,CE,NO,1750,R_11,0,R_11,546,R_11,52,R_11,2012,00,00,00,-98.701432,42.874026,31,015,46275,2081,43,9758,1234,3103,0,1,0,.
-ND,ND0012,002,ND0012-002,CARNEGIE REGIONAL LIBRARY,49 WEST 7TH STREET,GRAFTON,58237,1050,WALSH,7013522754,CE,CC,8692,R_11,0,R_11,2000,R_11,50,R_11,2012,00,00,00,-97.41183,48.417585,38,099,31820,11032,33,9580,1077,3800,0,1,0,.
-CT,CT0193,002,3900,EASTFORD PUBLIC LIBRARY,179 EASTFORD RD.,EASTFORD,06242,0908,WINDHAM,8609740125,CE,NO,1512,R_11,0,R_11,1216,R_11,52,R_11,2012,00,00,00,-72.080012,41.901001,09,015,00000,118205,42,9022,1049,0902,0,1,49340,0
-IL,IL0421,002,3053900,PECATONICA PUBLIC LIBRARY DISTRICT,400 WEST ELEVENTH STREET,PECATONICA,61063,9173,WINNEBAGO,8152392616,CE,NC,11800,R_11,0,R_11,2080,R_11,-1,U_11,2012,00,00,00,-89.365175,42.304798,17,201,58408,293660,42,43,2010,1716,0,1,40420,0
-AR,AR0019,002,AR019-002,LYDA MILLER PUBLIC LIBRARY,2609 HIGHWAY 367 S,BALD KNOB,72010,0287,WHITE,5017245452,BR,NO,2100,R_11,0,R_11,-1,U_11,52,R_11,2012,00,06,00,-91.563371,35.315974,05,145,03280,78089,32,706,4050,0502,0,1,42620,1
-TX,TX0473,002,513,ELGIN PUBLIC LIBRARY,404 N MAIN ST,ELGIN,78621,2625,BASTROP,5122815678,CE,NC,12738,R_11,0,R_11,1983,R_11,52,R_11,2012,00,00,00,-97.371621,30.350555,48,021,23044,75028,32,9502,2001,4810,0,1,12420,0
-MA,MA0164,002,MALDEN,MALDEN PUBLIC LIBRARY,36 SALEM ST.,MALDEN,02148,5208,MIDDLESEX,7813240218,CE,NC,44000,R_11,0,R_11,3126,R_11,52,R_11,2012,00,00,00,-71.066333,42.427158,25,017,37875,1522050,21,3416,5007,2505,0,1,14460,0
-WI,WI0409,002,WI0409,LOWELL PUBLIC LIBRARY,105 N. RIVER ST.,LOWELL,53557,0015,DODGE,9209275700,CE,NO,900,R_11,0,R_11,1040,R_11,52,R_11,2012,00,00,00,-88.79318,43.345758,55,027,00000,88721,42,9618,1101,5505,X,15,13180,1
-PA,PA0413,002,901630213,BURGETTSTOWN COMMUNITY LIBRARY,2 KERR ST,BURGETTSTOWN,15021,1127,WASHINGTON,7249479780,CE,NC,6781,R_11,0,R_11,2392,R_11,52,R_11,2012,00,00,00,-80.389683,40.379256,42,125,10224,208170,42,7127,1030,4218,0,1,38300,0
-NE,NE0091,002,101C,GILBERT PUBLIC LIBRARY,628 SECOND STREET,FRIEND,68359,1308,SALINE,4029475081,CE,NO,1678,R_11,0,R_11,1196,R_11,52,R_11,2012,00,00,00,-97.285879,40.653724,31,151,17775,14398,43,9607,2058,3103,0,1,0,.
-KS,KS0028,002,KS0160,KANOPOLIS PUBLIC LIBRARY,221 N. KANSAS,KANOPOLIS,67454,0205,ELLSWORTH,7854723053,CE,NO,897,R_11,0,R_11,832,R_11,52,R_11,2012,00,00,00,-98.157901,38.711743,20,053,35950,6478,42,866,2294,2001,0,1,0,.
-IL,IL0098,059,3010857,PORTAGE-CRAGIN BRANCH,5108 WEST BELMONT AVENUE,CHICAGO,60641,4206,COOK,3127440152,BR,CC,14000,R_11,0,R_11,2496,R_11,52,R_11,2012,00,00,00,-87.754563,41.938765,17,031,14000,5214098,11,1510.01001,2008,1704,0,1,16980,0
-PA,PA0083,002,905620545,SHEFFIELD TOWNSHIP LIBRARY,20 LEATHER ST,SHEFFIELD,16347,0607,WARREN,8149683439,CE,NO,1131,R_11,0,R_11,1352,R_11,52,R_11,2012,00,00,00,-79.033263,41.701956,42,123,69984,41480,42,9712,3019,4205,0,1,47620,1
-UT,UT0072,001,UT0072-001,SALEM CITY LIBRARY,59 SOUTH MAIN STREET,SALEM,84653,9601,UTAH,8014232622,CE,NC,4150,R_11,0,R_11,2216,R_11,52,R_11,2012,00,00,00,-111.673401,40.053702,49,049,65770,530104,32,104.0699997,2016,4903,0,1,39340,0
-NJ,NJ0048,002,NJ0048-002,ORADELL PUBLIC LIBRARY,375 KINDERKAMACK ROAD,ORADELL,07649,2122,BERGEN,2012622613,CE,NC,16000,R_11,0,R_11,3016,R_11,52,R_11,2012,00,00,00,-74.031827,40.953183,34,003,54990,912773,21,400.019989,4006,3405,0,1,35620,0
-WI,WI0191,002,WI0191,MENOMONEE FALLS PUBLIC LIBRARY,W156N8436 PILGRIM RD.,MENOMONEE FALLS,53051,3140,WAUKESHA,2625328930,CE,NC,52951,R_11,0,R_11,3328,R_11,52,R_11,2012,00,00,00,-88.103683,43.170852,55,133,51000,390701,21,2001.030029,3023,5505,0,1,33340,0
-NC,NC0033,002,C-FRANKLIN-F,FRANKLIN COUNTY LIBRARY,906 N MAIN ST,LOUISBURG,27549,2199,FRANKLIN,9194962111,CE,NC,6000,R_11,0,R_11,2704,R_11,52,R_11,2012,00,00,15,-78.296077,36.112774,37,069,39360,61181,32,603.0100098,1018,3701,0,1,39580,0
-FL,FL0042,006,FL0042-006,DUNBAR JUPITER HAMMON PUBLIC LIBRARY,3095 BLOUNT STREET,FORT MYERS,33916,4100,LEE,2395334150,BR,NC,11400,R_11,0,R_11,1992,R_11,52,R_11,2012,00,00,00,-81.84986,26.642837,12,071,24125,631602,13,5.019999981,2011,1219,0,1,15980,0
-LA,LA0038,005,LA0038,VERNON PARISH LIBRARY BOOKMOBILE,1401 NOLAN TRACE,LEESVILLE,71446,8378,VERNON,3372392027,BS,NO,-3,U_11,1,R_11,1248,R_11,52,R_11,2012,00,00,00,-93.133117,31.328122,22,115,00000,52178,43,9501,1052,2204,2,15,22860,1
-FL,FL0058,002,FL0058-002,NORTH PALM BEACH PUBLIC LIBRARY,303 ANCHORAGE DRIVE,NORTH PALM BEACH,33408,4990,PALM BEACH,5618413383,CE,NC,24893,R_11,0,R_11,3024,R_11,52,R_11,2012,00,00,00,-80.057184,26.810361,12,099,49600,1337512,21,7.019999981,1015,1218,0,1,33100,0
-MO,MO0037,002,MO0036-002,UNIVERSITY CITY PUBLIC LIBRARY,6701 DELMAR BLVD,UNIVERSITY CITY,63130,3199,ST. LOUIS,3147273150,CE,NC,43000,R_11,0,R_11,3744,R_11,52,R_11,2012,00,00,00,-90.308719,38.656351,29,189,75220,999321,21,2161,2010,2901,0,1,41180,0
-TX,TX0582,002,575,MCMULLEN MEMORIAL LIBRARY,900 N MAIN ST,HUNTINGTON,75949,M,ANGELINA,9368764516,CE,NO,2500,R_11,0,R_11,1792,R_11,52,R_11,2012,00,00,00,-94.572496,31.285469,48,005,35492,87276,42,13,1016,4801,0,1,31260,1
-CT,CT0109,002,9902,ATWATER MEMORIAL LIBRARY,1720 FOXON RD.,NORTH BRANFORD,06471,0258,NEW HAVEN,2033156020,CE,NC,13000,R_11,0,R_11,2830,R_11,52,R_11,2012,00,00,00,-72.765106,41.328739,09,009,00000,862812,21,1861,1010,0903,0,1,35300,0
-AZ,AZ0064,011,BR-GEASA,GEASA-MARANA BRANCH LIBRARY,13370 LON ADAMS ROAD,MARANA,85653,9050,PIMA,5205945255,BR,NC,2900,R_11,0,R_11,2034,R_11,52,R_11,2012,00,00,00,-111.208324,32.449152,04,019,44270,987573,42,44.29999924,2025,0401,0,1,46060,0
-TX,TX0216,005,30.53,THEODORE JOHNS BRANCH LIBRARY,4255 FANNETT RD,BEAUMONT,77705,M,JEFFERSON,4098425223,BR,CC,11970,R_11,0,R_11,3132,R_11,52,R_11,2012,00,00,00,-94.129488,30.045033,48,245,07000,253160,12,22,2023,4814,0,1,13140,0
-LA,LA0044,012,49,GENEALOGY BRANCH & ADM. OFFICE,200 EAST MULBERRY STREET,AMITE,70422,2524,TANGIPAHOA,9857487559,BR,NO,8000,R_11,0,R_11,1716,R_11,52,R_11,2012,00,00,00,-90.506645,30.728051,22,105,01885,122519,32,9534,3091,2205,0,1,25220,0
-MD,MD0024,005,MD0024-005,POCOMOKE BRANCH,301 MARKET ST.,POCOMOKE CITY,21851,1113,WORCESTER,4109570878,BR,NO,6728,R_11,0,R_11,2136,R_11,52,R_11,2012,00,00,00,-75.566713,38.074163,24,047,62475,51446,32,9515,3032,2401,0,1,41540,0
-IL,IL0318,003,3043000,MARION CARNEGIE LIBRARY,206 SOUTH MARKET STREET,MARION,62959,2516,WILLIAMSON,6189935935,CE,NO,21000,R_11,0,R_11,3129,R_11,-1,U_11,2012,00,00,00,-88.927162,37.729836,17,199,46916,66680,33,211,3029,1712,0,1,16060,0
-IA,IA0307,002,RD,FONTANELLE PUBLIC LIBRARY,303 WASHINGTON,FONTANELLE,50846,0387,ADAIR,6417454981,CE,NO,2000,R_11,0,R_11,1040,R_11,52,R_11,2012,00,00,00,-94.563977,41.289591,19,001,28290,7572,43,9603,4125,1903,0,1,0,.
-PR,PR0010,001,BP0009,MOROVIS ELECTRONIC MUNICIPAL LIBRARY,"JOSE DEL RIO ST., CORNER OF BETANCE",MOROVIS,00687,M,MOROVIS,7878626161,CE,NO,-1,U_11,0,R_11,-1,U_11,-1,U_11,2012,00,00,00,-66.427046,18.340777,72,101,29555,32753,21,9553,3018,7298,X,14,41980,0
-OH,OH0188,002,288C,IDA RUPP PUBLIC LIBRARY,310 MADISON ST.,PORT CLINTON,43452,1921,OTTAWA,4197323212,CE,NC,17075,R_11,0,R_11,3119,R_11,52,R_11,2012,00,00,00,-82.941131,41.510222,39,123,64150,41435,31,506,2000,3909,0,1,38840,1
-MD,MD0008,004,MD0008-004,NORTH EAST BRANCH,106 W. CECIL AVE.,NORTH EAST,21901,3506,CECIL,4109966269,BR,NO,2800,R_11,0,R_11,2600,R_11,52,R_11,2012,00,00,00,-75.945126,39.60096,24,015,56450,101628,31,309.0299988,1067,2401,0,1,37980,0
-LA,LA0055,007,59,DELMONT GARDENS BRANCH,3351 LORRAINE STREET,BATON ROUGE,70805,2724,EAST BATON ROUGE,2253547040,BR,CC,19267,R_11,0,R_11,3432,R_11,52,R_11,2012,00,00,00,-91.156007,30.496636,22,033,05000,441602,12,3,1023,2202,0,1,12940,0
-VA,VA0014,005,VA0014-005,PORTER BRANCH,2001 PARKWAY BOULEVARD,STAFFORD,22554,3972,STAFFORD,5406594909,BR,NO,23000,R_11,0,R_11,-1,U_11,-1,U_11,2012,00,00,00,-77.447472,38.464856,51,179,00000,132246,21,102.1399994,2007,5101,0,1,47900,0
-WI,WI0198,002,WI0198,MILTON PUBLIC LIBRARY,"430 E. HIGH ST., #100",MILTON,53563,1579,ROCK,6088687462,CE,NC,8000,R_11,0,R_11,2756,R_11,52,R_11,2012,00,00,00,-88.944389,42.77298,55,105,52200,160067,23,31,4016,5501,0,1,27500,0
-IA,IA0544,002,IA0563,WASHTA PUBLIC LIBRARY,100 S 5TH AVE,WASHTA,51061,1016,CHEROKEE,7124476546,CE,NO,1353,R_11,0,R_11,1222,R_11,52,R_11,2012,00,00,15,-95.720114,42.57533,19,035,82380,12029,43,803,2316,1904,0,1,0,.
-NH,NH0174,002,NH895,GILMAN LIBRARY,100 MAIN STREET,ALTON,03809,4618,BELKNAP,6038752550,CE,NO,6996,R_11,0,R_11,1872,R_11,52,R_11,2012,00,00,00,-71.213211,43.452041,33,001,00980,60189,42,9665,2060,3301,0,1,29060,1
-CT,CT0175,002,15900,WETHERSFIELD PUBLIC LIBRARY,515 SILAS DEANE HIGHWAY,WETHERSFIELD,06109,2216,HARTFORD,8605292665,CE,NC,32877,R_11,0,R_11,2968,R_11,52,R_11,2012,00,00,00,-72.663363,41.711653,09,003,84970,896248,21,4922,1007,0901,0,1,25540,0
-NH,NH0030,002,NH89126,MAXFIELD PUBLIC LIBRARY,8 ROUTE 129,LOUDON,03307,0814,MERRIMACK,6037985153,CE,NO,6800,R_11,0,R_11,1768,R_11,52,R_11,2012,00,00,00,-71.467819,43.286141,33,013,43300,146614,42,360,2046,3302,0,1,18180,1
-NY,NY0005,015,0800000000,NORTH PARK BRANCH LIBRARY,975 HERTEL AVENUE,BUFFALO,14216,2201,ERIE,7168753748,BR,CC,3967,R_11,0,R_11,1568,R_11,52,R_11,2012,00,00,00,-78.871968,42.94775,36,029,11000,919627,11,50,1005,3626,0,1,15380,0
-OK,OK0018,002,24,CHANDLER PUBLIC LIBRARY,1021 S MANVEL,CHANDLER,74834,3853,LINCOLN,4052583204,CE,NO,2500,R_11,0,R_11,2496,R_11,52,R_11,2012,00,00,00,-96.880778,35.700289,40,081,13500,34293,32,9617,4057,4003,0,1,36420,0
-ND,ND0037,002,ND0037-002,HANKINSON PUBLIC LIBRARY,315 MAIN AVENUE S,HANKINSON,58041,0244,RICHLAND,7012427929,CE,NO,400,R_11,0,R_11,1198,R_11,51,R_11,2012,00,00,00,-96.902081,46.069547,38,077,34900,16261,43,9714,2037,3800,0,1,47420,1
-IN,IN0207,009,8412,TRAFALGAR BRANCH,424 TOWER STREET,TRAFALGAR,46181,8876,JOHNSON,3178789560,BR,NO,19000,R_11,0,R_11,3172,R_11,52,R_11,2012,00,00,00,-86.148687,39.413997,18,081,76310,141439,41,6114,2051,1809,0,1,26900,0
-KY,KY0071,003,MARION,MARION COUNTY PUBLIC LIBRARY,201 EAST MAIN STREET,LEBANON,40033,1133,MARION,2706924698,BS,NO,-3,U_11,1,R_11,1012,R_11,44,R_11,2012,00,00,00,-85.250194,37.570737,21,155,44344,20049,32,9702,2017,2101,0,1,0,.
-NY,NY0562,033,6800310000,LEFRAK CITY,98-30 57 AVENUE,CORONA,11368,4696,QUEENS,7185927677,BR,CC,18000,R_11,0,R_11,1820,R_11,52,R_11,2012,00,00,00,-73.861226,40.738002,36,081,51000,2256438,11,455,3000,3614,4,15,35620,0
-NC,NC0014,003,R-PETTIGREW-P,PERQUIMANS COUNTY LIBRARY,110 W ACADEMY ST,HERTFORD,27944,1306,PERQUIMANS,2524265319,BR,NO,5320,R_11,0,R_11,2278,R_11,52,R_11,2012,00,00,15,-76.471096,36.188915,37,143,30900,13458,42,9202.009766,3019,3701,0,1,21020,1
-IL,IL0140,002,3022800,DIVERNON TOWNSHIP LIBRARY,221 SOUTH SECOND STREET,DIVERNON,62530,0140,SANGAMON,2176283813,CE,NO,1300,R_11,0,R_11,1560,R_11,-1,U_11,2012,00,00,00,-89.65852,39.565293,17,167,20045,198949,41,33,4122,1718,0,1,44100,0
-GA,GA0010,005,6282PICKENS,PICKENS COUNTY PUBLIC LIBRARY,100 LIBRARY LANE,JASPER,30143,1364,PICKENS,7066925411,BR,NO,11000,R_11,0,R_11,3320,R_11,52,R_11,2012,00,00,00,-84.420188,34.467478,13,227,41932,29431,32,502,1040,1309,2,14,12060,0
-MN,MN0085,002,S1310,BLOOMING PRAIRIE BRANCH LIBRARY,138 HIGHWAY AVE SOUTH,BLOOMING PRAIRIE,55917,0187,STEELE,5075837750,BR,NO,3000,R_11,0,R_11,2213,R_11,52,R_11,2012,00,00,00,-93.049227,43.866202,27,147,06580,36541,43,9608,4004,2701,0,5,36940,1
-MD,MD0003,039,MD0003,SOUTHEAST ANCHOR BRANCH,3601 EASTERN AVENUE,BALTIMORE,21224,M,BALTIMORE CITY,4103961580,BR,CC,27000,R_11,0,R_11,2527,R_11,52,R_11,2012,00,00,00,-76.567069,39.286587,24,510,04000,620216,11,2609,1010,2403,0,1,12580,0
-TX,TX0012,002,109,DENTON PUBLIC LIBRARY,502 OAKLAND ST,DENTON,76201,3102,DENTON,9403498750,CE,NC,22876,R_11,0,R_11,3161,R_11,52,R_11,2012,00,00,00,-97.129918,33.219673,48,121,19972,685601,12,206.0099945,3025,4826,0,1,19100,0
-MT,MT0044,002,747,LAUREL PUBLIC LIBRARY,720 WEST 3RD ST,LAUREL,59044,0068,YELLOWSTONE,4066284961,CE,NO,6000,R_11,0,R_11,2338,R_11,51,R_11,2012,00,00,00,-108.780888,45.671408,30,111,42700,149907,31,19.01000023,2007,3000,0,1,13740,0
-NJ,NJ0194,008,1225F,ISELIN BRANCH,1081 GREEN ST.,ISELIN,08830,2171,MIDDLESEX,7327267073,BR,NC,9000,R_11,0,R_11,-1,U_11,52,R_11,2012,00,00,00,-74.313243,40.571154,34,023,34470,816618,21,26.03000069,2000,3406,0,1,35620,0
-FL,FL0005,003,FL0005-003,ALAFAYA BRANCH,12000 EAST COLONIAL DRIVE,ORLANDO,32826,4705,ORANGE,4078357323,BR,CC,12000,R_11,0,R_11,3150,R_11,52,R_11,2012,00,00,00,-81.203356,28.566308,12,095,00410,1171018,21,167.2700043,1000,1209,0,1,36740,0
-LA,LA0002,003,10,MONTGOMERY BRANCH,940 CADDO STREET,MONTGOMERY,71454,0157,GRANT,3186463660,BR,NO,770,R_11,0,R_11,1456,R_11,52,R_11,2012,00,00,00,-92.889812,31.667523,22,043,51620,22115,42,202,1127,2205,0,1,10780,0
-IA,IA0400,002,VT,CRESCO PUBLIC LIBRARY,320 N ELM ST,CRESCO,52136,1452,HOWARD,5635472540,CE,NO,7800,R_11,0,R_11,3120,R_11,52,R_11,2012,00,00,15,-92.11626,43.374078,19,089,17220,9533,33,9601,4021,1901,0,1,0,.
-IL,IL0031,002,3003100,BARCLAY PUBLIC LIBRARY DISTRICT,220 SOUTH MAIN STREET,WARRENSBURG,62573,0349,MACON,2176723621,CE,NO,5000,R_11,0,R_11,2600,R_11,-1,U_11,2012,00,00,00,-89.062322,39.929914,17,115,78916,110616,41,28,1137,1713,0,1,19500,0
-CT,CT0022,002,2100,DAVID M. HUNT LIBRARY,63 MAIN ST.,CANAAN,06031,0217,LITCHFIELD,8608247424,CE,NO,3600,R_11,0,R_11,1456,R_11,52,R_11,2012,00,00,00,-73.328793,42.026425,09,005,10870,188923,42,2602,1020,0905,0,1,45860,1
-CA,CA0119,002,M715,CROWELL PUBLIC LIBRARY,1890 HUNTINGTON DR.,SAN MARINO,91108,2595,LOS ANGELES,6263000777,CE,NC,29000,R_11,0,R_11,3224,R_11,52,R_11,2012,00,00,00,-118.112501,34.119348,06,037,68224,9889025,21,4641,4005,0627,0,1,31080,0
-MN,MN0043,009,M0715,NEW BRIGHTON LIBRARY,400 10TH STREET NW,NEW BRIGHTON,55112,6806,RAMSEY,6517246002,BR,NC,6250,R_11,0,R_11,294,R_11,9,R_11,2012,02,00,00,-93.191665,45.065114,27,123,45430,515336,21,412,1003,2704,0,1,33460,0
-GA,GA0008,018,GA0008,TYBEE,403 BUTLER AVENUE,TYBEE ISLAND,31328,9719,CHATHAM,9127867733,BR,NC,4000,R_11,0,R_11,1528,R_11,52,R_11,2012,00,00,00,-80.842618,32.008883,13,051,78036,271896,31,111.0299988,2022,1301,0,1,42340,0
-NH,NH0042,002,NH89137,WADLEIGH MEMORIAL LIBRARY,49 NASHUA STREET,MILFORD,03055,3753,HILLSBOROUGH,6036732408,CE,NO,13500,R_11,0,R_11,2923,R_11,52,R_11,2012,00,00,00,-71.648234,42.835511,33,011,47940,401842,22,162.0099945,5021,3302,0,1,31700,0
-NY,NY0069,002,7800020610,FREE LIBRARY OF THE BELMONT LITERARY AND HIST,2 WILLETS AVENUE,BELMONT,14813,1025,ALLEGANY,5852685308,CE,NO,8000,R_11,0,R_11,1000,R_11,50,R_11,2012,00,00,00,-78.034633,42.223293,36,003,05815,48744,42,9506,3060,3623,0,1,0,.
-IL,IL0243,002,3034900,HIGHWOOD PUBLIC LIBRARY,102 HIGHWOOD AVENUE,HIGHWOOD,60040,1520,LAKE,8474325404,CE,NC,10363,R_11,0,R_11,2128,R_11,-1,U_11,2012,00,00,00,-87.810707,42.201979,17,097,34865,701575,21,8652,1007,1710,0,1,16980,0
-VA,VA0005,008,VA0005-008,WESTOVER BRANCH LIBRARY,1644 N.MCKINLEY STREET,ARLINGTON,22205,2855,ARLINGTON,7032285260,BR,NC,16403,R_11,0,R_11,-1,U_11,-1,U_11,2012,00,00,00,-77.140244,38.888422,51,013,03000,216118,12,1010,4003,5108,4,15,47900,0
-AZ,AZ0111,002,CE-COPPER,COPPER QUEEN LIBRARY,6 MAIN STREET,BISBEE,85603,9901,COCHISE,5204324232,CE,NO,9000,R_11,0,R_11,2236,R_11,52,R_11,2012,00,00,00,-109.915161,31.441616,04,003,06260,132770,33,10,2015,0402,0,1,43420,0
-SD,SD0042,002,SD0042-002,DOROTHEE PIKE MEMORIAL LIBRARY,225 N MAIN AVE,LAKE PRESTON,57249,0036,KINGSBURY,6058474843,CE,NO,392,R_11,0,R_11,988,R_11,52,R_11,2012,00,00,15,-97.377319,44.362653,46,077,35500,5152,43,9581,1191,4600,0,1,0,.
-WI,WI0322,002,WI0322,TOMAHAWK PUBLIC LIBRARY,300 W. LINCOLN AVE.,TOMAHAWK,54487,1202,LINCOLN,7154532455,CE,NO,11465,R_11,0,R_11,2596,R_11,52,R_11,2012,00,00,00,-89.733724,45.473925,55,069,80125,28492,32,9602,2040,5507,0,1,32980,1
-NY,NY0146,002,2400113890,PECK MEMORIAL LIBRARY,24 MAIN STREET,MARATHON,13803,0325,CORTLAND,6078496135,CE,NO,3052,R_11,0,R_11,1490,R_11,52,R_11,2012,00,00,00,-76.033345,42.441233,36,023,45392,49619,42,9711,3064,3622,0,1,18660,1
-DC,DC0001,017,SOW,SOUTHWEST BRANCH LIBRARY,"900 WESLEY PLACE, SW",WASHINGTON,20024,4212,DIST OF COLUMBIA,2027244752,BR,CC,22032,R_11,0,R_11,2496,R_11,52,R_11,2012,00,00,00,-77.016219,38.879185,11,001,50000,619020,11,102,2044,1198,0,1,47900,0
-OR,OR0140,001,OR0140,LANE LIBRARY DISTRICT,64 W. OREGON AVE.,CRESWELL,97426,0366,LANE,5038953053,CE,NC,4950,R_11,0,R_11,2300,R_11,52,R_11,2012,00,00,00,-123.019529,43.917861,41,039,16950,353481,31,11.01000023,3086,4104,0,1,21660,0
-IL,IL0445,004,3056601,YORK STREET BRANCH LIBRARY,639 YORK STREET,QUINCY,62301,3918,ADAMS,2172231309,BR,NO,2775,R_11,0,R_11,2020,R_11,52,R_11,2012,00,00,00,-91.40595,39.929353,17,001,62367,67169,33,7,2054,1718,0,1,39500,1
-OH,OH0226,003,326B01,BROOKFIELD BRANCH LIBRARY,7032 GROVE ST,BROOKFIELD,44403,0052,TRUMBULL,3304488134,BR,NC,4200,R_11,0,R_11,2652,R_11,52,R_11,2012,00,00,00,-80.565232,41.232504,39,155,09200,208874,21,9314,4006,3913,0,1,49660,0
-WI,WI0052,002,WI0052,CAMPBELLSPORT PUBLIC LIBRARY,220 N. HELENA,CAMPBELLSPORT,53010,0405,FOND DU LAC,9205338534,CE,NC,10440,R_11,0,R_11,2020,R_11,52,R_11,2012,00,00,00,-88.276351,43.599297,55,039,12325,101856,42,422,3028,5506,0,1,22540,0
-AR,AR0001,017,AR0001-13,OLEY E. ROOKER LIBRARY,11 OTTER CREEK CT.,LITTLE ROCK,72210,M,PULASKI,5019075991,BR,NC,13550,R_11,0,R_11,-1,U_11,52,R_11,2012,00,00,00,-92.427498,34.659253,05,119,41000,386822,41,42.20000076,3009,0502,0,1,30780,0
-GA,GA0022,042,GA0022,NORTHEAST-SPRUILL OAKS REGIONAL LIBRARY,9560 SPRUILL ROAD,ALPHARETTA,30022,5550,FULTON,7703608820,BR,NC,25000,R_11,0,R_11,2882,R_11,52,R_11,2012,00,00,00,-84.22258,34.01515,13,121,42425,949580,21,114.25,2005,1306,0,1,12060,0
-ME,ME0247,002,93,NORTH GORHAM PUBLIC LIBRARY,2 STANDISH NECK ROAD,GORHAM,04038,2469,CUMBERLAND,2078922575,CE,NO,1200,R_11,0,R_11,750,R_11,52,R_11,2012,00,00,00,-70.45554,43.798963,23,005,00000,282669,32,41,2006,2301,0,1,38860,0
-ME,ME0079,002,83,FORT KENT PUBLIC LIBRARY,1 MONUMENT SQUARE,FORT KENT,04743,1216,AROOSTOOK,2078343048,CE,NO,3153,R_11,0,R_11,1579,R_11,52,R_11,2012,00,00,00,-68.595746,47.250002,23,003,25720,71388,43,9506,3020,2302,4,14,0,.
-MI,MI0336,002,MI0336-002,THOMAS TOWNSHIP LIBRARY,8207 SHIELDS DRIVE,SAGINAW,48609,4814,SAGINAW,9897813770,CE,CC,14360,R_11,0,R_11,2732,R_11,52,R_11,2012,00,00,00,-84.074739,43.41882,26,145,73560,198990,22,120.0299988,1011,2604,0,1,40980,0
-TN,TN0133,006,KN001-05,FARRAGUT BRANCH LIBRARY,417 N. CAMPBELL STATION ROAD,KNOXVILLE,37934,2708,KNOX,8657771750,BR,CC,10100,R_11,0,R_11,2860,R_11,52,R_11,2012,00,00,00,-84.168446,35.88913,47,093,25760,436877,21,58.09000015,2020,4702,0,1,28940,0
-TX,TX0160,002,245,RED WALLER COMMUNITY LIBRARY,109 MELTON ST,MALAKOFF,75148,9347,HENDERSON,9034891818,CE,NO,4800,R_11,0,R_11,1500,R_11,50,R_11,2012,00,00,00,-96.014132,32.170392,48,213,46224,78753,33,9510,2059,4805,0,1,11980,1
-LA,LA0058,008,LA0058,MILTON H. LATTER MEMORIAL BRANCH,5120 ST. CHARLES AVENUE,NEW ORLEANS,70115,4941,ORLEANS,5045962625,BR,CC,9000,R_11,0,R_11,2704,R_11,52,R_11,2012,00,00,00,-90.109987,29.926997,22,071,55000,360341,11,108,1015,2202,0,1,35380,0
-WV,WV0020,005,019C,BOOKMOBILE,3100 EMERSON AVE.,PARKERSBURG,26104,2414,WOOD,3044204587,BS,NC,-3,U_11,1,R_11,1040,R_11,52,R_11,2012,00,00,00,-81.53833,39.28578,54,107,62140,86837,13,3,2003,5401,0,1,37620,0
-CA,CA0019,004,M615,CARNEGIE TECHNOLOGY CENTER BRANCH,420 HEBER AVE.,CALEXICO,92231,2559,IMPERIAL,7603575525,BR,NC,4000,R_11,0,R_11,708,R_11,52,R_11,2012,00,00,00,-115.493508,32.669062,06,025,09710,175897,31,121,3011,0651,0,1,20940,0
-PA,PA0385,026,926510006,LAWNCREST BRANCH LIBRARY,6098 RISING SUN AVENUE,PHILADELPHIA,19111,6009,PHILADELPHIA,2156850549,BR,CC,7184,R_11,0,R_11,-1,U_11,-1,U_11,2012,00,00,00,-75.099695,40.045973,42,101,60000,1538567,11,305.019989,1004,4213,0,1,37980,0
-NY,NY0306,002,3600414840,C. W. CLARK MEMORIAL LIBRARY,160 NORTH MAIN STREET,ORISKANY FALLS,13425,0670,ONEIDA,3158217850,CE,NC,3141,R_11,0,R_11,2060,R_11,52,R_11,2012,00,00,00,-75.459544,42.940144,36,065,55376,234155,42,256,2068,3622,0,1,46540,0
-NC,NC0061,005,C-UNION-W,WAXHAW LIBRARY,509 S PROVIDENCE ST,WAXHAW,28173,9339,UNION,7048433131,BR,NC,2900,R_11,0,R_11,1489,R_11,52,R_11,2012,00,00,15,-80.741342,34.922191,37,179,71460,205193,41,210.1399994,1009,3709,0,1,16740,0
-IL,IL0194,002,3029200,FRANKLIN PARK PUBLIC LIBRARY DISTRICT,10311 GRAND AVENUE,FRANKLIN PARK,60131,2225,COOK,8474556016,CE,NC,25000,R_11,0,R_11,3424,R_11,-1,U_11,2012,00,00,00,-87.882502,41.930234,17,031,27702,5214098,21,8114.02002,2009,1705,0,1,16980,0
-OH,OH0116,004,216B02,BALTIMORE BRANCH LIBRARY,205 E. MARKET ST.,BALTIMORE,43105,1326,FAIRFIELD,7408628505,BR,NC,3866,R_11,0,R_11,1632,R_11,51,R_11,2012,00,00,00,-82.600057,39.84531,39,045,03758,147355,31,303,3030,3915,0,1,18140,0
-MI,MI0360,002,MI0360-002,WATERVLIET DISTRICT LIBRARY,333 NORTH MAIN STREET,WATERVLIET,49098,9750,BERRIEN,2694636382,CE,NC,10272,R_11,0,R_11,2340,R_11,52,R_11,2012,00,00,00,-86.259586,42.19054,26,021,84500,156489,31,103,4027,2606,0,1,35660,0
-AL,AL0108,019,AL0108-019,BIRMINGHAM PUBLIC - TITUSVILLE BRANCH,26TH AVENUE SW,BIRMINGHAM,35211,2909,JEFFERSON,2053221140,BR,CC,9000,R_11,0,R_11,2385,R_11,52,R_11,2012,00,00,00,-86.833206,33.491538,01,073,07000,658967,12,51.00999832,3026,0107,4,15,13820,0
-TX,TX0443,005,86.66,SOUTH REGIONAL LIBRARY,2101 LAKE ROBBINS DR,THE WOODLANDS,77380,M,MONTGOMERY,9364427727,BR,NC,30000,R_11,0,R_11,3141,R_11,52,R_11,2012,00,00,00,-95.465652,30.162471,48,339,72656,471704,23,6917,1033,4808,0,1,26420,0
-FL,FL0012,042,FL0012-042,TAMARAC BRANCH,8701 W. COMMERCIAL BLVD.,TAMARAC,33351,4310,BROWARD,9547202282,BR,NC,30000,R_11,0,R_11,2672,R_11,52,R_11,2012,00,00,00,-80.267013,26.193663,12,011,39550,1784956,21,601.1400146,1004,1220,0,1,33100,0
-PA,PA0007,002,901300753,EVA K BOWLBY PUBLIC LIBRARY,311 NORTH WEST ST,WAYNESBURG,15370,1238,GREENE,7246279776,CE,NO,9791,R_11,0,R_11,2401,R_11,52,R_11,2012,00,00,00,-80.191465,39.899297,42,059,81832,38424,32,9706,3034,4218,0,1,0,.
-IN,IN0068,012,2713,CENTRE TOWNSHIP BRANCH,1150 EAST KERN ROAD,SOUTH BEND,46614,6472,ST JOSEPH,5742513700,BR,CC,17160,R_11,0,R_11,2496,R_11,52,R_11,2012,00,00,00,-86.232966,41.607649,18,141,71000,266661,12,119,2011,1802,0,1,43780,0
-MI,MI0284,002,MI0284-002,REED CITY PUBLIC LIBRARY,410 WEST UPTON AVENUE,REED CITY,49677,1152,OSCEOLA,2318322131,CE,NO,3649,R_11,0,R_11,1872,R_11,52,R_11,2012,00,00,00,-85.516822,43.875995,26,133,67820,23455,33,9706,2012,2604,0,1,0,.
-VT,VT0042,002,CORNWALL,CORNWALL FREE PUBLIC,2629 RT 30,CORNWALL,05753,9299,ADDISON,8024623615,CE,NO,315,R_11,0,R_11,1664,R_11,52,R_11,2012,00,00,00,-73.209627,43.961155,50,001,00000,36785,41,9609,1014,5000,0,1,0,.
-TX,TX0065,002,157,LEE PUBLIC LIBRARY,312 W PACIFIC AVE,GLADEWATER,75647,2135,GREGG,9038452640,CE,NC,5500,R_11,0,R_11,2048,R_11,52,R_11,2012,00,00,00,-94.946153,32.535873,48,183,29660,122325,31,102,3008,4801,0,1,30980,0
-CT,CT0105,002,9500,PUBLIC LIBRARY OF NEW LONDON,63 HUNTINGTON ST.,NEW LONDON,06320,6194,NEW LONDON,8604471411,CE,CC,22000,R_11,0,R_11,2678,R_11,52,R_11,2012,00,00,00,-72.099541,41.354992,09,011,52280,274091,13,6905,2026,0902,0,1,35980,0
-NJ,NJ0195,009,1300G,MARLBORO LIBRARY,1 LIBRARY COURT,MARLBORO,07746,1102,MONMOUTH,7325369406,BR,NC,18100,R_11,0,R_11,-1,U_11,52,R_11,2012,00,00,00,-74.262612,40.330578,34,025,63900,630092,21,8097.009766,2021,3406,0,1,35620,0
-IL,IL0037,002,3003700,BEDFORD PARK PUBLIC LIBRARY DISTRICT,7816 WEST 65TH PLACE,BEDFORD PARK,60501,1914,COOK,7084586826,CE,NC,17000,R_11,0,R_11,3016,R_11,-1,U_11,2012,00,00,00,-87.816069,41.772696,17,031,04572,5214098,21,8205.009766,1026,1703,0,1,16980,0
-KS,KS0191,002,KS0059,CONWAY SPRINGS CITY LIBRARY,210 W. SPRINGS AVE,CONWAY SPRINGS,67031,0183,SUMNER,6204562859,CE,NO,980,R_11,0,R_11,1040,R_11,52,R_11,2012,00,00,00,-97.63682,37.390358,20,191,15325,23835,42,9622,3026,2004,0,1,48620,0
-NH,NH0055,002,NH89149,NEWFIELDS PUBLIC LIBRARY,76 MAIN STREET,NEWFIELDS,03856,0200,ROCKINGHAM,6037788169,CE,NO,-1,U_11,0,R_11,1456,R_11,52,R_11,2012,00,00,00,-70.932892,43.041363,33,015,00000,296085,31,675.0100098,2014,3301,0,1,14460,0
-WI,WI0308,002,WI0308,JEAN M. THOMSEN MEMORIAL LIBRARY,105 N. GERSHWIN ST.,STETSONVILLE,54480,0099,TAYLOR,7156782892,CE,NO,3990,R_11,0,R_11,1772,R_11,52,R_11,2012,00,00,00,-90.311907,45.077181,55,119,77100,20759,42,9606,3046,5507,0,1,0,.
-FL,FL0011,019,FL0011-019,SUNTREE/VIERA PUBLIC LIBRARY,902 JORDAN BLASS DRIVE,MELBOURNE,32940,3350,BREVARD,3212554404,BR,NO,15000,R_11,0,R_11,2716,R_11,-1,U_11,2012,00,00,00,-80.679796,28.215339,12,009,00000,544322,21,641.2700195,4005,1208,0,1,37340,0
-NY,NY0071,002,7800021050,ESSENTIAL CLUB FREE LIBRARY,11 PRATT STREET,CANASERAGA,14822,9728,ALLEGANY,6075456443,CE,NO,2500,R_11,0,R_11,1144,R_11,52,R_11,2012,00,00,00,-77.775745,42.461616,36,003,12177,48744,42,9501,1037,3623,0,1,0,.
-WI,WI0134,002,WI0134,HILLSBORO PUBLIC LIBRARY,819 HIGH AVE.,HILLSBORO,54634,0468,VERNON,6084892192,CE,NO,6800,R_11,0,R_11,1716,R_11,52,R_11,2012,00,00,00,-90.341447,43.652202,55,123,34825,30043,43,9601,2032,5503,0,1,0,.
-FL,FL0021,002,FL0021-002,COLUMBIA COUNTY PUBLIC LIBRARY,308 N.W. COLUMBIA AVE.,LAKE CITY,32055,2891,COLUMBIA,3867581018,CE,NO,15400,R_11,0,R_11,2942,R_11,52,R_11,2012,00,00,00,-82.63798,30.192204,12,023,37775,67383,33,1105,2048,1203,0,1,29380,1
-FL,FL0001,005,FL0001-005,BELLEVIEW PUBLIC LIBRARY,13145 SE HWY 484,BELLEVIEW,34420,5831,MARION,3524382500,BR,NC,16000,R_11,0,R_11,2614,R_11,52,R_11,2012,00,00,00,-82.065647,29.028237,12,083,00000,332472,23,11.02000046,2051,1211,0,1,36100,0
-TN,TN0135,015,NA001-14,MADISON BRANCH LIBRARY,610 GALLATIN PIKE SOUTH,MADISON,37115,2192,DAVIDSON,6158625868,BR,CC,20680,R_11,0,R_11,2600,R_11,52,R_11,2012,00,00,00,-86.71406,36.257013,47,037,52006,635592,11,107.0100021,4001,4705,0,1,34980,0
-NH,NH0220,002,NH8995,WOODSVILLE FREE PUBLIC LIBRARY,14 SCHOOL LANE,WOODSVILLE,03785,1227,GRAFTON,6037473483,CE,NO,1360,IP10,0,R_11,1092,R_11,52,R_11,2012,00,00,00,-72.036754,44.150391,33,009,87140,88954,43,9606,3035,3302,0,1,17200,1
-NY,NY0651,002,8000580910,BROOKHAVEN FREE,273 BEAVER DAM RD,BROOKHAVEN,11719,9607,SUFFOLK,6312861923,CE,NC,5047,R_11,0,R_11,3140,R_11,52,R_11,2012,00,00,00,-72.911466,40.773631,36,103,09000,1500338,21,1592.040039,3015,3601,0,1,35620,0
-IL,IL0480,002,3060300,SCHILLER PARK PUBLIC LIBRARY,4200 OLD RIVER ROAD,SCHILLER PARK,60176,1630,COOK,8476780433,CE,NC,12500,R_11,0,R_11,3328,R_11,-1,U_11,2012,00,00,00,-87.860692,41.955696,17,031,68081,5214098,21,7708,1039,1705,0,1,16980,0
-NY,NY0004,019,0400300000,CROWN HEIGHTS,560 NEW YORK AVENUE,BROOKLYN,11225,5296,KINGS,7187731180,BR,CC,9280,R_11,0,R_11,2392,R_11,52,R_11,2012,00,00,00,-73.947776,40.661073,36,047,51000,2540230,11,804,1001,3609,0,1,35620,0
-OH,OH0039,011,139BK,STARK COUNTY DISTRICT LIBRARY,715 MARKET AVE.,CANTON,44702,1080,STARK,3304520665,BS,CC,-3,U_11,5,R_11,5976,R_11,52,R_11,2012,00,00,00,-81.373216,40.803018,39,151,12000,374328,13,7001,1048,3907,4,16,15940,0
-GA,GA0036,008,GA0036,MARION COUNTY PUBLIC LIBRARY,123 EAST 5TH AVE.,BUENNA VISTA,31803,0391,MARION,2296496385,BR,NO,4311,R_11,0,R_11,1588,R_11,52,R_11,2012,00,00,00,-84.522819,32.318596,13,197,11728,8706,42,9202,3058,1302,0,1,17980,0
-NJ,NJ0195,006,1300D,HAZLET MEMORIAL LIBRAY,251 MIDDLE ROAD,HAZLET,07730,1904,MONMOUTH,7322647164,BR,NC,10200,R_11,0,R_11,-1,U_11,52,R_11,2012,00,00,00,-74.176972,40.427274,34,025,00000,630092,21,8021,5017,3406,0,1,35620,0
-SC,SC8004,006,SC8004-002,NINETY-SIX BRANCH LIBRARY,100 SOUTH CAMBRIDGE STREET,NINETY-SIX,29666,1111,GREENWOOD,8645434749,BR,NO,3750,R_11,0,R_11,2314,R_11,52,R_11,2012,00,00,00,-82.023766,34.175093,45,047,50290,69739,42,9707.019531,4019,4503,0,1,24940,1
-WI,WI0382,002,WI0382,DRESSER PUBLIC LIBRARY,117 S. CENTRAL AVE.,DRESSER,54009,0547,POLK,7157552944,CE,NO,1274,R_11,0,R_11,1250,R_11,50,R_11,2012,00,00,00,-92.633499,45.355307,55,095,20850,43984,42,9607,3074,5507,0,1,0,.
-OK,OK0061,002,65,MAYSVILLE PUBLIC LIBRARY,508 WILLIAMS,MAYSVILLE,73057,0878,GARVIN,4058674748,CE,NO,3125,R_11,0,R_11,2184,R_11,52,R_11,2012,00,00,00,-97.40843,34.817807,40,049,47150,27387,42,6816,1114,4004,0,1,0,.
-CA,CA0027,006,M623,ATLANTIC BRANCH,2269 S. ATLANTIC BLVD,COMMERCE,90040,3955,LOS ANGELES,3237801176,BR,NC,3852,R_11,0,R_11,2080,R_11,52,R_11,2012,00,00,00,-118.164562,34.005171,06,037,14974,9889025,21,5323.02002,4024,0640,0,1,31080,0
-NJ,NJ0114,005,0705C,FRANKLIN BRANCH,192 DODD STREET,EAST ORANGE,07017,2107,ESSEX,9732667053,BR,NC,500,R_11,0,R_11,-1,U_11,52,R_11,2012,00,00,00,-74.21016,40.781824,34,013,19390,786754,21,100,1001,3410,0,1,35620,0
-OH,OH0177,002,277C,MORLEY LIBRARY,184 PHELPS ST.,PAINESVILLE,44077,3926,LAKE,4403523383,CE,NC,17500,R_11,0,R_11,3951,R_11,52,R_11,2012,00,00,00,-81.243767,41.727571,39,085,59416,229873,21,2045,1008,3914,0,1,17460,0
-NY,NY0420,002,5600221120,CARTHAGE FREE LIBRARY,412 BUDD STREET,CARTHAGE,13619,1206,JEFFERSON,3154932620,CE,NO,4800,R_11,0,R_11,2392,R_11,52,R_11,2012,00,00,00,-75.609727,43.979609,36,045,12683,118163,33,609,3023,3621,0,1,48060,0
-KY,KY0073,003,MARTIN,RUFUS M. REED PUBLIC LIBRARY,1442 RIVERFRONT ROAD,LOVELY,41231,0359,MARTIN,6063956500,BR,NO,3400,R_11,0,R_11,2080,R_11,52,R_11,2012,00,00,00,-82.367849,37.78319,21,159,00000,12839,42,9503,1003,2105,X,14,0,.
-MT,MT0051,005,1216,FRENCHTOWN SCHOOL AND COMMUNITY LIBRARY (MISSOULA CO.),16495 MAIN STREET,FRENCHTOWN,59834,M,MISSOULA,4066262730,BR,NO,4500,R_11,0,R_11,1040,R_11,52,R_11,2012,00,14,00,-114.227048,47.011982,30,063,29350,110114,42,16,1029,3000,0,1,33540,0
-PA,PA0147,002,908561265,MARY S BIESECKER PUB LIBRARY,230 S ROSINA AVE,SOMERSET,15501,1937,SOMERSET,8144454011,CE,NC,3531,R_11,0,R_11,2345,R_11,52,R_11,2012,00,00,00,-79.082912,40.00724,42,111,71776,77382,32,210,3002,4212,0,1,43740,1
-IA,IA0251,002,MM,ALGONA PUBLIC LIBRARY,210 N PHILLIPS,ALGONA,50511,1735,KOSSUTH,5152955476,CE,NO,16000,R_11,0,R_11,2912,R_11,52,R_11,2012,00,00,15,-94.226822,43.070044,19,109,01135,15369,33,9505,2042,1904,0,1,0,.
-NY,NY0004,061,0400300000,WINDSOR TERRACE,160 EAST 5TH STREET,BROOKLYN,11218,1702,KINGS,7186869707,BR,CC,7500,R_11,0,R_11,2392,R_11,52,R_11,2012,00,00,00,-73.97657,40.648829,36,047,51000,2540230,11,500,2004,3609,0,1,35620,0
-IN,IN0177,002,7284,BLOOMFIELD-EASTERN GREENE COUNTY PUBLIC LIBRARY,125 SOUTH FRANKLIN STREET,BLOOMFIELD,47424,1406,GREENE,8123844125,CE,NO,13400,R_11,0,R_11,2808,R_11,52,R_11,2012,00,00,00,-86.93852,39.02559,18,055,05716,33061,32,9554,4026,1808,0,1,0,.
-NJ,NJ0223,002,NJ0223-002,BUTLER PUBLIC LIBRARY,1 ACE ROAD,BUTLER,07045,1375,MORRIS,2018383262,CE,NC,6324,R_11,0,R_11,2659,R_11,52,R_11,2012,00,00,00,-74.342983,41.003358,34,027,09040,495941,21,405,1007,3411,0,1,35620,0
-TX,TX0247,011,329.299,CINCO RANCH BRANCH LIBRARY,2620 COMMERCIAL CENTER BLVD,KATY,77494,6407,FORT BEND,2813951311,BR,NC,33500,R_11,0,R_11,2524,R_11,52,R_11,2012,00,00,00,-95.787502,29.725205,48,157,00000,607952,41,6731.02002,1089,4822,2,15,26420,0
-MN,MN0035,015,M0213,ANOKA: RUM RIVER,4201 6TH AVE NW,ANOKA,55303,M,ANOKA,7635764695,BR,NC,30150,R_11,0,R_11,2312,R_11,52,R_11,2012,00,00,15,-93.380862,45.22981,27,003,01720,333226,21,505.0499878,3007,2706,0,1,33460,0
-OH,OH0002,019,102B17,HIGHLAND SQUARE BRANCH LIBRARY,807 W MARKET ST,AKRON,44303,1010,SUMMIT,3303762927,BR,CC,11500,R_11,0,R_11,2964,R_11,52,R_11,2012,00,06,15,-81.543313,41.096492,39,153,01000,541227,12,5073,3001,3913,0,1,10420,0
-MA,MA0015,002,ATHOL,ATHOL PUBLIC LIBRARY,568 MAIN STEET,ATHOL,01331,1888,WORCESTER,9782499515,CE,NO,8000,R_11,0,R_11,2350,R_11,52,R_11,2012,00,00,15,-72.228447,42.593893,25,027,02515,803785,32,7031,1026,2502,4,14,49340,0
-CA,CA0105,022,M701,SOUTHGATE LIBRARY,6132 66TH AVE.,SACRAMENTO,95823,2706,SACRAMENTO,9162642700,BR,CC,12000,R_11,0,R_11,1980,R_11,49,R_11,2012,00,00,00,-121.433101,38.492443,06,067,24498,1436262,21,50.02000046,1004,0607,0,1,40900,0
-CT,CT0124,002,11000,PLAINVILLE PUBLIC LIBRARY,56 EAST MAIN ST.,PLAINVILLE,06062,1934,HARTFORD,8607931446,CE,NC,30000,R_11,0,R_11,3016,R_11,52,R_11,2012,00,00,00,-72.865298,41.671845,09,003,00000,896248,21,4205,4005,0905,0,1,25540,0
-TX,TX0069,002,160,LIBRARY OF GRAHAM,910 CHERRY ST,GRAHAM,76450,3547,YOUNG,9405490600,CE,NO,14000,R_11,0,R_11,2184,R_11,52,R_11,2012,00,00,00,-98.587919,33.099243,48,503,30392,18385,33,9505,2005,4819,0,1,0,.
-MI,MI0322,002,MI0322-002,SAINT CLAIR SHORES PUBLIC LIBRARY,22500 ELEVEN MILE ROAD,SAINT CLAIR SHORES,48081,1399,MACOMB,5867719020,CE,NC,30000,R_11,0,R_11,3092,R_11,52,R_11,2012,00,00,00,-82.887251,42.496575,26,099,70760,842881,21,2511,1001,2609,0,1,19820,0
-TX,TX0183,002,269,WARD COUNTY LIBRARY,409 S DWIGHT ST,MONAHANS,79756,4609,WARD,4329433332,CE,NO,15000,R_11,0,R_11,2452,R_11,52,R_11,2012,00,00,00,-102.887829,31.59298,48,475,48936,10713,32,9502,1052,4823,0,5,0,.
-AL,AL0248,001,506-003,SOUTHSIDE PUBLIC LIBRARY,2255 HIGHWAY 77,SOUTHSIDE,35907,7905,ETOWAH,2564420105,CE,NO,1000,R_11,0,R_11,1404,R_11,52,R_11,2012,00,00,00,-86.022239,33.918787,01,055,71832,104298,23,105.0100021,3044,0104,0,1,23460,0
-ME,ME0296,002,268,SWANS ISLAND PUBLIC LIBRARY,451 ATLANTIC ROAD,SWAN'S ISLAND,04685,0012,HANCOCK,2075264330,CE,NO,3500,R_11,0,R_11,1200,R_11,52,R_11,2012,00,00,00,-68.417378,44.174678,23,009,00000,54551,43,9662,2136,2302,0,1,0,.
-KS,KS0200,002,KS0020,BELLE PLAINE PUBLIC LIBRARY,222 W. 5TH,BELLE PLAINE,67013,0700,SUMNER,6204883431,CE,NO,1987,R_11,0,R_11,1508,R_11,52,R_11,2012,00,00,00,-97.281473,37.393656,20,191,05500,23835,42,9621,4011,2004,0,1,48620,0
-CA,CA0144,006,M740,SUTTER BRANCH LIBRARY,2147 CALIFORNIA ST.,SUTTER,95982,2546,SUTTER,5307550485,BR,NC,970,R_11,0,R_11,996,R_11,52,R_11,2012,00,00,00,-121.748948,39.158979,06,101,77378,94951,23,508,2033,0603,0,1,49700,0
-DC,DC0001,012,NOE,NORTHEAST BRANCH LIBRARY,"330 7TH STREET, NE",WASHINGTON,20002,6104,DIST OF COLUMBIA,2026983320,BR,CC,13900,R_11,0,R_11,2496,R_11,52,R_11,2012,00,00,00,-76.996192,38.893985,11,001,50000,619020,11,83.01999664,1014,1198,0,1,47900,0
-KY,KY0109,007,WARREN,GRAHAM DRIVE COMMUNITY BRANCH,305 GRAHAM DRIVE,BOWLING GREEN,42101,1140,WARREN,2707811441,BR,CC,1627,R_11,0,R_11,2496,R_11,52,R_11,2012,00,00,00,-86.441796,37.003255,21,227,08902,115491,13,102,3031,2102,0,1,14540,0
-FL,FL0025,015,FL0025-015,KENDALL BRANCH LIBRARY,9101 S.W. 97 AVENUE,MIAMI,33176,1985,MIAMI-DADE,3052790520,BR,CC,14000,R_11,0,R_11,2559,R_11,52,R_11,2012,00,00,00,-80.350799,25.685623,12,086,36100,2565440,13,84.05000305,1009,1227,0,1,33100,0
-PA,PA0532,002,902024075,SENIOR BUS,1301 BEAVER AVENUE,PITTSBURGH,15233,2342,ALLEGHENY,4123211853,BS,CC,-3,U_11,1,R_11,403,R_11,50,R_11,2012,00,00,00,-80.028932,40.453027,42,003,61000,1227442,11,9806,1011,4214,0,1,38300,0
-NJ,NJ0204,002,NJ0204-002,FAIR HAVEN PUBLIC LIBRARY,748 RIVER ROAD,FAIR HAVEN,07704,3397,MONMOUTH,2017475031,CE,NC,1750,R_11,0,R_11,-1,U_11,52,R_11,2012,00,00,00,-74.035853,40.364273,34,025,22440,630092,21,8037,5000,3404,0,1,35620,0
-VI,VI0002,002,VI0002-002,ELAINE I. SPRAUVE PUBLIC LIBRARY,EST. ENIGHED,CRUZ BAY,00830,M,ST. JOHN,3407766359,BR,NO,1200,R_11,0,R_11,-1,U_11,-1,U_11,2012,00,00,00,-64.740289,18.338647,78,020,00000,-1,41,9501,3092,7898,X,14,0,.
-NC,NC0104,001,C-CHATHAM-BS,CHATHAM COUNTY BOOKMOBILE,500 N 2ND AVE,SILER CITY,27344,3123,CHATHAM,9195427725,BS,NC,-3,U_11,1,R_11,1000,R_11,50,R_11,2012,00,14,15,-79.46469,35.727411,37,037,61860,65280,32,204.0200043,1049,3702,0,1,20500,0
-TX,TX0402,003,49.362,SOUTHMOST BRANCH LIBRARY,4320 SOUTHMOST BLVD,BROWNSVILLE,78521,M,CAMERON,9565481055,BR,CC,20000,R_11,0,R_11,2602,R_11,52,R_11,2012,00,00,00,-97.449958,25.896555,48,061,10768,412577,12,133.0599976,1008,4834,0,1,15180,0
-OH,OH0163,008,OH0163-008,BUCKEYE LAKE BRANCH,41 W FIRST ST,BUCKEYE LAKE,43008,1708,LICKING,7409280472,BR,NO,1200,R_11,0,R_11,2433,R_11,52,R_11,2012,00,00,00,-82.482686,39.933065,39,089,09890,167194,23,7583,3012,3912,0,1,18140,0
-GA,GA0023,006,6632HILTON,IDA HILTON PUBLIC LIBRARY,1105 NORTHWAY,DARIEN,31305,1227,MCINTOSH,9124272124,BR,NO,6000,R_11,0,R_11,2110,R_11,-1,U_11,2012,00,00,00,-81.417136,31.394249,13,191,00000,14229,31,1102,1079,1301,X,14,15260,0
-LA,LA0057,015,60,ROSEDALE BRANCH,4036 JEFFERSON HIGHWAY,OLD JEFFERSON,70121,1630,JEFFERSON,5048384350,BR,NC,7138,R_11,0,R_11,2496,R_11,52,R_11,2012,00,00,00,-90.168164,29.961761,22,051,38145,433523,21,245,1006,2201,0,1,35380,0
-WI,WI0059,002,WI0059,CHIPPEWA FALLS PUBLIC LIBRARY,105 W. CENTRAL ST.,CHIPPEWA FALLS,54729,2397,CHIPPEWA,7157231146,CE,NC,18025,R_11,0,R_11,2682,R_11,52,R_11,2012,00,00,00,-91.393911,44.935456,55,017,14575,62857,23,103,1021,5503,0,1,20740,0
-WI,WI0065,002,WI0065,COLFAX PUBLIC LIBRARY,613 MAIN ST.,COLFAX,54730,0525,DUNN,7159624334,CE,NO,1800,R_11,0,R_11,1820,R_11,52,R_11,2012,00,00,00,-91.727195,44.999765,55,033,16275,44021,42,9703,3057,5503,0,1,32860,1
-DC,DC0001,016,SOE,SOUTHEAST BRANCH LIBRARY,"403 7TH STREET, SE",WASHINGTON,20003,2700,DIST OF COLUMBIA,2026983377,BR,CC,6431,R_11,0,R_11,2496,R_11,52,R_11,2012,00,00,00,-76.996189,38.88387,11,001,50000,619020,11,65,2000,1198,0,1,47900,0
-MI,MI0376,002,MI0376-002,GEORGE W. SPINDLER MEMORIAL LIBRARY,186 NORTH MAIN STREET,WOODLAND,48897,0068,BARRY,2693674694,CE,NO,1738,R_11,0,R_11,1040,R_11,52,R_11,2012,00,00,00,-85.13415,42.728953,26,015,88420,58936,42,101,2030,2603,0,1,24340,0
-WV,WV0040,002,39,CLARKSBURG-HARRISON CO. PUBLIC LIBRARY,404 W. PIKE STREET,CLARKSBURG,26301,2712,HARRISON,3046272236,CE,NO,30442,R_11,0,R_11,3484,R_11,52,R_11,2012,00,00,00,-80.340592,39.280797,54,033,15628,69298,32,301,1055,5401,0,1,17220,1
-IL,IL0524,002,3064900,THREE RIVERS PUBLIC LIBRARY DISTRICT,25207 WEST CHANNON DRIVE,CHANNAHON,60410,0300,WILL,8154676200,CE,NC,9200,R_11,0,R_11,3464,R_11,-1,U_11,2012,00,00,00,-88.224651,41.434087,17,197,12476,680584,21,8833.049805,1044,1716,0,1,16980,0
-IA,IA0048,002,BW,BERTHA BARTLETT PUBLIC LIBRARY,503 BROAD ST,STORY CITY,50248,1133,STORY,5157332685,CE,NO,8000,R_11,0,R_11,2738,R_11,52,R_11,2012,00,00,15,-93.597291,42.187095,19,169,75675,90816,31,106,1023,1904,0,1,11180,0
-MT,MT0011,003,1634,BUTTE-SILVER BOW PUBLIC LIBRARY SOUTH BRANCH,3100 HARRISON AVE,BUTTE,59701,M,SILVER BOW,4067233361,BR,NO,1200,R_11,0,R_11,1008,R_11,28,R_11,2012,02,00,00,-112.508,45.979868,30,093,11397,34362,33,7,1022,3000,0,1,15580,1
-NM,NM0110,001,NM0110-001,PUEBLO DE ABIQUIU LIBRARY & CULTURAL CENTER,COUNTY RD. #187 BUILDING #29,ABIQUIU,87510,0838,RIO ARRIBA,5056854884,CE,NO,400,R_11,0,R_11,1329,R_11,52,R_11,2012,00,00,00,-106.231605,36.156557,35,039,00000,40353,42,4,2205,3503,X,14,21580,1
-OR,OR0072,002,OR0072,SCAPPOOSE PUBLIC LIBRARY,52469 SE 2ND ST,SCAPPOOSE,97056,0400,COLUMBIA,5035437123,CE,NC,11020,R_11,0,R_11,2190,R_11,52,R_11,2012,00,00,00,-122.876339,45.756527,41,009,65500,49357,32,9709,2004,4101,0,1,38900,0
-TN,TN0132,005,CH001-04,EASTGATE BRANCH LIBRARY,5705 MARLIN ROAD,CHATTANOOGA,37411,5508,HAMILTON,4237575310,BR,CC,9084,R_11,0,R_11,2340,R_11,52,R_11,2012,00,00,00,-85.211325,35.007946,47,065,14000,340870,12,30,2044,4703,0,1,16860,0
-UT,UT0013,002,CACHCO1501,RICHMOND PUBLIC LIBRARY,38 WEST MAIN STREET,RICHMOND,84333,1409,CACHE,4352585525,CE,NO,2700,R_11,0,R_11,1565,R_11,52,R_11,2012,00,00,00,-111.809843,41.922642,49,005,63680,114559,41,1.00999999,3061,4901,0,1,30860,0
-IN,IN0123,002,5092,LOGANSPORT-CASS COUNTY PUBLIC LIBRARY,616 EAST BROADWAY,LOGANSPORT,46947,3155,CASS,5747536383,CE,NO,40448,R_11,0,R_11,3900,R_11,52,R_11,2012,00,00,00,-86.362725,40.754552,18,017,44658,38810,32,9515,2020,1804,0,1,30900,1
-WV,WV0082,002,81,GREENBRIER COUNTY PUBLIC LIBRARY,152 ROBERT W. MCCORMICK DRIVE,LEWISBURG,24901,1312,GREENBRIER,3046477568,CE,NO,13500,R_11,0,R_11,3068,R_11,52,R_11,2012,00,00,00,-80.448962,37.803713,54,025,46636,35710,33,9506,1041,5403,4,14,0,.
-ME,ME0042,002,43,CARIBOU PUBLIC LIBRARY,30 HIGH STREET,CARIBOU,04736,2796,AROOSTOOK,2074934214,CE,NO,11000,R_11,0,R_11,2548,R_11,52,R_11,2012,00,00,00,-68.010616,46.860615,23,003,10565,71388,33,9513,3015,2302,0,1,0,.
-MA,MA0091,002,ERVING,ERVING PUBLIC LIBRARY,17 MOORE ST.,ERVING,01344,9717,FRANKLIN,4134233348,CE,NO,1865,R_11,0,R_11,1044,R_11,52,R_11,2012,00,00,00,-72.490752,42.584079,25,011,41410,71646,32,404,2075,2502,0,1,24640,1
-CA,CA0068,026,M664,GEORGE SOUTH MERCED BRANCH,401 LESHER DRIVE,MERCED,95340,6572,MERCED,2097253909,BR,CC,1200,R_11,0,R_11,1300,R_11,52,R_11,2012,00,00,00,-120.504007,37.293699,06,047,46898,259966,13,15.02999973,1021,0616,0,1,32900,0
-MI,MI0240,012,MI0240-012,NORTH MUSKEGON BRANCH LIBRARY,1522 RUDDIMAN DRIVE,NORTH MUSKEGON,49445,3038,MUSKEGON,2317446080,BR,NC,3700,R_11,0,R_11,2496,R_11,52,R_11,2012,00,00,00,-86.267321,43.256368,26,121,58640,170021,22,15,2009,2602,0,1,34740,0
-VA,VA0076,002,VA0076-002,SHENANDOAH COUNTY LIBRARY,514 STONEY CREEK BLVD.,EDINBURG,22824,9142,SHENANDOAH,5409848200,CE,NO,13000,R_11,0,R_11,-1,U_11,-1,U_11,2012,00,00,00,-78.569018,38.825883,51,171,25008,42270,42,406,2037,5106,0,1,0,.
-NJ,NJ0188,002,NJ0188-002,SAYREVILLE FREE PUBLIC LIBRARY,1050 WASHINGTON ROAD,PARLIN,08859,1061,MIDDLESEX,2017270212,CE,NC,19500,R_11,0,R_11,3172,R_11,52,R_11,2012,00,00,00,-74.315701,40.464356,34,023,65790,816618,21,73.01000214,2000,3406,0,1,35620,0
-NY,NY0251,002,3200482455,ALICE CURTIS DESMOND AND HAMILTON FISH LIBRARY,472 ROUTE 403,GARRISON,10524,9803,PUTNAM,8454243020,CE,NC,9467,R_11,0,R_11,2258,R_11,52,R_11,2012,00,00,00,-73.94238,41.375377,36,079,00000,99916,41,108,2025,3618,0,1,35620,0
-GA,GA0004,015,GA0004,GORDON PUBLIC LIBRARY,284 MILLEDGEVILLE ROAD,GORDON,31031,0336,WILKINSON,4786285352,BR,NO,3385,R_11,0,R_11,2524,R_11,52,R_11,2012,00,00,00,-83.32934,32.882452,13,319,33980,9453,42,9603,3018,1308,0,1,0,.
-NJ,NJ0284,006,1900D,SUSSEX-WANTAGE BRANCH,69 ROUTE 639,SUSSEX,07461,2301,SUSSEX,9738753940,BR,NC,1400,R_11,0,R_11,-1,U_11,52,R_11,2012,00,00,00,-74.608678,41.210574,34,037,71670,148517,32,3712,1004,3405,4,16,35620,0
-NE,NE0038,002,040C,BRUNSWICK PUBLIC LIBRARY,303 FRANKLIN ST.,BRUNSWICK,68720,0011,ANTELOPE,4028422105,CE,NO,810,R_11,0,R_11,832,R_11,52,R_11,2012,00,00,00,-97.97071,42.292652,31,003,00000,6631,43,9796,2236,3103,0,1,0,.
-KS,KS0155,002,KS0111,GOODLAND PUBLIC LIBRARY,812 BROADWAY,GOODLAND,67735,3037,SHERMAN,7858995461,CE,NO,13500,R_11,0,R_11,2810,R_11,52,R_11,2012,00,00,00,-101.710437,39.351002,20,181,26875,6039,33,4537,4002,2001,0,1,0,.
-MI,MI0361,019,MI0361-002,WAYNE COUNTY PUBLIC LIBRARY,30555 MICHIGAN AVENUE,WESTLAND,48186,5310,WAYNE,7347277310,CE,NC,7000,R_11,0,R_11,2172,R_11,52,R_11,2012,00,00,00,-83.340555,42.286454,26,163,86000,1801789,21,5687,4017,2613,0,1,19820,0
-TX,TX0583,002,577,DILLEY PUBLIC LIBRARY,231 W FM 117,DILLEY,78017,3503,FRIO,8309651951,CE,NO,2340,R_11,0,R_11,2000,R_11,50,R_11,2012,00,00,00,-99.170073,28.67396,48,163,20428,17367,33,9503,1253,4823,0,1,0,.
-GA,GA0022,026,GA0022,PEACHTREE BRANCH LIBRARY,"1315 PEACHTREE STREET, N.E.",ATLANTA,30309,7515,FULTON,4048857830,BR,CC,10000,R_11,0,R_11,1806,R_11,52,R_11,2012,00,00,00,-84.38501,33.790479,13,121,04000,949580,11,4,1013,1305,0,1,12060,0
-IN,IN0180,003,7531,ELLETTSVILLE BRANCH,600 WEST TEMPERANCE,ELLETTSVILLE,47429,1324,MONROE,8123492515,BR,NC,12000,R_11,0,R_11,3328,R_11,52,R_11,2012,00,00,00,-86.625077,39.234002,18,105,20800,140063,23,13.02999973,1050,1809,0,1,14020,0
-WI,WI0023,002,WI0023,BELLEVILLE PUBLIC LIBRARY,130 S. VINE ST.,BELLEVILLE,53508,0140,DANE,6084241812,CE,NC,4600,R_11,0,R_11,2652,R_11,52,R_11,2012,00,00,00,-89.53399,42.858436,55,025,06300,496456,42,126,3121,5502,0,1,31540,0
-CA,CA0146,004,M742,EL RETIRO BRANCH LIBRARY,126 VISTA DEL PARQUE,REDONDO BEACH,90277,6112,LOS ANGELES,3103750922,BR,NC,4158,R_11,0,R_11,2222,R_11,52,R_11,2012,00,00,00,-118.379754,33.815298,06,037,80000,9889025,12,6513.040039,2005,0633,0,1,31080,0
-TX,TX0099,040,189.360,MCGOVERN-STELLA LINK BRANCH LIBRARY,7405 STELLA LINK RD,HOUSTON,77025,M,HARRIS,8323932630,BR,CC,20393,R_11,0,R_11,2098,R_11,50,R_11,2012,00,00,00,-95.440039,29.698416,48,201,35000,4173695,11,4130,2005,4807,0,1,26420,0
http://git-wip-us.apache.org/repos/asf/cassandra/blob/56cfc6ea/examples/pig/test/populate-cli.txt
----------------------------------------------------------------------
diff --git a/examples/pig/test/populate-cli.txt b/examples/pig/test/populate-cli.txt
deleted file mode 100644
index b2dda58..0000000
--- a/examples/pig/test/populate-cli.txt
+++ /dev/null
@@ -1,134 +0,0 @@
-create keyspace PigTest with placement_strategy = 'org.apache.cassandra.locator.SimpleStrategy' and strategy_options={replication_factor:1};
-use PigTest;
-create column family SomeApp with
-key_validation_class = UTF8Type and
-default_validation_class = LexicalUUIDType and
-comparator = UTF8Type and
-column_metadata =
-[
- {column_name: name, validation_class: UTF8Type, index_type: KEYS},
- {column_name: vote_type, validation_class: UTF8Type},
- {column_name: rating, validation_class: Int32Type},
- {column_name: score, validation_class: LongType},
- {column_name: percent, validation_class: FloatType},
- {column_name: atomic_weight, validation_class: DoubleType},
- {column_name: created, validation_class: DateType},
-];
-
-create column family CopyOfSomeApp with
-key_validation_class = UTF8Type and
-default_validation_class = LexicalUUIDType and
-comparator = UTF8Type and
-column_metadata =
-[
- {column_name: name, validation_class: UTF8Type, index_type: KEYS},
- {column_name: vote_type, validation_class: UTF8Type},
- {column_name: rating, validation_class: Int32Type},
- {column_name: score, validation_class: LongType},
- {column_name: percent, validation_class: FloatType},
- {column_name: atomic_weight, validation_class: DoubleType},
- {column_name: created, validation_class: DateType},
-];
-
-set SomeApp['foo']['name'] = 'User Foo';
-set SomeApp['foo']['vote_type'] = 'like';
-set SomeApp['foo']['rating'] = 8;
-set SomeApp['foo']['score'] = 125000;
-set SomeApp['foo']['percent'] = '85.0';
-set SomeApp['foo']['atomic_weight'] = '2.7182818284590451';
-set SomeApp['foo']['created'] = 1335890877;
-
-set SomeApp['bar']['name'] = 'User Bar';
-set SomeApp['bar']['vote_type'] = 'like';
-set SomeApp['bar']['rating'] = 9;
-set SomeApp['bar']['score'] = 15000;
-set SomeApp['bar']['percent'] = '35.0';
-set SomeApp['bar']['atomic_weight'] = '3.1415926535897931';
-set SomeApp['bar']['created'] = 1335890877;
-
-set SomeApp['baz']['name'] = 'User Baz';
-set SomeApp['baz']['vote_type'] = 'dislike';
-set SomeApp['baz']['rating'] = 3;
-set SomeApp['baz']['score'] = 512000;
-set SomeApp['baz']['percent'] = '95.3';
-set SomeApp['baz']['atomic_weight'] = '1.61803399';
-set SomeApp['baz']['created'] = 1335890877;
-set SomeApp['baz']['extra1'] = lexicaluuid();
-set SomeApp['baz']['extra2'] = lexicaluuid();
-set SomeApp['baz']['extra3'] = lexicaluuid();
-
-set SomeApp['qux']['name'] = 'User Qux';
-set SomeApp['qux']['vote_type'] = 'dislike';
-set SomeApp['qux']['rating'] = 2;
-set SomeApp['qux']['score'] = 12000;
-set SomeApp['qux']['percent'] = '64.7';
-set SomeApp['qux']['atomic_weight'] = '0.660161815846869';
-set SomeApp['qux']['created'] = 1335890877;
-set SomeApp['qux']['extra1'] = lexicaluuid();
-set SomeApp['qux']['extra2'] = lexicaluuid();
-set SomeApp['qux']['extra3'] = lexicaluuid();
-set SomeApp['qux']['extra4'] = lexicaluuid();
-set SomeApp['qux']['extra5'] = lexicaluuid();
-set SomeApp['qux']['extra6'] = lexicaluuid();
-set SomeApp['qux']['extra7'] = lexicaluuid();
-
-create column family U8 with
- key_validation_class = UTF8Type and
- comparator = UTF8Type;
-
-create column family Bytes with
- key_validation_class = BytesType and
- comparator = UTF8Type;
-
-set U8['foo']['x'] = ascii('Z');
-set Bytes[ascii('foo')]['x'] = ascii('Z');
-
-create column family CC with
- key_validation_class = UTF8Type and
- default_validation_class=CounterColumnType
- and comparator=UTF8Type;
-
-incr CC['chuck']['kick'];
-incr CC['chuck']['kick'];
-incr CC['chuck']['kick'];
-incr CC['chuck']['fist'];
-
-create column family Compo
- with key_validation_class = UTF8Type
- and default_validation_class = UTF8Type
- and comparator = 'CompositeType(UTF8Type,UTF8Type)';
-
-set Compo['punch']['bruce:lee'] = 'ouch';
-set Compo['punch']['bruce:bruce'] = 'hunh?';
-set Compo['kick']['bruce:lee'] = 'oww';
-set Compo['kick']['bruce:bruce'] = 'watch it, mate';
-
-create column family CompoInt
- with key_validation_class = UTF8Type
- and default_validation_class = UTF8Type
- and comparator = 'CompositeType(LongType,LongType)';
-
-set CompoInt['clock']['1:0'] = 'z';
-set CompoInt['clock']['1:30'] = 'zzzz';
-set CompoInt['clock']['2:30'] = 'daddy?';
-set CompoInt['clock']['6:30'] = 'coffee...';
-
-create column family CompoIntCopy
- with key_validation_class = UTF8Type
- and default_validation_class = UTF8Type
- and comparator = 'CompositeType(LongType,LongType)';
-
-create column family CompoKey
- with key_validation_class = 'CompositeType(UTF8Type,LongType)'
- and default_validation_class = UTF8Type
- and comparator = LongType;
-
-set CompoKey['clock:10']['1'] = 'z';
-set CompoKey['clock:20']['1'] = 'zzzz';
-set CompoKey['clock:30']['2'] = 'daddy?';
-set CompoKey['clock:40']['6'] = 'coffee...';
-
-create column family CompoKeyCopy
- with key_validation_class = 'CompositeType(UTF8Type,LongType)'
- and default_validation_class = UTF8Type
- and comparator = LongType;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/56cfc6ea/examples/pig/test/populate-cql.txt
----------------------------------------------------------------------
diff --git a/examples/pig/test/populate-cql.txt b/examples/pig/test/populate-cql.txt
deleted file mode 100644
index 00b81db..0000000
--- a/examples/pig/test/populate-cql.txt
+++ /dev/null
@@ -1,33 +0,0 @@
-CREATE KEYSPACE cql3ks WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1 };
-USE cql3ks;
-
-CREATE TABLE test (a int PRIMARY KEY, b int);
-CREATE TABLE moredata (x int PRIMARY KEY, y int);
-INSERT INTO test (a,b) VALUES (1,1);
-INSERT INTO test (a,b) VALUES (2,2);
-INSERT INTO test (a,b) VALUES (3,3);
-INSERT INTO moredata (x, y) VALUES (4,4);
-INSERT INTO moredata (x, y) VALUES (5,5);
-INSERT INTO moredata (x, y) VALUES (6,6);
-
-CREATE TABLE compotable (a int, b int, c text, d text, PRIMARY KEY (a,b,c));
-INSERT INTO compotable (a, b , c , d ) VALUES ( 1,1,'One','match');
-INSERT INTO compotable (a, b , c , d ) VALUES ( 2,2,'Two','match');
-INSERT INTO compotable (a, b , c , d ) VALUES ( 3,3,'Three','match');
-INSERT INTO compotable (a, b , c , d ) VALUES ( 4,4,'Four','match');
-
-create table compmore (id int PRIMARY KEY, x int, y int, z text, data text);
-INSERT INTO compmore (id, x, y, z,data) VALUES (1,5,6,'Fix','nomatch');
-INSERT INTO compmore (id, x, y, z,data) VALUES (2,6,5,'Sive','nomatch');
-INSERT INTO compmore (id, x, y, z,data) VALUES (3,7,7,'Seven','match');
-INSERT INTO compmore (id, x, y, z,data) VALUES (4,8,8,'Eight','match');
-INSERT INTO compmore (id, x, y, z,data) VALUES (5,9,10,'Ninen','nomatch');
-
-
-CREATE TABLE collectiontable(m text PRIMARY KEY, n map<text, text>);
-UPDATE collectiontable SET n['key1'] = 'value1' WHERE m = 'book1';
-UPDATE collectiontable SET n['key2'] = 'value2' WHERE m = 'book2';
-UPDATE collectiontable SET n['key3'] = 'value3' WHERE m = 'book3';
-UPDATE collectiontable SET n['key4'] = 'value4' WHERE m = 'book4';
-
-
http://git-wip-us.apache.org/repos/asf/cassandra/blob/56cfc6ea/examples/pig/test/test_cql_storage.pig
----------------------------------------------------------------------
diff --git a/examples/pig/test/test_cql_storage.pig b/examples/pig/test/test_cql_storage.pig
deleted file mode 100644
index 822748e..0000000
--- a/examples/pig/test/test_cql_storage.pig
+++ /dev/null
@@ -1,14 +0,0 @@
-moretestvalues= LOAD 'cql://cql3ks/moredata/' USING CqlNativeStorage;
-insertformat= FOREACH moretestvalues GENERATE TOTUPLE(TOTUPLE('a',x)),TOTUPLE(y);
-STORE insertformat INTO 'cql://cql3ks/test?output_query=UPDATE+cql3ks.test+set+b+%3D+%3F' USING CqlNativeStorage;
-
--- composite key
-moredata = load 'cql://cql3ks/compmore' USING CqlNativeStorage;
-insertformat = FOREACH moredata GENERATE TOTUPLE (TOTUPLE('a',x),TOTUPLE('b',y), TOTUPLE('c',z)),TOTUPLE(data);
-STORE insertformat INTO 'cql://cql3ks/compotable?output_query=UPDATE%20cql3ks.compotable%20SET%20d%20%3D%20%3F' USING CqlNativeStorage;
-
--- collection column
-collectiontable = LOAD 'cql://cql3ks/collectiontable/' USING CqlNativeStorage;
--- recs= (((m,kk)),((map,(m,mm),(n,nn))))
-recs= FOREACH collectiontable GENERATE TOTUPLE(TOTUPLE('m', m) ), TOTUPLE(TOTUPLE('map', TOTUPLE('m', 'mm'), TOTUPLE('n', 'nn')));
-store recs INTO 'cql://cql3ks/collectiontable?output_query=update+cql3ks.collectiontable+set+n+%3D+%3F' USING CqlNativeStorage();
http://git-wip-us.apache.org/repos/asf/cassandra/blob/56cfc6ea/examples/pig/test/test_storage.pig
----------------------------------------------------------------------
diff --git a/examples/pig/test/test_storage.pig b/examples/pig/test/test_storage.pig
deleted file mode 100644
index 026cb02..0000000
--- a/examples/pig/test/test_storage.pig
+++ /dev/null
@@ -1,85 +0,0 @@
-rows = LOAD 'cassandra://PigTest/SomeApp' USING CassandraStorage();
--- full copy
-STORE rows INTO 'cassandra://PigTest/CopyOfSomeApp' USING CassandraStorage();
--- single tuple
-onecol = FOREACH rows GENERATE key, percent;
-STORE onecol INTO 'cassandra://PigTest/CopyOfSomeApp' USING CassandraStorage();
--- bag only
-other = FOREACH rows GENERATE key, columns;
-STORE other INTO 'cassandra://PigTest/CopyOfSomeApp' USING CassandraStorage();
-
-
--- filter
-likes = FILTER rows by vote_type.value eq 'like' and rating.value > 5;
-dislikes_extras = FILTER rows by vote_type.value eq 'dislike' AND COUNT(columns) > 0;
-
--- store these too
-STORE likes INTO 'cassandra://PigTest/CopyOfSomeApp' USING CassandraStorage();
-STORE dislikes_extras INTO 'cassandra://PigTest/CopyOfSomeApp' USING CassandraStorage();
-
--- filter to fully visible rows (no uuid columns) and dump
-visible = FILTER rows BY COUNT(columns) == 0;
-dump visible;
-
-
-
--- test key types with a join
-U8 = load 'cassandra://PigTest/U8' using CassandraStorage();
-Bytes = load 'cassandra://PigTest/Bytes' using CassandraStorage();
-
--- cast key to chararray
-b = foreach Bytes generate (chararray)key, columns;
-
--- key in Bytes is a bytearray, U8 chararray
-a = join Bytes by key, U8 by key;
-dump a
-
--- key should now be cast into a chararray
-c = join b by (chararray)key, U8 by (chararray)key;
-dump c
-
-
---
--- Test counter column family support
---
-CC = load 'cassandra://PigTest/CC' using CassandraStorage();
-
-total_hits = foreach CC generate key, SUM(columns.value);
-
-dump total_hits;
-
---
--- Test CompositeType
---
-
-compo = load 'cassandra://PigTest/Compo' using CassandraStorage();
-
-compo = foreach compo generate key as method, flatten(columns);
-
-lee = filter compo by columns::name == ('bruce','lee');
-
-dump lee;
-
-night = load 'cassandra://PigTest/CompoInt' using CassandraStorage();
-night = foreach night generate flatten(columns);
-night = foreach night generate (int)columns::name.$0+(double)columns::name.$1/60 as hour, columns::value as noise;
-
--- What happens at the darkest hour?
-darkest = filter night by hour > 2 and hour < 5;
-
-dump darkest;
-
-compo_int_rows = LOAD 'cassandra://PigTest/CompoInt' USING CassandraStorage();
-STORE compo_int_rows INTO 'cassandra://PigTest/CompoIntCopy' USING CassandraStorage();
-
---
--- Test CompositeKey
---
-
-compokeys = load 'cassandra://PigTest/CompoKey' using CassandraStorage();
-compokeys = filter compokeys by key.$1 == 40;
-
-dump compokeys;
-
-compo_key_rows = LOAD 'cassandra://PigTest/CompoKey' USING CassandraStorage();
-STORE compo_key_rows INTO 'cassandra://PigTest/CompoKeyCopy' USING CassandraStorage();
[3/5] cassandra git commit: Remove Pig support
Posted by al...@apache.org.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/56cfc6ea/test/pig/org/apache/cassandra/pig/CqlTableTest.java
----------------------------------------------------------------------
diff --git a/test/pig/org/apache/cassandra/pig/CqlTableTest.java b/test/pig/org/apache/cassandra/pig/CqlTableTest.java
deleted file mode 100644
index 3902fce..0000000
--- a/test/pig/org/apache/cassandra/pig/CqlTableTest.java
+++ /dev/null
@@ -1,303 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.cassandra.pig;
-
-import java.io.IOException;
-import java.util.Iterator;
-
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.pig.data.Tuple;
-import org.apache.thrift.TException;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class CqlTableTest extends PigTestBase
-{
- private static String[] statements = {
- "DROP KEYSPACE IF EXISTS cql3ks",
- "CREATE KEYSPACE cql3ks WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 1}",
- "USE cql3ks;",
-
- "CREATE TABLE cqltable (key1 text, key2 int, column1 int, column2 float, primary key(key1, key2))",
- "INSERT INTO cqltable (key1, key2, column1, column2) values ('key1', 111, 100, 10.1)",
- "CREATE TABLE compactcqltable (key1 text, column1 int, column2 float, primary key(key1)) WITH COMPACT STORAGE",
- "INSERT INTO compactcqltable (key1, column1, column2) values ('key1', 100, 10.1)",
-
- "CREATE TABLE test (a int PRIMARY KEY, b int);",
- "CREATE INDEX test_b on test (b);",
-
- "CREATE TABLE moredata (x int PRIMARY KEY, y int);",
- "CREATE TABLE test_bulk (a int PRIMARY KEY, b int);",
- "INSERT INTO test_bulk (a,b) VALUES (1,1);",
- "INSERT INTO test_bulk (a,b) VALUES (2,2);",
- "INSERT INTO test_bulk (a,b) VALUES (3,3);",
- "INSERT INTO test (a,b) VALUES (1,1);",
- "INSERT INTO test (a,b) VALUES (2,2);",
- "INSERT INTO test (a,b) VALUES (3,3);",
- "INSERT INTO moredata (x, y) VALUES (4,4);",
- "INSERT INTO moredata (x, y) VALUES (5,5);",
- "INSERT INTO moredata (x, y) VALUES (6,6);",
-
- "CREATE TABLE compotable (a int, b int, c text, d text, PRIMARY KEY (a,b,c));",
- "INSERT INTO compotable (a, b , c , d ) VALUES ( 1,1,'One','match');",
- "INSERT INTO compotable (a, b , c , d ) VALUES ( 2,2,'Two','match');",
- "INSERT INTO compotable (a, b , c , d ) VALUES ( 3,3,'Three','match');",
- "INSERT INTO compotable (a, b , c , d ) VALUES ( 4,4,'Four','match');",
-
- "create table compmore (id int PRIMARY KEY, x int, y int, z text, data text);",
- "INSERT INTO compmore (id, x, y, z,data) VALUES (1,5,6,'Fix','nomatch');",
- "INSERT INTO compmore (id, x, y, z,data) VALUES (2,6,5,'Sive','nomatch');",
- "INSERT INTO compmore (id, x, y, z,data) VALUES (3,7,7,'Seven','match');",
- "INSERT INTO compmore (id, x, y, z,data) VALUES (4,8,8,'Eight','match');",
- "INSERT INTO compmore (id, x, y, z,data) VALUES (5,9,10,'Ninen','nomatch');",
-
- "CREATE TABLE collectiontable(m text PRIMARY KEY, n map<text, text>);",
- "UPDATE collectiontable SET n['key1'] = 'value1' WHERE m = 'book1';",
- "UPDATE collectiontable SET n['key2'] = 'value2' WHERE m = 'book2';",
- "UPDATE collectiontable SET n['key3'] = 'value3' WHERE m = 'book3';",
- "UPDATE collectiontable SET n['key4'] = 'value4' WHERE m = 'book4';",
- "CREATE TABLE nulltable(m text PRIMARY KEY, n map<text, text>);",
- "UPDATE nulltable SET n['key1'] = 'value1' WHERE m = 'book1';",
- "UPDATE nulltable SET n['key2'] = 'value2' WHERE m = 'book2';",
- "UPDATE nulltable SET n['key3'] = 'value3' WHERE m = 'book3';",
- "UPDATE nulltable SET n['key4'] = 'value4' WHERE m = 'book4';",
- };
-
- @BeforeClass
- public static void setup() throws IOException, ConfigurationException, TException
- {
- startCassandra();
- executeCQLStatements(statements);
- startHadoopCluster();
- }
-
- @Test
- public void testCqlNativeStorageSchema() throws IOException
- {
- //input_cql=select * from cqltable where token(key1) > ? and token(key1) <= ?
- cqlTableSchemaTest("rows = LOAD 'cql://cql3ks/cqltable?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20cqltable%20where%20token(key1)%20%3E%20%3F%20and%20token(key1)%20%3C%3D%20%3F' USING CqlNativeStorage();");
-
- //input_cql=select * from compactcqltable where token(key1) > ? and token(key1) <= ?
- compactCqlTableSchemaTest("rows = LOAD 'cql://cql3ks/compactcqltable?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20compactcqltable%20where%20token(key1)%20%3E%20%3F%20and%20token(key1)%20%3C%3D%20%3F' USING CqlNativeStorage();");
- }
-
- private void compactCqlTableSchemaTest(String initialQuery) throws IOException
- {
- pig.registerQuery(initialQuery);
- Iterator<Tuple> it = pig.openIterator("rows");
- if (it.hasNext()) {
- Tuple t = it.next();
- Assert.assertEquals(t.get(0).toString(), "key1");
- Assert.assertEquals(t.get(1), 100);
- Assert.assertEquals(t.get(2), 10.1f);
- Assert.assertEquals(3, t.size());
- }
- else
- {
- Assert.fail("Failed to get data for query " + initialQuery);
- }
- }
-
- private void cqlTableSchemaTest(String initialQuery) throws IOException
- {
- pig.registerQuery(initialQuery);
- Iterator<Tuple> it = pig.openIterator("rows");
- if (it.hasNext()) {
- Tuple t = it.next();
- Assert.assertEquals(t.get(0).toString(), "key1");
- Assert.assertEquals(t.get(1), 111);
- Assert.assertEquals(t.get(2), 100);
- Assert.assertEquals(t.get(3), 10.1f);
- Assert.assertEquals(4, t.size());
- }
- else
- {
- Assert.fail("Failed to get data for query " + initialQuery);
- }
- }
-
- @Test
- public void testCqlNativeStorageSingleKeyTable() throws IOException
- {
- //input_cql=select * from moredata where token(x) > ? and token(x) <= ?
- singleKeyTableTest("moretestvalues= LOAD 'cql://cql3ks/moredata?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20moredata%20where%20token(x)%20%3E%20%3F%20and%20token(x)%20%3C%3D%20%3F' USING CqlNativeStorage();");
- }
-
- private void singleKeyTableTest(String initialQuery) throws IOException
- {
- pig.setBatchOn();
- pig.registerQuery(initialQuery);
- pig.registerQuery("insertformat= FOREACH moretestvalues GENERATE TOTUPLE(TOTUPLE('a',x)),TOTUPLE(y);");
- pig.registerQuery("STORE insertformat INTO 'cql://cql3ks/test?" + defaultParameters + nativeParameters + "&output_query=UPDATE+cql3ks.test+set+b+%3D+%3F' USING CqlNativeStorage();");
- pig.executeBatch();
- //(5,5)
- //(6,6)
- //(4,4)
- //(2,2)
- //(3,3)
- //(1,1)
- //input_cql=select * from test where token(a) > ? and token(a) <= ?
- pig.registerQuery("result= LOAD 'cql://cql3ks/test?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20test%20where%20token(a)%20%3E%20%3F%20and%20token(a)%20%3C%3D%20%3F' USING CqlNativeStorage();");
- Iterator<Tuple> it = pig.openIterator("result");
- int count = 0;
- while (it.hasNext()) {
- Tuple t = it.next();
- Assert.assertEquals(t.get(0), t.get(1));
- count ++;
- }
- Assert.assertEquals(6, count);
- }
-
- @Test
- public void testCqlNativeStorageCompositeKeyTable() throws IOException
- {
- //input_cql=select * from compmore where token(id) > ? and token(id) <= ?
- compositeKeyTableTest("moredata= LOAD 'cql://cql3ks/compmore?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20compmore%20where%20token(id)%20%3E%20%3F%20and%20token(id)%20%3C%3D%20%3F' USING CqlNativeStorage();");
- }
-
- private void compositeKeyTableTest(String initialQuery) throws IOException
- {
- pig.setBatchOn();
- pig.registerQuery(initialQuery);
- pig.registerQuery("insertformat = FOREACH moredata GENERATE TOTUPLE (TOTUPLE('a',x),TOTUPLE('b',y), TOTUPLE('c',z)),TOTUPLE(data);");
- pig.registerQuery("STORE insertformat INTO 'cql://cql3ks/compotable?" + defaultParameters + nativeParameters + "&output_query=UPDATE%20cql3ks.compotable%20SET%20d%20%3D%20%3F' USING CqlNativeStorage();");
- pig.executeBatch();
-
- //(5,6,Fix,nomatch)
- //(3,3,Three,match)
- //(1,1,One,match)
- //(2,2,Two,match)
- //(7,7,Seven,match)
- //(8,8,Eight,match)
- //(6,5,Sive,nomatch)
- //(4,4,Four,match)
- //(9,10,Ninen,nomatch)
- //input_cql=select * from compotable where token(a) > ? and token(a) <= ?
- pig.registerQuery("result= LOAD 'cql://cql3ks/compotable?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20compotable%20where%20token(a)%20%3E%20%3F%20and%20token(a)%20%3C%3D%20%3F' USING CqlNativeStorage();");
- Iterator<Tuple> it = pig.openIterator("result");
- int count = 0;
- while (it.hasNext()) {
- it.next();
- count ++;
- }
- Assert.assertEquals(count, 9);
- }
-
- @Test
- public void testCqlNativeStorageCollectionColumnTable() throws IOException
- {
- //input_cql=select * from collectiontable where token(m) > ? and token(m) <= ?
- CollectionColumnTableTest("collectiontable= LOAD 'cql://cql3ks/collectiontable?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20collectiontable%20where%20token(m)%20%3E%20%3F%20and%20token(m)%20%3C%3D%20%3F' USING CqlNativeStorage();");
- }
-
- private void CollectionColumnTableTest(String initialQuery) throws IOException
- {
- pig.setBatchOn();
- pig.registerQuery(initialQuery);
- pig.registerQuery("recs= FOREACH collectiontable GENERATE TOTUPLE(TOTUPLE('m', m) ), TOTUPLE(TOTUPLE('map', TOTUPLE('m', 'mm'), TOTUPLE('n', 'nn')));");
- pig.registerQuery("STORE recs INTO 'cql://cql3ks/collectiontable?" + defaultParameters + nativeParameters + "&output_query=update+cql3ks.collectiontable+set+n+%3D+%3F' USING CqlNativeStorage();");
- pig.executeBatch();
-
- //(book2,((m,mm),(n,nn)))
- //(book3,((m,mm),(n,nn)))
- //(book4,((m,mm),(n,nn)))
- //(book1,((m,mm),(n,nn)))
- //input_cql=select * from collectiontable where token(m) > ? and token(m) <= ?
- pig.registerQuery("result= LOAD 'cql://cql3ks/collectiontable?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20collectiontable%20where%20token(m)%20%3E%20%3F%20and%20token(m)%20%3C%3D%20%3F' USING CqlNativeStorage();");
- Iterator<Tuple> it = pig.openIterator("result");
- if (it.hasNext()) {
- Tuple t = it.next();
- Tuple t1 = (Tuple) t.get(1);
- Assert.assertEquals(t1.size(), 2);
- Tuple element1 = (Tuple) t1.get(0);
- Tuple element2 = (Tuple) t1.get(1);
- Assert.assertEquals(element1.get(0), "m");
- Assert.assertEquals(element1.get(1), "mm");
- Assert.assertEquals(element2.get(0), "n");
- Assert.assertEquals(element2.get(1), "nn");
- }
- else
- {
- Assert.fail("Can't fetch any data");
- }
- }
-
- @Test
- public void testCqlNativeStorageNullTuples() throws IOException
- {
- //input_cql=select * from collectiontable where token(m) > ? and token(m) <= ?
- NullTupleTest("nulltable= LOAD 'cql://cql3ks/collectiontable?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20nulltable%20where%20token(m)%20%3E%20%3F%20and%20token(m)%20%3C%3D%20%3F' USING CqlNativeStorage();");
- }
-
- private void NullTupleTest(String initialQuery) throws IOException
- {
- pig.setBatchOn();
- pig.registerQuery(initialQuery);
- pig.registerQuery("recs= FOREACH nulltable GENERATE TOTUPLE(TOTUPLE('m', m) ), TOTUPLE(TOTUPLE('map', TOTUPLE('m', null), TOTUPLE('n', null)));");
- pig.registerQuery("STORE recs INTO 'cql://cql3ks/nulltable?" + defaultParameters + nativeParameters + "&output_query=update+cql3ks.nulltable+set+n+%3D+%3F' USING CqlNativeStorage();");
- pig.executeBatch();
-
- pig.registerQuery("result= LOAD 'cql://cql3ks/nulltable?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20nulltable%20where%20token(m)%20%3E%20%3F%20and%20token(m)%20%3C%3D%20%3F' USING CqlNativeStorage();");
- Iterator<Tuple> it = pig.openIterator("result");
- if (it.hasNext()) {
- Tuple t = it.next();
- Tuple t1 = (Tuple) t.get(1);
- Assert.assertEquals(t1.size(), 2);
- Tuple element1 = (Tuple) t1.get(0);
- Tuple element2 = (Tuple) t1.get(1);
- Assert.assertEquals(element1.get(0), "m");
- Assert.assertEquals(element1.get(1), "");
- Assert.assertEquals(element2.get(0), "n");
- Assert.assertEquals(element2.get(1), "");
- }
- else
- {
- Assert.fail("Can't fetch any data");
- }
- }
-
- @Test
- public void testCqlStorageSingleKeyTableBulkLoad() throws TException, IOException
- {
- pig.setBatchOn();
- //input_cql=select * from moredata where token(x) > ? and token(x) <= ?
- pig.registerQuery("moretestvalues= LOAD 'cql://cql3ks/moredata?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20moredata%20where%20token(x)%20%3E%20%3F%20and%20token(x)%20%3C%3D%20%3F' USING CqlNativeStorage();");
- pig.registerQuery("insertformat= FOREACH moretestvalues GENERATE TOTUPLE(x, y);");
- pig.registerQuery("STORE insertformat INTO 'cql://cql3ks/test_bulk?" + defaultParameters + nativeParameters + "&bulk_output_format=true&bulk_cf_schema=CREATE%20TABLE%20cql3ks.test_bulk%20(a%20int%20PRIMARY%20KEY%2C%20b%20int)&bulk_insert_statement=Insert%20into%20cql3ks.test_bulk(a%2C%20b)%20values(%3F%2C%3F)' USING CqlNativeStorage();");
- pig.executeBatch();
-
- //(5,5)
- //(6,6)
- //(4,4)
- //(2,2)
- //(3,3)
- //(1,1)
- //input_cql=select * from test_bulk1 where token(a) > ? and token(a) <= ?
- pig.registerQuery("result= LOAD 'cql://cql3ks/test_bulk?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20test_bulk%20where%20token(a)%20%3E%20%3F%20and%20token(a)%20%3C%3D%20%3F' USING CqlNativeStorage();");
- Iterator<Tuple> it = pig.openIterator("result");
- int count = 0;
- while (it.hasNext()) {
- Tuple t = it.next();
- Assert.assertEquals(t.get(0), t.get(1));
- count ++;
- }
- Assert.assertEquals(6, count);
- }
-}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/56cfc6ea/test/pig/org/apache/cassandra/pig/PigTestBase.java
----------------------------------------------------------------------
diff --git a/test/pig/org/apache/cassandra/pig/PigTestBase.java b/test/pig/org/apache/cassandra/pig/PigTestBase.java
deleted file mode 100644
index 067397b..0000000
--- a/test/pig/org/apache/cassandra/pig/PigTestBase.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.cassandra.pig;
-
-import java.io.IOException;
-
-import com.datastax.driver.core.Cluster;
-import com.datastax.driver.core.Session;
-import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.marshal.TypeParser;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.exceptions.SyntaxException;
-import org.apache.cassandra.service.EmbeddedCassandraService;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.pig.ExecType;
-import org.apache.pig.PigServer;
-import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
-import org.apache.pig.impl.PigContext;
-import org.apache.pig.test.MiniCluster;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-
-public class PigTestBase extends SchemaLoader
-{
- protected static EmbeddedCassandraService cassandra;
- protected static Configuration conf;
- protected static MiniCluster cluster;
- protected static PigServer pig;
- protected static String defaultParameters= "init_address=localhost&rpc_port=9170&partitioner=org.apache.cassandra.dht.Murmur3Partitioner";
- protected static String nativeParameters = "&core_conns=2&max_conns=10&min_simult_reqs=3&max_simult_reqs=10&native_timeout=10000000" +
- "&native_read_timeout=10000000&send_buff_size=4096&receive_buff_size=4096&solinger=3" +
- "&tcp_nodelay=true&reuse_address=true&keep_alive=true&native_port=9042";
-
- static
- {
- System.setProperty("logback.configurationFile", "logback-test.xml");
- System.setProperty("cassandra.config", "cassandra_pig.yaml");
- }
-
- @AfterClass
- public static void oneTimeTearDown() throws Exception {
- cluster.shutDown();
- }
-
- @Before
- public void beforeTest() throws Exception {
- pig = new PigServer(new PigContext(ExecType.LOCAL, ConfigurationUtil.toProperties(conf)));
- PigContext.initializeImportList("org.apache.cassandra.hadoop.pig");
- }
-
- @After
- public void tearDown() throws Exception {
- pig.shutdown();
- }
-
- protected static Session getClient()
- {
- Cluster cluster = Cluster.builder().addContactPoints("localhost").withPort(9042).build();
- return cluster.connect();
- }
-
- protected static void startCassandra() throws IOException
- {
- Schema.instance.clear(); // Schema are now written on disk and will be reloaded
- cassandra = new EmbeddedCassandraService();
- cassandra.start();
- }
-
- protected static void startHadoopCluster()
- {
- cluster = MiniCluster.buildCluster();
- conf = cluster.getConfiguration();
- }
-
- protected AbstractType parseType(String type) throws IOException
- {
- try
- {
- return TypeParser.parse(type);
- }
- catch (ConfigurationException | SyntaxException e)
- {
- throw new IOException(e);
- }
- }
-
- protected static void executeCQLStatements(String[] statements)
- {
- Session client = getClient();
-
- for (String statement : statements)
- {
- System.out.println("Executing statement: " + statement);
- client.execute(statement);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/56cfc6ea/test/pig/org/apache/cassandra/pig/ThriftColumnFamilyDataTypeTest.java
----------------------------------------------------------------------
diff --git a/test/pig/org/apache/cassandra/pig/ThriftColumnFamilyDataTypeTest.java b/test/pig/org/apache/cassandra/pig/ThriftColumnFamilyDataTypeTest.java
deleted file mode 100644
index 273cdff..0000000
--- a/test/pig/org/apache/cassandra/pig/ThriftColumnFamilyDataTypeTest.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.cassandra.pig;
-
-import java.io.IOException;
-
-import org.apache.cassandra.db.marshal.TimeUUIDType;
-import org.apache.cassandra.db.marshal.UUIDType;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.utils.Hex;
-import org.apache.pig.data.DataByteArray;
-import org.apache.pig.data.Tuple;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import static junit.framework.Assert.assertEquals;
-
-public class ThriftColumnFamilyDataTypeTest extends PigTestBase
-{
- private static String[] statements = {
- "DROP KEYSPACE IF EXISTS thrift_ks",
- "CREATE KEYSPACE thrift_ks WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};",
- "USE thrift_ks;",
-
- "CREATE TABLE some_app (" +
- "key text PRIMARY KEY," +
- "col_ascii ascii," +
- "col_bigint bigint," +
- "col_blob blob," +
- "col_boolean boolean," +
- "col_decimal decimal," +
- "col_double double," +
- "col_float float," +
- "col_inet inet," +
- "col_int int," +
- "col_text text," +
- "col_timestamp timestamp," +
- "col_timeuuid timeuuid," +
- "col_uuid uuid," +
- "col_varint varint)" +
- " WITH COMPACT STORAGE;",
-
- "INSERT INTO some_app (key, col_ascii, col_bigint, col_blob, col_boolean, col_decimal, col_double, col_float," +
- "col_inet, col_int, col_text, col_timestamp, col_uuid, col_varint, col_timeuuid) " +
- "VALUES ('foo', 'ascii', 12345678, 0xDEADBEEF, false, 23.345, 2.7182818284590451, 23.45, '127.0.0.1', 23, 'hello', " +
- "'2011-02-03T04:05:00+0000', 550e8400-e29b-41d4-a716-446655440000, 12345, e23f450f-53a6-11e2-7f7f-7f7f7f7f7f7f);",
-
- "CREATE TABLE cc (key text, name text, value counter, PRIMARY KEY (key, name)) WITH COMPACT STORAGE",
-
- "UPDATE cc SET value = value + 3 WHERE key = 'chuck' AND name = 'kick'",
- };
-
- @BeforeClass
- public static void setup() throws IOException, ConfigurationException
- {
- startCassandra();
- executeCQLStatements(statements);
- startHadoopCluster();
- }
-
- @Test
- public void testCassandraStorageDataType() throws IOException
- {
- pig.registerQuery("rows = LOAD 'cql://thrift_ks/some_app?" + defaultParameters + "' USING CqlNativeStorage();");
- Tuple t = pig.openIterator("rows").next();
-
- // key
- assertEquals("foo", t.get(0));
-
- // col_ascii
- Object column = t.get(1);
- assertEquals("ascii", column);
-
- // col_bigint
- column = t.get(2);
- assertEquals(12345678L, column);
-
- // col_blob
- column = t.get(3);
- assertEquals(new DataByteArray(Hex.hexToBytes("DEADBEEF")), column);
-
- // col_boolean
- column = t.get(4);
- assertEquals(false, column);
-
- // col_decimal
- column = t.get(5);
- assertEquals("23.345", column);
-
- // col_double
- column = t.get(6);
- assertEquals(2.7182818284590451d, column);
-
- // col_float
- column = t.get(7);
- assertEquals(23.45f, column);
-
- // col_inet
- column = t.get(8);
- assertEquals("127.0.0.1", column);
-
- // col_int
- column = t.get(9);
- assertEquals(23, column);
-
- // col_text
- column = t.get(10);
- assertEquals("hello", column);
-
- // col_timestamp
- column = t.get(11);
- assertEquals(1296705900000L, column);
-
- // col_timeuuid
- column = t.get(12);
- assertEquals(new DataByteArray((TimeUUIDType.instance.fromString("e23f450f-53a6-11e2-7f7f-7f7f7f7f7f7f").array())), column);
-
- // col_uuid
- column = t.get(13);
- assertEquals(new DataByteArray((UUIDType.instance.fromString("550e8400-e29b-41d4-a716-446655440000").array())), column);
-
- // col_varint
- column = t.get(14);
- assertEquals(12345, column);
-
- pig.registerQuery("cc_rows = LOAD 'cql://thrift_ks/cc?" + defaultParameters + "' USING CqlNativeStorage();");
- t = pig.openIterator("cc_rows").next();
-
- assertEquals("chuck", t.get(0));
-
- assertEquals("kick", t.get(1));
- assertEquals(3L, t.get(2));
- }
-}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/56cfc6ea/test/pig/org/apache/cassandra/pig/ThriftColumnFamilyTest.java
----------------------------------------------------------------------
diff --git a/test/pig/org/apache/cassandra/pig/ThriftColumnFamilyTest.java b/test/pig/org/apache/cassandra/pig/ThriftColumnFamilyTest.java
deleted file mode 100644
index 93ac5be..0000000
--- a/test/pig/org/apache/cassandra/pig/ThriftColumnFamilyTest.java
+++ /dev/null
@@ -1,617 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.cassandra.pig;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Iterator;
-
-import com.datastax.driver.core.ResultSet;
-import com.datastax.driver.core.Row;
-import com.datastax.driver.core.Session;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DataByteArray;
-import org.apache.pig.data.Tuple;
-
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class ThriftColumnFamilyTest extends PigTestBase
-{
- private static String[] statements = {
- "DROP KEYSPACE IF EXISTS thrift_ks",
- "CREATE KEYSPACE thrift_ks WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};",
- "USE thrift_ks;",
-
- "CREATE TABLE some_app (" +
- "key text PRIMARY KEY," +
- "name text," +
- "vote_type text," +
- "rating int," +
- "score bigint," +
- "percent float," +
- "atomic_weight double," +
- "created timestamp)" +
- " WITH COMPACT STORAGE;",
-
- "CREATE INDEX ON some_app(name);",
-
- "INSERT INTO some_app (key, name, vote_type, rating, score, percent, atomic_weight, created) " +
- "VALUES ('foo', 'User Foo', 'like', 8, 125000, 85.0, 2.7182818284590451, 1335890877);",
-
- "INSERT INTO some_app (key, name, vote_type, rating, score, percent, atomic_weight, created) " +
- "VALUES ('bar', 'User Bar', 'like', 9, 15000, 35.0, 3.1415926535897931, 1335890877);",
-
- "INSERT INTO some_app (key, name, vote_type, rating, score, percent, atomic_weight, created) " +
- "VALUES ('baz', 'User Baz', 'dislike', 3, 512000, 95.3, 1.61803399, 1335890877);",
-
- "INSERT INTO some_app (key, name, vote_type, rating, score, percent, atomic_weight, created) " +
- "VALUES ('qux', 'User Qux', 'dislike', 2, 12000, 64.7, 0.660161815846869, 1335890877);",
-
- "CREATE TABLE copy_of_some_app (" +
- "key text PRIMARY KEY," +
- "name text," +
- "vote_type text," +
- "rating int," +
- "score bigint," +
- "percent float," +
- "atomic_weight double," +
- "created timestamp)" +
- " WITH COMPACT STORAGE;",
-
- "CREATE INDEX ON copy_of_some_app(name);",
-
- "CREATE TABLE u8 (" +
- "key text," +
- "column1 text," +
- "value blob," +
- "PRIMARY KEY (key, column1))" +
- " WITH COMPACT STORAGE",
-
- "INSERT INTO u8 (key, column1, value) VALUES ('foo', 'x', asciiAsBlob('Z'))",
-
- "CREATE TABLE bytes (" +
- "key blob," +
- "column1 text," +
- "value blob," +
- "PRIMARY KEY (key, column1))" +
- " WITH COMPACT STORAGE",
-
- "INSERT INTO bytes (key, column1, value) VALUES (asciiAsBlob('foo'), 'x', asciiAsBlob('Z'))",
-
- "CREATE TABLE cc (key text, name text, value counter, PRIMARY KEY (key, name)) WITH COMPACT STORAGE",
-
- "UPDATE cc SET value = value + 3 WHERE key = 'chuck' AND name = 'kick'",
- "UPDATE cc SET value = value + 1 WHERE key = 'chuck' AND name = 'fist'",
-
- "CREATE TABLE compo (" +
- "key text," +
- "column1 text," +
- "column2 text," +
- "value text," +
- "PRIMARY KEY (key, column1, column2))" +
- " WITH COMPACT STORAGE",
-
- "INSERT INTO compo (key, column1, column2, value) VALUES ('punch', 'bruce', 'lee', 'ouch');",
- "INSERT INTO compo (key, column1, column2, value) VALUES ('punch', 'bruce', 'bruce', 'hunh?');",
- "INSERT INTO compo (key, column1, column2, value) VALUES ('kick', 'bruce', 'lee', 'oww');",
- "INSERT INTO compo (key, column1, column2, value) VALUES ('kick', 'bruce', 'bruce', 'watch it, mate');",
-
- "CREATE TABLE compo_int (" +
- "key text," +
- "column1 bigint," +
- "column2 bigint," +
- "value text," +
- "PRIMARY KEY (key, column1, column2))" +
- " WITH COMPACT STORAGE",
-
- "INSERT INTO compo_int (key, column1, column2, value) VALUES ('clock', 1, 0, 'z');",
- "INSERT INTO compo_int (key, column1, column2, value) VALUES ('clock', 1, 30, 'zzzz');",
- "INSERT INTO compo_int (key, column1, column2, value) VALUES ('clock', 2, 30, 'daddy?');",
- "INSERT INTO compo_int (key, column1, column2, value) VALUES ('clock', 6, 30, 'coffee...');",
-
- "CREATE TABLE compo_int_copy (" +
- "key text," +
- "column1 bigint," +
- "column2 bigint," +
- "value text," +
- "PRIMARY KEY (key, column1, column2))" +
- " WITH COMPACT STORAGE",
-
- "CREATE TABLE compo_key (" +
- "key text," +
- "column1 bigint," +
- "column2 bigint," +
- "value text," +
- "PRIMARY KEY ((key, column1), column2))" +
- " WITH COMPACT STORAGE",
-
- "INSERT INTO compo_key (key, column1, column2, value) VALUES ('clock', 10, 1, 'z');",
- "INSERT INTO compo_key (key, column1, column2, value) VALUES ('clock', 20, 1, 'zzzz');",
- "INSERT INTO compo_key (key, column1, column2, value) VALUES ('clock', 30, 2, 'daddy?');",
- "INSERT INTO compo_key (key, column1, column2, value) VALUES ('clock', 40, 6, 'coffee...');",
-
- "CREATE TABLE compo_key_copy (" +
- "key text," +
- "column1 bigint," +
- "column2 bigint," +
- "value text," +
- "PRIMARY KEY ((key, column1), column2))" +
- " WITH COMPACT STORAGE",
- };
-
- private static String[] deleteCopyOfSomeAppTableData = {
- "use thrift_ks;",
- "DELETE FROM copy_of_some_app WHERE key = 'foo';",
- "DELETE FROM copy_of_some_app WHERE key = 'bar';",
- "DELETE FROM copy_of_some_app WHERE key = 'baz';",
- "DELETE FROM copy_of_some_app WHERE key = 'qux';",
- };
-
- @BeforeClass
- public static void setup() throws IOException, ConfigurationException
- {
- startCassandra();
- executeCQLStatements(statements);
- startHadoopCluster();
- }
-
- @Test
- public void testCqlNativeStorage() throws IOException
- {
- //regular thrift column families
- //input_cql=select * from "some_app" where token(key) > ? and token(key) <= ?
- cqlStorageTest("data = load 'cql://thrift_ks/some_app?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20%22some_app%22%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' using CqlNativeStorage();");
-
- //Test counter column family
- //input_cql=select * from "cc" where token(key) > ? and token(key) <= ?
- cqlStorageCounterTableTest("cc_data = load 'cql://thrift_ks/cc?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20%22cc%22%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' using CqlNativeStorage();");
-
- //Test composite column family
- //input_cql=select * from "compo" where token(key) > ? and token(key) <= ?
- cqlStorageCompositeTableTest("compo_data = load 'cql://thrift_ks/compo?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20%22compo%22%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' using CqlNativeStorage();");
- }
-
- private void cqlStorageTest(String initialQuery) throws IOException
- {
- pig.registerQuery(initialQuery);
-
- //(bar,3.141592653589793,1335890877,User Bar,35.0,9,15000,like)
- //(baz,1.61803399,1335890877,User Baz,95.3,3,512000,dislike)
- //(foo,2.718281828459045,1335890877,User Foo,85.0,8,125000,like)
- //(qux,0.660161815846869,1335890877,User Qux,64.7,2,12000,dislike)
-
- //{key: chararray,atomic_weight: double,created: long,name: chararray,percent: float,rating: int,score: long,vote_type: chararray}
- Iterator<Tuple> it = pig.openIterator("data");
- int count = 0;
- while (it.hasNext()) {
- count ++;
- Tuple t = it.next();
- if ("bar".equals(t.get(0)))
- {
- Assert.assertEquals(t.get(1), 3.141592653589793d);
- Assert.assertEquals(t.get(3), "User Bar");
- Assert.assertEquals(t.get(4), 35.0f);
- Assert.assertEquals(t.get(5), 9);
- Assert.assertEquals(t.get(6), 15000L);
- Assert.assertEquals(t.get(7), "like");
- }
- else if ("baz".equals(t.get(0)))
- {
- Assert.assertEquals(t.get(1), 1.61803399d);
- Assert.assertEquals(t.get(3), "User Baz");
- Assert.assertEquals(t.get(4), 95.3f);
- Assert.assertEquals(t.get(5), 3);
- Assert.assertEquals(t.get(6), 512000L);
- Assert.assertEquals(t.get(7), "dislike");
- }
- else if ("foo".equals(t.get(0)))
- {
- Assert.assertEquals(t.get(0), "foo");
- Assert.assertEquals(t.get(1), 2.718281828459045d);
- Assert.assertEquals(t.get(3), "User Foo");
- Assert.assertEquals(t.get(4), 85.0f);
- Assert.assertEquals(t.get(5), 8);
- Assert.assertEquals(t.get(6), 125000L);
- Assert.assertEquals(t.get(7), "like");
- }
- else if ("qux".equals(t.get(0)))
- {
- Assert.assertEquals(t.get(0), "qux");
- Assert.assertEquals(t.get(1), 0.660161815846869d);
- Assert.assertEquals(t.get(3), "User Qux");
- Assert.assertEquals(t.get(4), 64.7f);
- Assert.assertEquals(t.get(5), 2);
- Assert.assertEquals(t.get(6), 12000L);
- Assert.assertEquals(t.get(7), "dislike");
- }
- }
- Assert.assertEquals(count, 4);
- }
-
- private void cqlStorageCounterTableTest(String initialQuery) throws IOException
- {
- pig.registerQuery(initialQuery);
-
- //(chuck,fist,1)
- //(chuck,kick,3)
-
- // {key: chararray,column1: chararray,value: long}
- Iterator<Tuple> it = pig.openIterator("cc_data");
- int count = 0;
- while (it.hasNext()) {
- count ++;
- Tuple t = it.next();
- if ("chuck".equals(t.get(0)) && "fist".equals(t.get(1)))
- Assert.assertEquals(t.get(2), 1L);
- else if ("chuck".equals(t.get(0)) && "kick".equals(t.get(1)))
- Assert.assertEquals(t.get(2), 3L);
- }
- Assert.assertEquals(count, 2);
- }
-
- private void cqlStorageCompositeTableTest(String initialQuery) throws IOException
- {
- pig.registerQuery(initialQuery);
-
- //(kick,bruce,bruce,watch it, mate)
- //(kick,bruce,lee,oww)
- //(punch,bruce,bruce,hunh?)
- //(punch,bruce,lee,ouch)
-
- //{key: chararray,column1: chararray,column2: chararray,value: chararray}
- Iterator<Tuple> it = pig.openIterator("compo_data");
- int count = 0;
- while (it.hasNext()) {
- count ++;
- Tuple t = it.next();
- if ("kick".equals(t.get(0)) && "bruce".equals(t.get(1)) && "bruce".equals(t.get(2)))
- Assert.assertEquals(t.get(3), "watch it, mate");
- else if ("kick".equals(t.get(0)) && "bruce".equals(t.get(1)) && "lee".equals(t.get(2)))
- Assert.assertEquals(t.get(3), "oww");
- else if ("punch".equals(t.get(0)) && "bruce".equals(t.get(1)) && "bruce".equals(t.get(2)))
- Assert.assertEquals(t.get(3), "hunh?");
- else if ("punch".equals(t.get(0)) && "bruce".equals(t.get(1)) && "lee".equals(t.get(2)))
- Assert.assertEquals(t.get(3), "ouch");
- }
- Assert.assertEquals(count, 4);
- }
-
- @Test
- public void testCqlNativeStorageSchema() throws IOException
- {
- //results: (qux,(atomic_weight,0.660161815846869),(created,1335890877),(name,User Qux),(percent,64.7),
- //(rating,2),(score,12000),(vote_type,dislike))
- pig.registerQuery("rows = LOAD 'cql://thrift_ks/some_app?" + defaultParameters + "' USING CqlNativeStorage();");
-
- //schema: {key: chararray,atomic_weight: (name: chararray,value: double),created: (name: chararray,value: long),
- //name: (name: chararray,value: chararray),percent: (name: chararray,value: float),
- //rating: (name: chararray,value: int),score: (name: chararray,value: long),
- //vote_type: (name: chararray,value: chararray),columns: {(name: chararray,value: chararray)}}
- Iterator<Tuple> it = pig.openIterator("rows");
- if (it.hasNext()) {
- Tuple t = it.next();
- String rowKey = t.get(0).toString();
- if ("qux".equals(rowKey))
- {
- Tuple column = (Tuple) t.get(1);
- Assert.assertEquals(column.get(0), "atomic_weight");
- Assert.assertEquals(column.get(1), 0.660161815846869d);
- column = (Tuple) t.get(3);
- Assert.assertEquals(column.get(0), "name");
- Assert.assertEquals(column.get(1), "User Qux");
- column = (Tuple) t.get(4);
- Assert.assertEquals(column.get(0), "percent");
- Assert.assertEquals(column.get(1), 64.7f);
- column = (Tuple) t.get(5);
- Assert.assertEquals(column.get(0), "rating");
- Assert.assertEquals(column.get(1), 2);
- column = (Tuple) t.get(6);
- Assert.assertEquals(column.get(0), "score");
- Assert.assertEquals(column.get(1), 12000L);
- column = (Tuple) t.get(7);
- Assert.assertEquals(column.get(0), "vote_type");
- Assert.assertEquals(column.get(1), "dislike");
- }
- }
- }
-
- @Test
- public void testCqlNativeStorageFullCopy() throws IOException
- {
- pig.setBatchOn();
- pig.registerQuery("rows = LOAD 'cql://thrift_ks/some_app?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20some_app%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' USING CqlNativeStorage();");
- pig.registerQuery("records = FOREACH rows GENERATE TOTUPLE(TOTUPLE('key', key)),TOTUPLE(atomic_weight, created, name, percent, rating, score, vote_type);");
- //full copy
- pig.registerQuery("STORE records INTO 'cql://thrift_ks/copy_of_some_app?" + defaultParameters + nativeParameters + "&output_query=UPDATE+thrift_ks.copy_of_some_app+set+atomic_weight+%3D+%3F,+created+%3D+%3F,+name+%3D+%3F,+percent+%3D+%3F,+rating+%3D+%3F,+score+%3D+%3F,+vote_type+%3D+%3F' USING CqlNativeStorage();");
- pig.executeBatch();
- Assert.assertEquals("User Qux", getColumnValue("thrift_ks", "copy_of_some_app", "name", "qux", "UTF8Type"));
- Assert.assertEquals("dislike", getColumnValue("thrift_ks", "copy_of_some_app", "vote_type", "qux", "UTF8Type"));
- Assert.assertEquals("64.7", getColumnValue("thrift_ks", "copy_of_some_app", "percent", "qux", "FloatType"));
- }
-
- @Test
- public void testCqlNativeStorageSingleTupleCopy() throws IOException
- {
- executeCQLStatements(deleteCopyOfSomeAppTableData);
- pig.setBatchOn();
- pig.registerQuery("rows = LOAD 'cql://thrift_ks/some_app?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20some_app%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' USING CqlNativeStorage();");
- //single tuple
- pig.registerQuery("onecol = FOREACH rows GENERATE TOTUPLE(TOTUPLE('key', key)), TOTUPLE(percent);");
- pig.registerQuery("STORE onecol INTO 'cql://thrift_ks/copy_of_some_app?" + defaultParameters + nativeParameters + "&output_query=UPDATE+thrift_ks.copy_of_some_app+set+percent+%3D+%3F' USING CqlNativeStorage();");
- pig.executeBatch();
- String value = getColumnValue("thrift_ks", "copy_of_some_app", "name", "qux", "UTF8Type");
- if (value != null)
- Assert.fail();
- value = getColumnValue("thrift_ks", "copy_of_some_app", "vote_type", "qux", "UTF8Type");
- if (value != null)
- Assert.fail();
- Assert.assertEquals("64.7", getColumnValue("thrift_ks", "copy_of_some_app", "percent", "qux", "FloatType"));
- }
-
- @Test
- public void testCqlNativeStorageBagOnlyCopy() throws IOException
- {
- executeCQLStatements(deleteCopyOfSomeAppTableData);
- pig.setBatchOn();
- pig.registerQuery("rows = LOAD 'cql://thrift_ks/some_app?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20some_app%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' USING CqlNativeStorage();");
- //bag only
- pig.registerQuery("other = FOREACH rows GENERATE TOTUPLE(TOTUPLE('key', key)), TOTUPLE();");
- pig.registerQuery("STORE other INTO 'cql://thrift_ks/copy_of_some_app?" + defaultParameters + nativeParameters + "' USING CqlNativeStorage();");
- pig.executeBatch();
- String value = getColumnValue("thrift_ks", "copy_of_some_app", "name", "qux", "UTF8Type");
- if (value != null)
- Assert.fail();
- value = getColumnValue("thrift_ks", "copy_of_some_app", "vote_type", "qux", "UTF8Type");
- if (value != null)
- Assert.fail();
- value = getColumnValue("thrift_ks", "copy_of_some_app", "percent", "qux", "FloatType");
- if (value != null)
- Assert.fail();
- }
-
- @Test
- public void testCqlNativeStorageFilter() throws IOException
- {
- executeCQLStatements(deleteCopyOfSomeAppTableData);
- pig.setBatchOn();
- pig.registerQuery("rows = LOAD 'cql://thrift_ks/some_app?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20some_app%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' USING CqlNativeStorage();");
-
- //filter
- pig.registerQuery("likes = FILTER rows by vote_type eq 'like' and rating > 5;");
- pig.registerQuery("records = FOREACH likes GENERATE TOTUPLE(TOTUPLE('key', key)),TOTUPLE(atomic_weight, created, name, percent, rating, score, vote_type);");
- pig.registerQuery("STORE records INTO 'cql://thrift_ks/copy_of_some_app?" + defaultParameters + nativeParameters + "&output_query=UPDATE+thrift_ks.copy_of_some_app+set+atomic_weight+%3D+%3F,+created+%3D+%3F,+name+%3D+%3F,+percent+%3D+%3F,+rating+%3D+%3F,+score+%3D+%3F,+vote_type+%3D+%3F' USING CqlNativeStorage();");
- pig.executeBatch();
-
- Assert.assertEquals("like", getColumnValue("thrift_ks", "copy_of_some_app", "vote_type", "bar", "UTF8Type"));
- Assert.assertEquals("like", getColumnValue("thrift_ks", "copy_of_some_app", "vote_type", "foo", "UTF8Type"));
- String value = getColumnValue("thrift_ks", "copy_of_some_app", "vote_type", "qux", "UTF8Type");
- if (value != null)
- Assert.fail();
- value = getColumnValue("thrift_ks", "copy_of_some_app", "vote_type", "baz", "UTF8Type");
-
- if (value != null)
- Assert.fail();
-
- executeCQLStatements(deleteCopyOfSomeAppTableData);
- pig.setBatchOn();
- pig.registerQuery("rows = LOAD 'cql://thrift_ks/some_app?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20some_app%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' USING CqlNativeStorage();");
- pig.registerQuery("dislikes_extras = FILTER rows by vote_type eq 'dislike';");
- pig.registerQuery("dislikes_records = FOREACH dislikes_extras GENERATE TOTUPLE(TOTUPLE('key', key)),TOTUPLE(atomic_weight, created, name, percent, rating, score, vote_type);");
- pig.registerQuery("STORE dislikes_records INTO 'cql://thrift_ks/copy_of_some_app?" + defaultParameters + nativeParameters + "&output_query=UPDATE+thrift_ks.copy_of_some_app+set+atomic_weight+%3D+%3F,+created+%3D+%3F,+name+%3D+%3F,+percent+%3D+%3F,+rating+%3D+%3F,+score+%3D+%3F,+vote_type+%3D+%3F' USING CqlNativeStorage();");
- pig.executeBatch();
- Assert.assertEquals("dislike", getColumnValue("thrift_ks", "copy_of_some_app", "vote_type", "baz", "UTF8Type"));
- Assert.assertEquals("dislike", getColumnValue("thrift_ks", "copy_of_some_app", "vote_type", "qux", "UTF8Type"));
- value = getColumnValue("thrift_ks", "copy_of_some_app", "vote_type", "bar", "UTF8Type");
- if (value != null)
- Assert.fail();
- value = getColumnValue("thrift_ks", "copy_of_some_app", "vote_type", "foo", "UTF8Type");
- if (value != null)
- Assert.fail();
- }
-
- @Test
- public void testCqlNativeStorageJoin() throws IOException
- {
- //test key types with a join
- pig.registerQuery("U8 = load 'cql://thrift_ks/u8?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20u8%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' using CqlNativeStorage();");
- pig.registerQuery("Bytes = load 'cql://thrift_ks/bytes?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20bytes%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' using CqlNativeStorage();");
-
- //cast key to chararray
- pig.registerQuery("b = foreach Bytes generate (chararray)key, column1, value;");
-
- //key in Bytes is a bytearray, U8 chararray
- //(foo,{(x,Z)},foo,{(x,Z)})
- pig.registerQuery("a = join Bytes by key, U8 by key;");
- Iterator<Tuple> it = pig.openIterator("a");
- if (it.hasNext()) {
- Tuple t = it.next();
- Assert.assertEquals(t.get(0), new DataByteArray("foo".getBytes()));
- Assert.assertEquals(t.get(1), "x");
- Assert.assertEquals(t.get(2), new DataByteArray("Z".getBytes()));
- Assert.assertEquals(t.get(3), "foo");
- Assert.assertEquals(t.get(4), "x");
- Assert.assertEquals(t.get(5), new DataByteArray("Z".getBytes()));
- }
- //key should now be cast into a chararray
- //(foo,{(x,Z)},foo,{(x,Z)})
- pig.registerQuery("c = join b by (chararray)key, U8 by (chararray)key;");
- it = pig.openIterator("c");
- if (it.hasNext()) {
- Tuple t = it.next();
- Assert.assertEquals(t.get(0), "foo");
- Assert.assertEquals(t.get(1), "x");
- Assert.assertEquals(t.get(2), new DataByteArray("Z".getBytes()));
- Assert.assertEquals(t.get(3), "foo");
- Assert.assertEquals(t.get(4), "x");
- Assert.assertEquals(t.get(5), new DataByteArray("Z".getBytes()));
- }
- }
-
- @Test
- public void testCqlNativeStorageCounterCF() throws IOException
- {
- //Test counter column family support
- pig.registerQuery("CC = load 'cql://thrift_ks/cc?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20cc%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' using CqlNativeStorage();");
- pig.registerQuery("A = foreach CC generate key, name, value;");
- pig.registerQuery("B = GROUP A BY key;");
- pig.registerQuery("total_hits = foreach B generate group, SUM(A.value);");
- //(chuck,4)
- Tuple t = pig.openIterator("total_hits").next();
- Assert.assertEquals(t.get(0), "chuck");
- Assert.assertEquals(t.get(1), 4l);
- }
-
- @Test
- public void testCqlNativeStorageCompositeColumnCF() throws IOException
- {
- //Test CompositeType
- pig.registerQuery("compo = load 'cql://thrift_ks/compo?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20compo%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' using CqlNativeStorage();");
- pig.registerQuery("lee = filter compo by column1 == 'bruce' AND column2 == 'lee';");
-
- //(kick,(bruce,lee),oww)
- //(punch,(bruce,lee),ouch)
- Iterator<Tuple> it = pig.openIterator("lee");
- int count = 0;
- while (it.hasNext()) {
- count ++;
- Tuple t = it.next();
- Assert.assertEquals(t.get(1), "bruce");
- Assert.assertEquals(t.get(2), "lee");
- if ("kick".equals(t.get(0)))
- Assert.assertEquals(t.get(3), "oww");
- else
- Assert.assertEquals(t.get(3), "ouch");
- }
- Assert.assertEquals(count, 2);
- pig.registerQuery("night = load 'cql://thrift_ks/compo_int?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20compo_int%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' using CqlNativeStorage();");
- pig.registerQuery("night = foreach night generate (int)column1+(double)column2/60 as hour, value as noise;");
-
- //What happens at the darkest hour?
- pig.registerQuery("darkest = filter night by hour > 2 and hour < 5;");
-
- //(2.5,daddy?)
- it = pig.openIterator("darkest");
- if (it.hasNext()) {
- Tuple t = it.next();
- Assert.assertEquals(t.get(0), 2.5d);
- Assert.assertEquals(t.get(1), "daddy?");
- }
- pig.setBatchOn();
- pig.registerQuery("compo_int_rows = LOAD 'cql://thrift_ks/compo_int?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20compo_int%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' using CqlNativeStorage();");
- pig.registerQuery("STORE compo_int_rows INTO 'cql://thrift_ks/compo_int_copy?" + defaultParameters + nativeParameters + "&output_query=UPDATE+thrift_ks.compo_int_copy+set+column1+%3D+%3F,+column2+%3D+%3F,+value+%3D+%3F' using CqlNativeStorage();");
- pig.executeBatch();
- pig.registerQuery("compocopy_int_rows = LOAD 'cql://thrift_ks/compo_int_copy?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20compo_int_copy%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' using CqlNativeStorage();");
- //(clock,{((1,0),z),((1,30),zzzz),((2,30),daddy?),((6,30),coffee...)})
- it = pig.openIterator("compocopy_int_rows");
- count = 0;
- if (it.hasNext()) {
- Tuple t = it.next();
- Assert.assertEquals(t.get(0), "clock");
- DataBag columns = (DataBag) t.get(1);
- for (Tuple t1 : columns)
- {
- count++;
- Tuple inner = (Tuple) t1.get(0);
- if ((Long) inner.get(0) == 1L && (Long) inner.get(1) == 0L)
- Assert.assertEquals(t1.get(1), "z");
- else if ((Long) inner.get(0) == 1L && (Long) inner.get(1) == 30L)
- Assert.assertEquals(t1.get(1), "zzzz");
- else if ((Long) inner.get(0) == 2L && (Long) inner.get(1) == 30L)
- Assert.assertEquals(t1.get(1), "daddy?");
- else if ((Long) inner.get(0) == 6L && (Long) inner.get(1) == 30L)
- Assert.assertEquals(t1.get(1), "coffee...");
- }
- Assert.assertEquals(count, 4);
- }
- }
-
- @Test
- public void testCqlNativeStorageCompositeKeyCF() throws IOException
- {
- //Test CompositeKey
- pig.registerQuery("compokeys = load 'cql://thrift_ks/compo_key?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20compo_key%20where%20token(key,column1)%20%3E%20%3F%20and%20token(key,column1)%20%3C%3D%20%3F' using CqlNativeStorage();");
- pig.registerQuery("compokeys = filter compokeys by column1 == 40;");
- //(clock,40,6,coffee...)
- Iterator<Tuple> it = pig.openIterator("compokeys");
- if (it.hasNext()) {
- Tuple t = it.next();
- Assert.assertEquals(t.get(0), "clock");
- Assert.assertEquals(t.get(1), 40L);
- Assert.assertEquals(t.get(2), 6L);
- Assert.assertEquals(t.get(3), "coffee...");
- }
- pig.setBatchOn();
- pig.registerQuery("compo_key_rows = LOAD 'cql://thrift_ks/compo_key?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20compo_key%20where%20token(key,column1)%20%3E%20%3F%20and%20token(key,column1)%20%3C%3D%20%3F' using CqlNativeStorage();");
- pig.registerQuery("compo_key_rows = FOREACH compo_key_rows GENERATE TOTUPLE(TOTUPLE('key',key),TOTUPLE('column1',column1),TOTUPLE('column2',column2)),TOTUPLE(value);");
- pig.registerQuery("STORE compo_key_rows INTO 'cql://thrift_ks/compo_key_copy?" + defaultParameters + nativeParameters + "&output_query=UPDATE+thrift_ks.compo_key_copy+set+value+%3D+%3F' using CqlNativeStorage();");
- pig.executeBatch();
- pig.registerQuery("compo_key_copy_rows = LOAD 'cql://thrift_ks/compo_key_copy?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20compo_key_copy%20where%20token(key,column1)%20%3E%20%3F%20and%20token(key,column1)%20%3C%3D%20%3F' using CqlNativeStorage();");
- //((clock,10),{(1,z)})
- //((clock,20),{(1,zzzz)})
- //((clock,30),{(2,daddy?)})
- //((clock,40),{(6,coffee...)})
- it = pig.openIterator("compo_key_copy_rows");
- int count = 0;
- while (it.hasNext()) {
- Tuple t = it.next();
- count ++;
- if ("clock".equals(t.get(0)) && (Long) t.get(1) == 10L)
- {
- Assert.assertEquals(t.get(2), 1L);
- Assert.assertEquals(t.get(3), "z");
- }
- else if ("clock".equals(t.get(0)) && (Long) t.get(1) == 40L)
- {
- Assert.assertEquals(t.get(2), 6L);
- Assert.assertEquals(t.get(3), "coffee...");
- }
- else if ("clock".equals(t.get(0)) && (Long) t.get(1) == 20L)
- {
- Assert.assertEquals(t.get(2), 1L);
- Assert.assertEquals(t.get(3), "zzzz");
- }
- else if ("clock".equals(t.get(0)) && (Long) t.get(1) == 30L)
- {
- Assert.assertEquals(t.get(2), 2L);
- Assert.assertEquals(t.get(3), "daddy?");
- }
- }
- Assert.assertEquals(4, count);
- }
-
- private String getColumnValue(String ks, String cf, String colName, String key, String validator) throws IOException
- {
- Session client = getClient();
- client.execute("USE " + ks);
-
- String query = String.format("SELECT %s FROM %s WHERE key = '%s'", colName, cf, key);
-
- ResultSet rows = client.execute(query);
- Row row = rows.one();
-
- if (row == null || row.isNull(0))
- return null;
-
- return parseType(validator).getString(row.getBytesUnsafe(0));
- }
-}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/56cfc6ea/test/pig/org/apache/pig/test/MiniCluster.java
----------------------------------------------------------------------
diff --git a/test/pig/org/apache/pig/test/MiniCluster.java b/test/pig/org/apache/pig/test/MiniCluster.java
deleted file mode 100644
index 95acb84..0000000
--- a/test/pig/org/apache/pig/test/MiniCluster.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.pig.test;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.mapred.MiniMRCluster;
-
-public class MiniCluster extends MiniGenericCluster {
- private MiniMRCluster m_mr = null;
- public MiniCluster() {
- super();
- }
-
- @Override
- protected void setupMiniDfsAndMrClusters() {
- try {
- System.setProperty("hadoop.log.dir", "build/test/logs");
- final int dataNodes = 4; // There will be 4 data nodes
- final int taskTrackers = 4; // There will be 4 task tracker nodes
-
- // Create the configuration hadoop-site.xml file
- File conf_dir = new File("build/classes/");
- conf_dir.mkdirs();
- File conf_file = new File(conf_dir, "hadoop-site.xml");
-
- conf_file.delete();
-
- // Builds and starts the mini dfs and mapreduce clusters
- Configuration config = new Configuration();
- if (FBUtilities.isWindows())
- config.set("fs.file.impl", WindowsLocalFileSystem.class.getName());
- m_dfs = new MiniDFSCluster(config, dataNodes, true, null);
- m_fileSys = m_dfs.getFileSystem();
- m_mr = new MiniMRCluster(taskTrackers, m_fileSys.getUri().toString(), 1);
-
- // Write the necessary config info to hadoop-site.xml
- m_conf = m_mr.createJobConf();
- m_conf.setInt("mapred.submit.replication", 2);
- m_conf.set("dfs.datanode.address", "0.0.0.0:0");
- m_conf.set("dfs.datanode.http.address", "0.0.0.0:0");
- m_conf.set("mapred.map.max.attempts", "2");
- m_conf.set("mapred.reduce.max.attempts", "2");
- m_conf.set("pig.jobcontrol.sleep", "100");
- try (OutputStream os = new FileOutputStream(conf_file))
- {
- m_conf.writeXml(os);
- }
-
- // Set the system properties needed by Pig
- System.setProperty("cluster", m_conf.get("mapred.job.tracker"));
- System.setProperty("namenode", m_conf.get("fs.default.name"));
- System.setProperty("junit.hadoop.conf", conf_dir.getPath());
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- protected void shutdownMiniMrClusters() {
- if (m_mr != null)
- m_mr.shutdown();
- m_mr = null;
- }
-}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/56cfc6ea/test/pig/org/apache/pig/test/MiniGenericCluster.java
----------------------------------------------------------------------
diff --git a/test/pig/org/apache/pig/test/MiniGenericCluster.java b/test/pig/org/apache/pig/test/MiniGenericCluster.java
deleted file mode 100644
index ac3f5bc..0000000
--- a/test/pig/org/apache/pig/test/MiniGenericCluster.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.pig.test;
-
-import java.io.*;
-import java.util.Properties;
-
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
-
-/**
- * This class builds a single instance of itself with the Singleton
- * design pattern. While building the single instance, it sets up a
- * mini cluster that actually consists of a mini DFS cluster and a
- * mini MapReduce cluster on the local machine and also sets up the
- * environment for Pig to run on top of the mini cluster.
- *
- * This class is the base class for MiniCluster, which has slightly
- * difference among different versions of hadoop. MiniCluster implementation
- * is located in $PIG_HOME/shims.
- */
-abstract public class MiniGenericCluster {
- protected MiniDFSCluster m_dfs = null;
- protected FileSystem m_fileSys = null;
- protected Configuration m_conf = null;
-
- protected final static MiniCluster INSTANCE = new MiniCluster();
- protected static boolean isSetup = true;
-
- protected MiniGenericCluster() {
- setupMiniDfsAndMrClusters();
- }
-
- abstract protected void setupMiniDfsAndMrClusters();
-
- /**
- * Returns the single instance of class MiniClusterBuilder that
- * represents the resouces for a mini dfs cluster and a mini
- * mapreduce cluster.
- */
- public static MiniCluster buildCluster() {
- if(! isSetup){
- INSTANCE.setupMiniDfsAndMrClusters();
- isSetup = true;
- }
- return INSTANCE;
- }
-
- public void shutDown(){
- INSTANCE.shutdownMiniDfsAndMrClusters();
- }
-
- protected void finalize() {
- shutdownMiniDfsAndMrClusters();
- }
-
- protected void shutdownMiniDfsAndMrClusters() {
- isSetup = false;
- shutdownMiniDfsClusters();
- shutdownMiniMrClusters();
- }
-
- protected void shutdownMiniDfsClusters() {
- try {
- if (m_fileSys != null) { m_fileSys.close(); }
- } catch (IOException e) {
- e.printStackTrace();
- }
- if (m_dfs != null) { m_dfs.shutdown(); }
- m_fileSys = null;
- m_dfs = null;
- }
-
- abstract protected void shutdownMiniMrClusters();
-
- public Properties getProperties() {
- errorIfNotSetup();
- return ConfigurationUtil.toProperties(m_conf);
- }
-
- public Configuration getConfiguration() {
- return new Configuration(m_conf);
- }
-
- public void setProperty(String name, String value) {
- errorIfNotSetup();
- m_conf.set(name, value);
- }
-
- public FileSystem getFileSystem() {
- errorIfNotSetup();
- return m_fileSys;
- }
-
- /**
- * Throw RunTimeException if isSetup is false
- */
- private void errorIfNotSetup(){
- if(isSetup)
- return;
- String msg = "function called on MiniCluster that has been shutdown";
- throw new RuntimeException(msg);
- }
-}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/56cfc6ea/test/pig/org/apache/pig/test/WindowsLocalFileSystem.java
----------------------------------------------------------------------
diff --git a/test/pig/org/apache/pig/test/WindowsLocalFileSystem.java b/test/pig/org/apache/pig/test/WindowsLocalFileSystem.java
deleted file mode 100644
index 9193341..0000000
--- a/test/pig/org/apache/pig/test/WindowsLocalFileSystem.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.pig.test;
-
-import java.io.IOException;
-
-import org.apache.hadoop.fs.LocalFileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Workaround for https://issues.apache.org/jira/browse/HADOOP-7682 used to allow the Pig-tests to run on Cygwin on
- * a Windows box. This workaround was suggested by Joshua Caplan in the comments of HADOOP-7682.
- */
-public final class WindowsLocalFileSystem extends LocalFileSystem
-{
- private final Logger logger = LoggerFactory.getLogger(getClass());
-
- public WindowsLocalFileSystem()
- {
- logger.warn("Using {} instead of org.apache.hadoop.fs.LocalFileSystem to avoid the problem linked to HADOOP-7682. " +
- "IOException thrown when setting permissions will be swallowed.", getClass().getName());
- }
-
- @Override
- public boolean mkdirs(Path path, FsPermission permission) throws IOException
- {
- boolean result = super.mkdirs(path);
- setPermission(path, permission);
- return result;
- }
-
- @Override
- public void setPermission(Path p, FsPermission permission) throws IOException
- {
- try
- {
- super.setPermission(p, permission);
- }
- catch (IOException e)
- {
- // Just swallow the Exception as logging it produces too much output.
- }
- }
-}
[4/5] cassandra git commit: Remove Pig support
Posted by al...@apache.org.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/56cfc6ea/src/java/org/apache/cassandra/hadoop/pig/CqlNativeStorage.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/hadoop/pig/CqlNativeStorage.java b/src/java/org/apache/cassandra/hadoop/pig/CqlNativeStorage.java
deleted file mode 100644
index ba0a37d..0000000
--- a/src/java/org/apache/cassandra/hadoop/pig/CqlNativeStorage.java
+++ /dev/null
@@ -1,1082 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.hadoop.pig;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import java.io.Serializable;
-import java.io.UnsupportedEncodingException;
-import java.net.URLDecoder;
-import java.nio.ByteBuffer;
-import java.util.*;
-
-import com.datastax.driver.core.ColumnMetadata;
-import com.datastax.driver.core.Metadata;
-import com.datastax.driver.core.Row;
-import com.datastax.driver.core.Session;
-import com.datastax.driver.core.TableMetadata;
-import com.datastax.driver.core.exceptions.NoHostAvailableException;
-
-import org.apache.cassandra.db.marshal.*;
-import org.apache.cassandra.exceptions.AuthenticationException;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.exceptions.SyntaxException;
-import org.apache.cassandra.hadoop.ConfigHelper;
-import org.apache.cassandra.hadoop.HadoopCompat;
-import org.apache.cassandra.hadoop.cql3.CqlBulkOutputFormat;
-import org.apache.cassandra.hadoop.cql3.CqlBulkRecordWriter;
-import org.apache.cassandra.hadoop.cql3.CqlConfigHelper;
-import org.apache.cassandra.hadoop.cql3.CqlRecordReader;
-import org.apache.cassandra.serializers.CollectionSerializer;
-import org.apache.cassandra.utils.*;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapreduce.*;
-import org.apache.pig.*;
-import org.apache.pig.Expression.OpType;
-import org.apache.pig.ResourceSchema.ResourceFieldSchema;
-import org.apache.pig.backend.executionengine.ExecException;
-import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit;
-import org.apache.pig.data.*;
-import org.apache.pig.impl.util.UDFContext;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.yaml.snakeyaml.external.biz.base64Coder.Base64Coder;
-
-@Deprecated
-public class CqlNativeStorage extends LoadFunc implements StoreFuncInterface, LoadMetadata
-{
- protected String DEFAULT_INPUT_FORMAT;
- protected String DEFAULT_OUTPUT_FORMAT;
-
- protected String username;
- protected String password;
- protected String keyspace;
- protected String column_family;
- protected String loadSignature;
- protected String storeSignature;
-
- protected Configuration conf;
- protected String inputFormatClass;
- protected String outputFormatClass;
- protected int splitSize = 64 * 1024;
- protected String partitionerClass;
- protected boolean usePartitionFilter = false;
- protected String initHostAddress;
- protected String rpcPort;
- protected int nativeProtocolVersion = 1;
-
- private static final Logger logger = LoggerFactory.getLogger(CqlNativeStorage.class);
- private static String BULK_OUTPUT_FORMAT = "org.apache.cassandra.hadoop.cql3.CqlBulkOutputFormat";
- private int pageSize = 1000;
- private String columns;
- private String outputQuery;
- private String whereClause;
-
- private RecordReader<Long, Row> reader;
- private RecordWriter<Map<String, ByteBuffer>, List<ByteBuffer>> writer;
- private String nativePort;
- private String nativeCoreConnections;
- private String nativeMaxConnections;
- private String nativeMaxSimultReqs;
- private String nativeConnectionTimeout;
- private String nativeReadConnectionTimeout;
- private String nativeReceiveBufferSize;
- private String nativeSendBufferSize;
- private String nativeSolinger;
- private String nativeTcpNodelay;
- private String nativeReuseAddress;
- private String nativeKeepAlive;
- private String nativeAuthProvider;
- private String nativeSSLTruststorePath;
- private String nativeSSLKeystorePath;
- private String nativeSSLTruststorePassword;
- private String nativeSSLKeystorePassword;
- private String nativeSSLCipherSuites;
- private String inputCql;
-
- private boolean bulkOutputFormat = false;
- private String bulkCfSchema;
- private String bulkInsertStatement;
- private String bulkOutputLocation;
- private int bulkBuffSize = -1;
- private int bulkStreamThrottle = -1;
- private int bulkMaxFailedHosts = -1;
- private boolean bulkDeleteSourceOnSuccess = true;
- private String bulkTableAlias;
-
- public CqlNativeStorage()
- {
- this(1000);
- }
-
- /** @param pageSize limit number of CQL rows to fetch in a thrift request */
- public CqlNativeStorage(int pageSize)
- {
- super();
- this.pageSize = pageSize;
- DEFAULT_INPUT_FORMAT = "org.apache.cassandra.hadoop.cql3.CqlInputFormat";
- DEFAULT_OUTPUT_FORMAT = "org.apache.cassandra.hadoop.cql3.CqlOutputFormat";
- }
-
- public void prepareToRead(RecordReader reader, PigSplit split)
- {
- this.reader = reader;
- if (reader instanceof CqlRecordReader) {
- nativeProtocolVersion = ((CqlRecordReader) reader).getNativeProtocolVersion();
- }
- }
-
- public void prepareToWrite(RecordWriter writer)
- {
- this.writer = writer;
- }
-
- /** get next row */
- public Tuple getNext() throws IOException
- {
- try
- {
- // load the next pair
- if (!reader.nextKeyValue())
- return null;
-
- TableInfo tableMetadata = getCfInfo(loadSignature);
- Row row = reader.getCurrentValue();
- Tuple tuple = TupleFactory.getInstance().newTuple(tableMetadata.getColumns().size());
- Iterator<ColumnInfo> itera = tableMetadata.getColumns().iterator();
- int i = 0;
- while (itera.hasNext())
- {
- ColumnInfo cdef = itera.next();
- ByteBuffer columnValue = row.getBytesUnsafe(cdef.getName());
- if (columnValue != null)
- {
- AbstractType<?> validator = getValidatorMap(tableMetadata).get(ByteBufferUtil.bytes(cdef.getName()));
- setTupleValue(tuple, i, cqlColumnToObj(ByteBufferUtil.bytes(cdef.getName()), columnValue,
- tableMetadata), validator);
- }
- else
- tuple.set(i, null);
- i++;
- }
- return tuple;
- }
- catch (InterruptedException e)
- {
- throw new IOException(e.getMessage());
- }
- }
-
- /** convert a cql column to an object */
- private Object cqlColumnToObj(ByteBuffer name, ByteBuffer columnValue, TableInfo cfDef) throws IOException
- {
- // standard
- Map<ByteBuffer,AbstractType> validators = getValidatorMap(cfDef);
- return StorageHelper.cassandraToObj(validators.get(name), columnValue, nativeProtocolVersion);
- }
-
- /** set the value to the position of the tuple */
- private void setTupleValue(Tuple tuple, int position, Object value, AbstractType<?> validator) throws ExecException
- {
- if (validator instanceof CollectionType)
- setCollectionTupleValues(tuple, position, value, validator);
- else
- StorageHelper.setTupleValue(tuple, position, value);
- }
-
- /** set the values of set/list at and after the position of the tuple */
- private void setCollectionTupleValues(Tuple tuple, int position, Object value, AbstractType<?> validator) throws ExecException
- {
- if (validator instanceof MapType)
- {
- setMapTupleValues(tuple, position, value, validator);
- return;
- }
- AbstractType elementValidator;
- if (validator instanceof SetType)
- elementValidator = ((SetType<?>) validator).getElementsType();
- else if (validator instanceof ListType)
- elementValidator = ((ListType<?>) validator).getElementsType();
- else
- return;
-
- int i = 0;
- Tuple innerTuple = TupleFactory.getInstance().newTuple(((Collection<?>) value).size());
- for (Object entry : (Collection<?>) value)
- {
- setTupleValue(innerTuple, i, cassandraToPigData(entry, elementValidator), elementValidator);
- i++;
- }
- tuple.set(position, innerTuple);
- }
-
- /** set the values of set/list at and after the position of the tuple */
- private void setMapTupleValues(Tuple tuple, int position, Object value, AbstractType<?> validator) throws ExecException
- {
- AbstractType<?> keyValidator = ((MapType<?, ?>) validator).getKeysType();
- AbstractType<?> valueValidator = ((MapType<?, ?>) validator).getValuesType();
-
- int i = 0;
- Tuple innerTuple = TupleFactory.getInstance().newTuple(((Map<?,?>) value).size());
- for(Map.Entry<?,?> entry : ((Map<Object, Object>)value).entrySet())
- {
- Tuple mapEntryTuple = TupleFactory.getInstance().newTuple(2);
- setTupleValue(mapEntryTuple, 0, cassandraToPigData(entry.getKey(), keyValidator), keyValidator);
- setTupleValue(mapEntryTuple, 1, cassandraToPigData(entry.getValue(), valueValidator), valueValidator);
- innerTuple.set(i, mapEntryTuple);
- i++;
- }
- tuple.set(position, innerTuple);
- }
-
- private Object cassandraToPigData(Object obj, AbstractType validator)
- {
- if (validator instanceof DecimalType || validator instanceof InetAddressType)
- return validator.getString(validator.decompose(obj));
- return obj;
- }
-
- /** get the columnfamily definition for the signature */
- protected TableInfo getCfInfo(String signature) throws IOException
- {
- UDFContext context = UDFContext.getUDFContext();
- Properties property = context.getUDFProperties(CqlNativeStorage.class);
- TableInfo cfInfo;
- try
- {
- cfInfo = cfdefFromString(property.getProperty(signature));
- }
- catch (ClassNotFoundException e)
- {
- throw new IOException(e);
- }
- return cfInfo;
- }
-
- /** return the CfInfo for the column family */
- protected TableMetadata getCfInfo(Session client)
- throws NoHostAvailableException,
- AuthenticationException,
- IllegalStateException
- {
- // get CF meta data
- return client.getCluster().getMetadata().getKeyspace(Metadata.quote(keyspace)).getTable(Metadata.quote(column_family));
- }
-
- /** convert key tuple to key map */
- private Map<String, ByteBuffer> tupleToKeyMap(Tuple t) throws IOException
- {
- Map<String, ByteBuffer> keys = new HashMap<String, ByteBuffer>();
- for (int i = 0; i < t.size(); i++)
- {
- if (t.getType(i) != DataType.TUPLE)
- throw new IOException("keys was not a tuple");
- Tuple inner = (Tuple) t.get(i);
- if (inner.size() != 2)
- throw new IOException("Keys were not in name and value pairs");
- Object name = inner.get(0);
- if (name == null)
- throw new IOException("Key name was empty");
- keys.put(name.toString(), objToBB(inner.get(1)));
- }
- return keys;
- }
-
- /** convert object to ByteBuffer */
- protected ByteBuffer objToBB(Object o)
- {
- if (o == null)
- return nullToBB();
- if (o instanceof java.lang.String)
- return ByteBuffer.wrap(new DataByteArray((String)o).get());
- if (o instanceof Integer)
- return Int32Type.instance.decompose((Integer)o);
- if (o instanceof Long)
- return LongType.instance.decompose((Long)o);
- if (o instanceof Float)
- return FloatType.instance.decompose((Float)o);
- if (o instanceof Double)
- return DoubleType.instance.decompose((Double)o);
- if (o instanceof UUID)
- return ByteBuffer.wrap(UUIDGen.decompose((UUID) o));
- if(o instanceof Tuple) {
- List<Object> objects = ((Tuple)o).getAll();
- //collections
- if (objects.size() > 0 && objects.get(0) instanceof String)
- {
- String collectionType = (String) objects.get(0);
- if ("set".equalsIgnoreCase(collectionType) ||
- "list".equalsIgnoreCase(collectionType))
- return objToListOrSetBB(objects.subList(1, objects.size()));
- else if ("map".equalsIgnoreCase(collectionType))
- return objToMapBB(objects.subList(1, objects.size()));
-
- }
- return objToCompositeBB(objects);
- }
-
- return ByteBuffer.wrap(((DataByteArray) o).get());
- }
-
- private ByteBuffer objToListOrSetBB(List<Object> objects)
- {
- List<ByteBuffer> serialized = new ArrayList<ByteBuffer>(objects.size());
- for(Object sub : objects)
- {
- ByteBuffer buffer = objToBB(sub);
- serialized.add(buffer);
- }
- return CollectionSerializer.pack(serialized, objects.size(), 3);
- }
-
- private ByteBuffer objToMapBB(List<Object> objects)
- {
- List<ByteBuffer> serialized = new ArrayList<ByteBuffer>(objects.size() * 2);
- for(Object sub : objects)
- {
- List<Object> keyValue = ((Tuple)sub).getAll();
- for (Object entry: keyValue)
- {
- ByteBuffer buffer = objToBB(entry);
- serialized.add(buffer);
- }
- }
- return CollectionSerializer.pack(serialized, objects.size(), 3);
- }
-
- private ByteBuffer objToCompositeBB(List<Object> objects)
- {
- List<ByteBuffer> serialized = new ArrayList<ByteBuffer>(objects.size());
- int totalLength = 0;
- for(Object sub : objects)
- {
- ByteBuffer buffer = objToBB(sub);
- serialized.add(buffer);
- totalLength += 2 + buffer.remaining() + 1;
- }
- ByteBuffer out = ByteBuffer.allocate(totalLength);
- for (ByteBuffer bb : serialized)
- {
- int length = bb.remaining();
- out.put((byte) ((length >> 8) & 0xFF));
- out.put((byte) (length & 0xFF));
- out.put(bb);
- out.put((byte) 0);
- }
- out.flip();
- return out;
- }
-
- /** send CQL query request using data from tuple */
- private void cqlQueryFromTuple(Map<String, ByteBuffer> key, Tuple t, int offset) throws IOException
- {
- for (int i = offset; i < t.size(); i++)
- {
- if (t.getType(i) != DataType.TUPLE)
- throw new IOException("Output type was not a tuple");
-
- Tuple inner = (Tuple) t.get(i);
- if (inner.size() > 0)
- {
- List<ByteBuffer> bindedVariables = bindedVariablesFromTuple(inner);
- if (bindedVariables.size() <= 0)
- throw new IOException("Missing binded variables");
- sendCqlQuery(key, bindedVariables);
- }
- }
- }
-
- /** compose a list of binded variables */
- private List<ByteBuffer> bindedVariablesFromTuple(Tuple t) throws IOException
- {
- List<ByteBuffer> variables = new ArrayList<ByteBuffer>();
- for (int i = 0; i < t.size(); i++)
- variables.add(objToBB(t.get(i)));
- return variables;
- }
-
- /** writer write the data by executing CQL query */
- private void sendCqlQuery(Map<String, ByteBuffer> key, List<ByteBuffer> bindedVariables) throws IOException
- {
- try
- {
- writer.write(key, bindedVariables);
- }
- catch (InterruptedException e)
- {
- throw new IOException(e);
- }
- }
-
- /** get the validators */
- protected Map<ByteBuffer, AbstractType> getValidatorMap(TableInfo cfDef) throws IOException
- {
- Map<ByteBuffer, AbstractType> validators = new HashMap<>();
- for (ColumnInfo cd : cfDef.getColumns())
- {
- if (cd.getTypeName() != null)
- {
- try
- {
- AbstractType validator = TypeParser.parseCqlName(cd.getTypeName());
- if (validator instanceof CounterColumnType)
- validator = LongType.instance;
- validators.put(ByteBufferUtil.bytes(cd.getName()), validator);
- }
- catch (ConfigurationException | SyntaxException e)
- {
- throw new IOException(e);
- }
- }
- }
- return validators;
- }
-
- /** schema: (value, value, value) where keys are in the front. */
- public ResourceSchema getSchema(String location, Job job) throws IOException
- {
- setLocation(location, job);
- TableInfo cfInfo = getCfInfo(loadSignature);
- // top-level schema, no type
- ResourceSchema schema = new ResourceSchema();
-
- // get default validators
- Map<ByteBuffer, AbstractType> validators = getValidatorMap(cfInfo);
-
- // will contain all fields for this schema
- List<ResourceFieldSchema> allSchemaFields = new ArrayList<ResourceFieldSchema>();
-
- for (ColumnInfo cdef : cfInfo.getColumns())
- {
- ResourceFieldSchema valSchema = new ResourceFieldSchema();
- AbstractType<?> validator = validators.get(ByteBufferUtil.bytes(cdef.getName()));
- valSchema.setName(cdef.getName());
- valSchema.setType(StorageHelper.getPigType(validator));
- allSchemaFields.add(valSchema);
- }
-
- // top level schema contains everything
- schema.setFields(allSchemaFields.toArray(new ResourceFieldSchema[allSchemaFields.size()]));
- return schema;
- }
-
- public void setPartitionFilter(Expression partitionFilter) throws IOException
- {
- UDFContext context = UDFContext.getUDFContext();
- Properties property = context.getUDFProperties(CqlNativeStorage.class);
- property.setProperty(StorageHelper.PARTITION_FILTER_SIGNATURE, partitionFilterToWhereClauseString(partitionFilter));
- }
-
- /**
- * Return cql where clauses for the corresponding partition filter. Make sure the data format matches
- * Only support the following Pig data types: int, long, float, double, boolean and chararray
- * */
- private String partitionFilterToWhereClauseString(Expression expression) throws IOException
- {
- Expression.BinaryExpression be = (Expression.BinaryExpression) expression;
- OpType op = expression.getOpType();
- String opString = op.toString();
- switch (op)
- {
- case OP_EQ:
- opString = " = ";
- case OP_GE:
- case OP_GT:
- case OP_LE:
- case OP_LT:
- String name = be.getLhs().toString();
- String value = be.getRhs().toString();
- return String.format("%s %s %s", name, opString, value);
- case OP_AND:
- return String.format("%s AND %s", partitionFilterToWhereClauseString(be.getLhs()), partitionFilterToWhereClauseString(be.getRhs()));
- default:
- throw new IOException("Unsupported expression type: " + opString);
- }
- }
-
- /** retrieve where clause for partition filter */
- private String getWhereClauseForPartitionFilter()
- {
- UDFContext context = UDFContext.getUDFContext();
- Properties property = context.getUDFProperties(CqlNativeStorage.class);
- return property.getProperty(StorageHelper.PARTITION_FILTER_SIGNATURE);
- }
-
- /**
- * output: (((name, value), (name, value)), (value ... value), (value...value))
- * bulk output: ((value ... value), (value...value))
- *
- * */
- public void putNext(Tuple t) throws IOException
- {
- if (t.size() < 1)
- {
- // simply nothing here, we can't even delete without a key
- logger.warn("Empty output skipped, filter empty tuples to suppress this warning");
- return;
- }
-
- if (t.getType(0) != DataType.TUPLE)
- throw new IOException("First argument in output must be a tuple");
-
- if (!bulkOutputFormat && t.getType(1) != DataType.TUPLE)
- throw new IOException("Second argument in output must be a tuple");
-
- if (bulkOutputFormat)
- {
- cqlQueryFromTuple(null, t, 0);
- }
- else
- {
- Map<String, ByteBuffer> key = tupleToKeyMap((Tuple)t.get(0));
- cqlQueryFromTuple(key, t, 1);
- }
- }
-
- /** set read configuration settings */
- public void setLocation(String location, Job job) throws IOException
- {
- conf = job.getConfiguration();
- setLocationFromUri(location);
-
- if (username != null && password != null)
- {
- ConfigHelper.setInputKeyspaceUserNameAndPassword(conf, username, password);
- CqlConfigHelper.setUserNameAndPassword(conf, username, password);
- }
- if (splitSize > 0)
- ConfigHelper.setInputSplitSize(conf, splitSize);
- if (partitionerClass!= null)
- ConfigHelper.setInputPartitioner(conf, partitionerClass);
- if (initHostAddress != null)
- ConfigHelper.setInputInitialAddress(conf, initHostAddress);
- if (rpcPort != null)
- ConfigHelper.setInputRpcPort(conf, rpcPort);
- if (nativePort != null)
- CqlConfigHelper.setInputNativePort(conf, nativePort);
- if (nativeCoreConnections != null)
- CqlConfigHelper.setInputCoreConnections(conf, nativeCoreConnections);
- if (nativeMaxConnections != null)
- CqlConfigHelper.setInputMaxConnections(conf, nativeMaxConnections);
- if (nativeMaxSimultReqs != null)
- CqlConfigHelper.setInputMaxSimultReqPerConnections(conf, nativeMaxSimultReqs);
- if (nativeConnectionTimeout != null)
- CqlConfigHelper.setInputNativeConnectionTimeout(conf, nativeConnectionTimeout);
- if (nativeReadConnectionTimeout != null)
- CqlConfigHelper.setInputNativeReadConnectionTimeout(conf, nativeReadConnectionTimeout);
- if (nativeReceiveBufferSize != null)
- CqlConfigHelper.setInputNativeReceiveBufferSize(conf, nativeReceiveBufferSize);
- if (nativeSendBufferSize != null)
- CqlConfigHelper.setInputNativeSendBufferSize(conf, nativeSendBufferSize);
- if (nativeSolinger != null)
- CqlConfigHelper.setInputNativeSolinger(conf, nativeSolinger);
- if (nativeTcpNodelay != null)
- CqlConfigHelper.setInputNativeTcpNodelay(conf, nativeTcpNodelay);
- if (nativeReuseAddress != null)
- CqlConfigHelper.setInputNativeReuseAddress(conf, nativeReuseAddress);
- if (nativeKeepAlive != null)
- CqlConfigHelper.setInputNativeKeepAlive(conf, nativeKeepAlive);
- if (nativeAuthProvider != null)
- CqlConfigHelper.setInputNativeAuthProvider(conf, nativeAuthProvider);
- if (nativeSSLTruststorePath != null)
- CqlConfigHelper.setInputNativeSSLTruststorePath(conf, nativeSSLTruststorePath);
- if (nativeSSLKeystorePath != null)
- CqlConfigHelper.setInputNativeSSLKeystorePath(conf, nativeSSLKeystorePath);
- if (nativeSSLTruststorePassword != null)
- CqlConfigHelper.setInputNativeSSLTruststorePassword(conf, nativeSSLTruststorePassword);
- if (nativeSSLKeystorePassword != null)
- CqlConfigHelper.setInputNativeSSLKeystorePassword(conf, nativeSSLKeystorePassword);
- if (nativeSSLCipherSuites != null)
- CqlConfigHelper.setInputNativeSSLCipherSuites(conf, nativeSSLCipherSuites);
-
- ConfigHelper.setInputColumnFamily(conf, keyspace, column_family);
- setConnectionInformation();
-
- CqlConfigHelper.setInputCQLPageRowSize(conf, String.valueOf(pageSize));
- if (inputCql != null)
- CqlConfigHelper.setInputCql(conf, inputCql);
- if (columns != null)
- CqlConfigHelper.setInputColumns(conf, columns);
- if (whereClause != null)
- CqlConfigHelper.setInputWhereClauses(conf, whereClause);
-
- String whereClauseForPartitionFilter = getWhereClauseForPartitionFilter();
- String wc = whereClause != null && !whereClause.trim().isEmpty()
- ? whereClauseForPartitionFilter == null ? whereClause: String.format("%s AND %s", whereClause.trim(), whereClauseForPartitionFilter)
- : whereClauseForPartitionFilter;
-
- if (wc != null)
- {
- logger.trace("where clause: {}", wc);
- CqlConfigHelper.setInputWhereClauses(conf, wc);
- }
- if (System.getenv(StorageHelper.PIG_INPUT_SPLIT_SIZE) != null)
- {
- try
- {
- ConfigHelper.setInputSplitSize(conf, Integer.parseInt(System.getenv(StorageHelper.PIG_INPUT_SPLIT_SIZE)));
- }
- catch (NumberFormatException e)
- {
- throw new IOException("PIG_INPUT_SPLIT_SIZE is not a number", e);
- }
- }
-
- if (ConfigHelper.getInputInitialAddress(conf) == null)
- throw new IOException("PIG_INPUT_INITIAL_ADDRESS or PIG_INITIAL_ADDRESS environment variable not set");
- if (ConfigHelper.getInputPartitioner(conf) == null)
- throw new IOException("PIG_INPUT_PARTITIONER or PIG_PARTITIONER environment variable not set");
- if (loadSignature == null)
- loadSignature = location;
-
- initSchema(loadSignature);
- }
-
- /** set store configuration settings */
- public void setStoreLocation(String location, Job job) throws IOException
- {
- conf = HadoopCompat.getConfiguration(job);
- setLocationFromUri(location);
-
- if (username != null && password != null)
- ConfigHelper.setOutputKeyspaceUserNameAndPassword(conf, username, password);
- if (splitSize > 0)
- ConfigHelper.setInputSplitSize(conf, splitSize);
- if (partitionerClass!= null)
- ConfigHelper.setOutputPartitioner(conf, partitionerClass);
- if (rpcPort != null)
- {
- ConfigHelper.setOutputRpcPort(conf, rpcPort);
- ConfigHelper.setInputRpcPort(conf, rpcPort);
- }
- if (initHostAddress != null)
- {
- ConfigHelper.setOutputInitialAddress(conf, initHostAddress);
- ConfigHelper.setInputInitialAddress(conf, initHostAddress);
- }
-
- ConfigHelper.setOutputColumnFamily(conf, keyspace, column_family);
- CqlConfigHelper.setOutputCql(conf, outputQuery);
-
- if (bulkOutputFormat)
- {
- DEFAULT_OUTPUT_FORMAT = BULK_OUTPUT_FORMAT;
- if (bulkCfSchema != null)
- CqlBulkOutputFormat.setTableSchema(conf, column_family, bulkCfSchema);
- else
- throw new IOException("bulk_cf_schema is missing in input url parameter");
- if (bulkInsertStatement != null)
- CqlBulkOutputFormat.setTableInsertStatement(conf, column_family, bulkInsertStatement);
- else
- throw new IOException("bulk_insert_statement is missing in input url parameter");
- if (bulkTableAlias != null)
- CqlBulkOutputFormat.setTableAlias(conf, bulkTableAlias, column_family);
- CqlBulkOutputFormat.setDeleteSourceOnSuccess(conf, bulkDeleteSourceOnSuccess);
- if (bulkOutputLocation != null)
- conf.set(CqlBulkRecordWriter.OUTPUT_LOCATION, bulkOutputLocation);
- if (bulkBuffSize > 0)
- conf.set(CqlBulkRecordWriter.BUFFER_SIZE_IN_MB, String.valueOf(bulkBuffSize));
- if (bulkStreamThrottle > 0)
- conf.set(CqlBulkRecordWriter.STREAM_THROTTLE_MBITS, String.valueOf(bulkStreamThrottle));
- if (bulkMaxFailedHosts > 0)
- conf.set(CqlBulkRecordWriter.MAX_FAILED_HOSTS, String.valueOf(bulkMaxFailedHosts));
- if (partitionerClass!= null)
- ConfigHelper.setInputPartitioner(conf, partitionerClass);
- }
-
- setConnectionInformation();
-
- if (ConfigHelper.getOutputRpcPort(conf) == 0)
- throw new IOException("PIG_OUTPUT_RPC_PORT or PIG_RPC_PORT environment variable not set");
- if (ConfigHelper.getOutputInitialAddress(conf) == null)
- throw new IOException("PIG_OUTPUT_INITIAL_ADDRESS or PIG_INITIAL_ADDRESS environment variable not set");
- if (ConfigHelper.getOutputPartitioner(conf) == null)
- throw new IOException("PIG_OUTPUT_PARTITIONER or PIG_PARTITIONER environment variable not set");
-
- initSchema(storeSignature);
- }
-
- /** Methods to get the column family schema from Cassandra */
- protected void initSchema(String signature) throws IOException
- {
- Properties properties = UDFContext.getUDFContext().getUDFProperties(CqlNativeStorage.class);
-
- // Only get the schema if we haven't already gotten it
- if (!properties.containsKey(signature))
- {
- try (Session client = CqlConfigHelper.getInputCluster(ConfigHelper.getInputInitialAddress(conf), conf).connect())
- {
- client.execute("USE " + keyspace);
-
- // compose the CfDef for the columfamily
- TableMetadata cfInfo = getCfInfo(client);
-
- if (cfInfo != null)
- {
- properties.setProperty(signature, cfdefToString(cfInfo));
- }
- else
- throw new IOException(String.format("Table '%s' not found in keyspace '%s'",
- column_family,
- keyspace));
- }
- catch (Exception e)
- {
- throw new IOException(e);
- }
- }
- }
-
-
- /** convert CfDef to string */
- protected static String cfdefToString(TableMetadata cfDef) throws IOException
- {
- TableInfo tableInfo = new TableInfo(cfDef);
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- try (ObjectOutputStream oos = new ObjectOutputStream( baos ))
- {
- oos.writeObject(tableInfo);
- }
-
- return new String( Base64Coder.encode(baos.toByteArray()) );
- }
-
- /** convert string back to CfDef */
- protected static TableInfo cfdefFromString(String st) throws IOException, ClassNotFoundException
- {
- byte [] data = Base64Coder.decode( st );
- try (ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(data)))
- {
- Object o = ois.readObject();
- return (TableInfo)o;
- }
- }
-
- /** decompose the query to store the parameters in a map */
- public static Map<String, String> getQueryMap(String query) throws UnsupportedEncodingException
- {
- String[] params = query.split("&");
- Map<String, String> map = new HashMap<String, String>(params.length);
- for (String param : params)
- {
- String[] keyValue = param.split("=");
- map.put(keyValue[0], URLDecoder.decode(keyValue[1], "UTF-8"));
- }
- return map;
- }
-
- private void setLocationFromUri(String location) throws IOException
- {
- try
- {
- if (!location.startsWith("cql://"))
- throw new Exception("Bad scheme: " + location);
-
- String[] urlParts = location.split("\\?");
- if (urlParts.length > 1)
- {
- Map<String, String> urlQuery = getQueryMap(urlParts[1]);
-
- // each page row size
- if (urlQuery.containsKey("page_size"))
- pageSize = Integer.parseInt(urlQuery.get("page_size"));
-
- // output prepared statement
- if (urlQuery.containsKey("output_query"))
- outputQuery = urlQuery.get("output_query");
-
- if (urlQuery.containsKey("bulk_output_format"))
- bulkOutputFormat = Boolean.valueOf(urlQuery.get("bulk_output_format"));
- if (urlQuery.containsKey("bulk_cf_schema"))
- bulkCfSchema = urlQuery.get("bulk_cf_schema");
- if (urlQuery.containsKey("bulk_insert_statement"))
- bulkInsertStatement = urlQuery.get("bulk_insert_statement");
- if (urlQuery.containsKey("bulk_output_location"))
- bulkOutputLocation = urlQuery.get("bulk_output_location");
- if (urlQuery.containsKey("bulk_buff_size"))
- bulkBuffSize = Integer.valueOf(urlQuery.get("bulk_buff_size"));
- if (urlQuery.containsKey("bulk_stream_throttle"))
- bulkStreamThrottle = Integer.valueOf(urlQuery.get("bulk_stream_throttle"));
- if (urlQuery.containsKey("bulk_max_failed_hosts"))
- bulkMaxFailedHosts = Integer.valueOf(urlQuery.get("bulk_max_failed_hosts"));
- if (urlQuery.containsKey("bulk_delete_source"))
- bulkDeleteSourceOnSuccess = Boolean.parseBoolean(urlQuery.get("bulk_delete_source"));
- if (urlQuery.containsKey("bulk_table_alias"))
- bulkTableAlias = urlQuery.get("bulk_table_alias");
-
- //split size
- if (urlQuery.containsKey("split_size"))
- splitSize = Integer.parseInt(urlQuery.get("split_size"));
- if (urlQuery.containsKey("partitioner"))
- partitionerClass = urlQuery.get("partitioner");
- if (urlQuery.containsKey("use_secondary"))
- usePartitionFilter = Boolean.parseBoolean(urlQuery.get("use_secondary"));
- if (urlQuery.containsKey("init_address"))
- initHostAddress = urlQuery.get("init_address");
-
- if (urlQuery.containsKey("native_port"))
- nativePort = urlQuery.get("native_port");
- if (urlQuery.containsKey("core_conns"))
- nativeCoreConnections = urlQuery.get("core_conns");
- if (urlQuery.containsKey("max_conns"))
- nativeMaxConnections = urlQuery.get("max_conns");
- if (urlQuery.containsKey("max_simult_reqs"))
- nativeMaxSimultReqs = urlQuery.get("max_simult_reqs");
- if (urlQuery.containsKey("native_timeout"))
- nativeConnectionTimeout = urlQuery.get("native_timeout");
- if (urlQuery.containsKey("native_read_timeout"))
- nativeReadConnectionTimeout = urlQuery.get("native_read_timeout");
- if (urlQuery.containsKey("rec_buff_size"))
- nativeReceiveBufferSize = urlQuery.get("rec_buff_size");
- if (urlQuery.containsKey("send_buff_size"))
- nativeSendBufferSize = urlQuery.get("send_buff_size");
- if (urlQuery.containsKey("solinger"))
- nativeSolinger = urlQuery.get("solinger");
- if (urlQuery.containsKey("tcp_nodelay"))
- nativeTcpNodelay = urlQuery.get("tcp_nodelay");
- if (urlQuery.containsKey("reuse_address"))
- nativeReuseAddress = urlQuery.get("reuse_address");
- if (urlQuery.containsKey("keep_alive"))
- nativeKeepAlive = urlQuery.get("keep_alive");
- if (urlQuery.containsKey("auth_provider"))
- nativeAuthProvider = urlQuery.get("auth_provider");
- if (urlQuery.containsKey("trust_store_path"))
- nativeSSLTruststorePath = urlQuery.get("trust_store_path");
- if (urlQuery.containsKey("key_store_path"))
- nativeSSLKeystorePath = urlQuery.get("key_store_path");
- if (urlQuery.containsKey("trust_store_password"))
- nativeSSLTruststorePassword = urlQuery.get("trust_store_password");
- if (urlQuery.containsKey("key_store_password"))
- nativeSSLKeystorePassword = urlQuery.get("key_store_password");
- if (urlQuery.containsKey("cipher_suites"))
- nativeSSLCipherSuites = urlQuery.get("cipher_suites");
- if (urlQuery.containsKey("input_cql"))
- inputCql = urlQuery.get("input_cql");
- if (urlQuery.containsKey("columns"))
- columns = urlQuery.get("columns");
- if (urlQuery.containsKey("where_clause"))
- whereClause = urlQuery.get("where_clause");
- if (urlQuery.containsKey("rpc_port"))
- rpcPort = urlQuery.get("rpc_port");
- }
- String[] parts = urlParts[0].split("/+");
- String[] credentialsAndKeyspace = parts[1].split("@");
- if (credentialsAndKeyspace.length > 1)
- {
- String[] credentials = credentialsAndKeyspace[0].split(":");
- username = credentials[0];
- password = credentials[1];
- keyspace = credentialsAndKeyspace[1];
- }
- else
- {
- keyspace = parts[1];
- }
- column_family = parts[2];
- }
- catch (Exception e)
- {
- throw new IOException("Expected 'cql://[username:password@]<keyspace>/<columnfamily>" +
- "[?[page_size=<size>][&columns=<col1,col2>][&output_query=<prepared_statement>]" +
- "[&where_clause=<clause>][&split_size=<size>][&partitioner=<partitioner>][&use_secondary=true|false]" +
- "[&init_address=<host>][&native_port=<native_port>][&core_conns=<core_conns>]" +
- "[&max_conns=<max_conns>][&min_simult_reqs=<min_simult_reqs>][&max_simult_reqs=<max_simult_reqs>]" +
- "[&native_timeout=<native_timeout>][&native_read_timeout=<native_read_timeout>][&rec_buff_size=<rec_buff_size>]" +
- "[&send_buff_size=<send_buff_size>][&solinger=<solinger>][&tcp_nodelay=<tcp_nodelay>][&reuse_address=<reuse_address>]" +
- "[&keep_alive=<keep_alive>][&auth_provider=<auth_provider>][&trust_store_path=<trust_store_path>]" +
- "[&key_store_path=<key_store_path>][&trust_store_password=<trust_store_password>]" +
- "[&key_store_password=<key_store_password>][&cipher_suites=<cipher_suites>][&input_cql=<input_cql>]" +
- "[columns=<columns>][where_clause=<where_clause>]" +
- "[&bulk_cf_schema=bulk_cf_schema][&bulk_insert_statement=bulk_insert_statement][&bulk_table_alias=<bulk_table_alias>]" +
- "[&bulk_output_location=<bulk_output_location>][&bulk_buff_size=<bulk_buff_size>][&bulk_delete_source=<bulk_delete_source>]" +
- "[&bulk_stream_throttle=<bulk_stream_throttle>][&bulk_max_failed_hosts=<bulk_max_failed_hosts>]]': " + e.getMessage());
- }
- }
-
- public ByteBuffer nullToBB()
- {
- return ByteBuffer.wrap(new byte[0]);
- }
-
- /** output format */
- public OutputFormat getOutputFormat() throws IOException
- {
- try
- {
- return FBUtilities.construct(outputFormatClass, "outputformat");
- }
- catch (ConfigurationException e)
- {
- throw new IOException(e);
- }
- }
-
- public void cleanupOnFailure(String failure, Job job)
- {
- }
-
- public void cleanupOnSuccess(String location, Job job) throws IOException {
- }
-
- /** return partition keys */
- public String[] getPartitionKeys(String location, Job job) throws IOException
- {
- if (!usePartitionFilter)
- return null;
- TableInfo tableMetadata = getCfInfo(loadSignature);
- String[] partitionKeys = new String[tableMetadata.getPartitionKey().size()];
- for (int i = 0; i < tableMetadata.getPartitionKey().size(); i++)
- {
- partitionKeys[i] = tableMetadata.getPartitionKey().get(i).getName();
- }
- return partitionKeys;
- }
-
- public void checkSchema(ResourceSchema schema) throws IOException
- {
- // we don't care about types, they all get casted to ByteBuffers
- }
-
- public ResourceStatistics getStatistics(String location, Job job)
- {
- return null;
- }
-
- @Override
- public InputFormat getInputFormat() throws IOException
- {
- try
- {
- return FBUtilities.construct(inputFormatClass, "inputformat");
- }
- catch (ConfigurationException e)
- {
- throw new IOException(e);
- }
- }
-
- public String relToAbsPathForStoreLocation(String location, Path curDir) throws IOException
- {
- return relativeToAbsolutePath(location, curDir);
- }
-
- @Override
- public String relativeToAbsolutePath(String location, Path curDir) throws IOException
- {
- return location;
- }
-
- @Override
- public void setUDFContextSignature(String signature)
- {
- this.loadSignature = signature;
- }
-
- /** StoreFunc methods */
- public void setStoreFuncUDFContextSignature(String signature)
- {
- this.storeSignature = signature;
- }
-
- /** set hadoop cassandra connection settings */
- protected void setConnectionInformation() throws IOException
- {
- StorageHelper.setConnectionInformation(conf);
- if (System.getenv(StorageHelper.PIG_INPUT_FORMAT) != null)
- inputFormatClass = getFullyQualifiedClassName(System.getenv(StorageHelper.PIG_INPUT_FORMAT));
- else
- inputFormatClass = DEFAULT_INPUT_FORMAT;
- if (System.getenv(StorageHelper.PIG_OUTPUT_FORMAT) != null)
- outputFormatClass = getFullyQualifiedClassName(System.getenv(StorageHelper.PIG_OUTPUT_FORMAT));
- else
- outputFormatClass = DEFAULT_OUTPUT_FORMAT;
- }
-
- /** get the full class name */
- protected String getFullyQualifiedClassName(String classname)
- {
- return classname.contains(".") ? classname : "org.apache.cassandra.hadoop." + classname;
- }
-}
-
-class TableInfo implements Serializable
-{
- private final List<ColumnInfo> columns;
- private final List<ColumnInfo> partitionKey;
- private final String name;
-
- public TableInfo(TableMetadata tableMetadata)
- {
- List<ColumnMetadata> cmColumns = tableMetadata.getColumns();
- columns = new ArrayList<>(cmColumns.size());
- for (ColumnMetadata cm : cmColumns)
- {
- columns.add(new ColumnInfo(this, cm));
- }
- List<ColumnMetadata> cmPartitionKey = tableMetadata.getPartitionKey();
- partitionKey = new ArrayList<>(cmPartitionKey.size());
- for (ColumnMetadata cm : cmPartitionKey)
- {
- partitionKey.add(new ColumnInfo(this, cm));
- }
- name = tableMetadata.getName();
- }
-
- public List<ColumnInfo> getPartitionKey()
- {
- return partitionKey;
- }
-
- public List<ColumnInfo> getColumns()
- {
- return columns;
- }
-
- public String getName()
- {
- return name;
- }
-}
-
-class ColumnInfo implements Serializable
-{
- private final TableInfo table;
- private final String name;
- private final String typeName;
-
- public ColumnInfo(TableInfo tableInfo, ColumnMetadata columnMetadata)
- {
- table = tableInfo;
- name = columnMetadata.getName();
- typeName = columnMetadata.getType().toString();
- }
-
- public String getName()
- {
- return name;
- }
-
- public String getTypeName()
- {
- return typeName;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/cassandra/blob/56cfc6ea/src/java/org/apache/cassandra/hadoop/pig/StorageHelper.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/hadoop/pig/StorageHelper.java b/src/java/org/apache/cassandra/hadoop/pig/StorageHelper.java
deleted file mode 100644
index d700cb7..0000000
--- a/src/java/org/apache/cassandra/hadoop/pig/StorageHelper.java
+++ /dev/null
@@ -1,122 +0,0 @@
-package org.apache.cassandra.hadoop.pig;
-
-import java.math.BigInteger;
-import java.nio.ByteBuffer;
-import java.util.Date;
-import java.util.UUID;
-
-import org.apache.cassandra.db.marshal.*;
-import org.apache.cassandra.hadoop.ConfigHelper;
-import org.apache.cassandra.serializers.CollectionSerializer;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.UUIDGen;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.pig.backend.executionengine.ExecException;
-import org.apache.pig.data.DataByteArray;
-import org.apache.pig.data.DataType;
-import org.apache.pig.data.Tuple;
-
-@Deprecated
-public class StorageHelper
-{
- // system environment variables that can be set to configure connection info:
- // alternatively, Hadoop JobConf variables can be set using keys from ConfigHelper
- public final static String PIG_INPUT_RPC_PORT = "PIG_INPUT_RPC_PORT";
- public final static String PIG_INPUT_INITIAL_ADDRESS = "PIG_INPUT_INITIAL_ADDRESS";
- public final static String PIG_INPUT_PARTITIONER = "PIG_INPUT_PARTITIONER";
- public final static String PIG_OUTPUT_RPC_PORT = "PIG_OUTPUT_RPC_PORT";
- public final static String PIG_OUTPUT_INITIAL_ADDRESS = "PIG_OUTPUT_INITIAL_ADDRESS";
- public final static String PIG_OUTPUT_PARTITIONER = "PIG_OUTPUT_PARTITIONER";
- public final static String PIG_RPC_PORT = "PIG_RPC_PORT";
- public final static String PIG_INITIAL_ADDRESS = "PIG_INITIAL_ADDRESS";
- public final static String PIG_PARTITIONER = "PIG_PARTITIONER";
- public final static String PIG_INPUT_FORMAT = "PIG_INPUT_FORMAT";
- public final static String PIG_OUTPUT_FORMAT = "PIG_OUTPUT_FORMAT";
- public final static String PIG_INPUT_SPLIT_SIZE = "PIG_INPUT_SPLIT_SIZE";
-
-
- public final static String PARTITION_FILTER_SIGNATURE = "cassandra.partition.filter";
-
- protected static void setConnectionInformation(Configuration conf)
- {
- if (System.getenv(PIG_RPC_PORT) != null)
- {
- ConfigHelper.setInputRpcPort(conf, System.getenv(PIG_RPC_PORT));
- ConfigHelper.setOutputRpcPort(conf, System.getenv(PIG_RPC_PORT));
- }
-
- if (System.getenv(PIG_INPUT_RPC_PORT) != null)
- ConfigHelper.setInputRpcPort(conf, System.getenv(PIG_INPUT_RPC_PORT));
- if (System.getenv(PIG_OUTPUT_RPC_PORT) != null)
- ConfigHelper.setOutputRpcPort(conf, System.getenv(PIG_OUTPUT_RPC_PORT));
-
- if (System.getenv(PIG_INITIAL_ADDRESS) != null)
- {
- ConfigHelper.setInputInitialAddress(conf, System.getenv(PIG_INITIAL_ADDRESS));
- ConfigHelper.setOutputInitialAddress(conf, System.getenv(PIG_INITIAL_ADDRESS));
- }
- if (System.getenv(PIG_INPUT_INITIAL_ADDRESS) != null)
- ConfigHelper.setInputInitialAddress(conf, System.getenv(PIG_INPUT_INITIAL_ADDRESS));
- if (System.getenv(PIG_OUTPUT_INITIAL_ADDRESS) != null)
- ConfigHelper.setOutputInitialAddress(conf, System.getenv(PIG_OUTPUT_INITIAL_ADDRESS));
-
- if (System.getenv(PIG_PARTITIONER) != null)
- {
- ConfigHelper.setInputPartitioner(conf, System.getenv(PIG_PARTITIONER));
- ConfigHelper.setOutputPartitioner(conf, System.getenv(PIG_PARTITIONER));
- }
- if(System.getenv(PIG_INPUT_PARTITIONER) != null)
- ConfigHelper.setInputPartitioner(conf, System.getenv(PIG_INPUT_PARTITIONER));
- if(System.getenv(PIG_OUTPUT_PARTITIONER) != null)
- ConfigHelper.setOutputPartitioner(conf, System.getenv(PIG_OUTPUT_PARTITIONER));
- }
-
- protected static Object cassandraToObj(AbstractType validator, ByteBuffer value, int nativeProtocolVersion)
- {
- if (validator instanceof DecimalType || validator instanceof InetAddressType)
- return validator.getString(value);
-
- if (validator instanceof CollectionType)
- {
- // For CollectionType, the compose() method assumes the v3 protocol format of collection, which
- // is not correct here since we query using the CQL-over-thrift interface which use the pre-v3 format
- return ((CollectionSerializer)validator.getSerializer()).deserializeForNativeProtocol(value, nativeProtocolVersion);
- }
-
- return validator.compose(value);
- }
-
- /** set the value to the position of the tuple */
- protected static void setTupleValue(Tuple pair, int position, Object value) throws ExecException
- {
- if (value instanceof BigInteger)
- pair.set(position, ((BigInteger) value).intValue());
- else if (value instanceof ByteBuffer)
- pair.set(position, new DataByteArray(ByteBufferUtil.getArray((ByteBuffer) value)));
- else if (value instanceof UUID)
- pair.set(position, new DataByteArray(UUIDGen.decompose((java.util.UUID) value)));
- else if (value instanceof Date)
- pair.set(position, TimestampType.instance.decompose((Date) value).getLong());
- else
- pair.set(position, value);
- }
-
- /** get pig type for the cassandra data type*/
- protected static byte getPigType(AbstractType type)
- {
- if (type instanceof LongType || type instanceof DateType || type instanceof TimestampType) // DateType is bad and it should feel bad
- return DataType.LONG;
- else if (type instanceof IntegerType || type instanceof Int32Type) // IntegerType will overflow at 2**31, but is kept for compatibility until pig has a BigInteger
- return DataType.INTEGER;
- else if (type instanceof AsciiType || type instanceof UTF8Type || type instanceof DecimalType || type instanceof InetAddressType)
- return DataType.CHARARRAY;
- else if (type instanceof FloatType)
- return DataType.FLOAT;
- else if (type instanceof DoubleType)
- return DataType.DOUBLE;
- else if (type instanceof AbstractCompositeType || type instanceof CollectionType)
- return DataType.TUPLE;
-
- return DataType.BYTEARRAY;
- }
-}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/56cfc6ea/test/conf/cassandra_pig.yaml
----------------------------------------------------------------------
diff --git a/test/conf/cassandra_pig.yaml b/test/conf/cassandra_pig.yaml
deleted file mode 100644
index ce71410..0000000
--- a/test/conf/cassandra_pig.yaml
+++ /dev/null
@@ -1,42 +0,0 @@
-#
-# Warning!
-# Consider the effects on 'o.a.c.i.s.LegacySSTableTest' before changing schemas in this file.
-#
-cluster_name: Test Cluster
-memtable_allocation_type: heap_buffers
-commitlog_sync: batch
-commitlog_sync_batch_window_in_ms: 1.0
-commitlog_segment_size_in_mb: 5
-commitlog_directory: build/test/cassandra/commitlog
-hints_directory: build/test/cassandra/hints
-partitioner: org.apache.cassandra.dht.Murmur3Partitioner
-listen_address: 127.0.0.1
-storage_port: 7010
-rpc_port: 9170
-start_native_transport: true
-native_transport_port: 9042
-column_index_size_in_kb: 4
-saved_caches_directory: build/test/cassandra/saved_caches
-data_file_directories:
- - build/test/cassandra/data
-disk_access_mode: mmap
-seed_provider:
- - class_name: org.apache.cassandra.locator.SimpleSeedProvider
- parameters:
- - seeds: "127.0.0.1"
-endpoint_snitch: org.apache.cassandra.locator.SimpleSnitch
-dynamic_snitch: true
-request_scheduler: org.apache.cassandra.scheduler.RoundRobinScheduler
-request_scheduler_id: keyspace
-server_encryption_options:
- internode_encryption: none
- keystore: conf/.keystore
- keystore_password: cassandra
- truststore: conf/.truststore
- truststore_password: cassandra
-incremental_backups: true
-concurrent_compactors: 4
-compaction_throughput_mb_per_sec: 0
-row_cache_class_name: org.apache.cassandra.cache.OHCProvider
-row_cache_size_in_mb: 16
-enable_user_defined_functions: true
http://git-wip-us.apache.org/repos/asf/cassandra/blob/56cfc6ea/test/pig/org/apache/cassandra/pig/CqlRecordReaderTest.java
----------------------------------------------------------------------
diff --git a/test/pig/org/apache/cassandra/pig/CqlRecordReaderTest.java b/test/pig/org/apache/cassandra/pig/CqlRecordReaderTest.java
deleted file mode 100644
index b2a74b6..0000000
--- a/test/pig/org/apache/cassandra/pig/CqlRecordReaderTest.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.pig;
-
-import java.io.IOException;
-import java.util.Iterator;
-
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.pig.data.Tuple;
-import org.apache.thrift.TException;
-
-public class CqlRecordReaderTest extends PigTestBase
-{
- private static String[] statements = {
- "DROP KEYSPACE IF EXISTS cql3ks",
- "CREATE KEYSPACE cql3ks WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 1};",
- "USE cql3ks;",
-
- "CREATE TABLE cqltable (" +
- "pk1 int," +
- "pk2 int," +
- "pk3 int," +
- "ck1 int," +
- "ck2 int," +
- "data text," +
- "primary key((pk1,pk2,pk3),ck1,ck2));",
- "INSERT INTO cqltable(pk1, pk2, pk3, ck1, ck2, data) VALUES (11, 12, 13, 14, 15, 'value1');",
-
- "CREATE TABLE \"MixedCaseCqlTable\" (" +
- "pk1 int," +
- "\"PK2\" int," +
- "pk3 int," +
- "\"CK1\" int," +
- "ck2 int," +
- "data text," +
- "primary key((pk1,\"PK2\",pk3),\"CK1\",ck2));",
- "INSERT INTO \"MixedCaseCqlTable\"(pk1, \"PK2\", pk3, \"CK1\", ck2, data) VALUES (11, 12, 13, 14, 15, 'value1');",
- };
-
- @BeforeClass
- public static void setup() throws IOException, ConfigurationException, TException
- {
- startCassandra();
- executeCQLStatements(statements);
- startHadoopCluster();
- }
-
- @Test
- public void defaultCqlQueryTest() throws Exception
- {
- String initialQuery = "rows = LOAD 'cql://cql3ks/cqltable?" + defaultParameters + nativeParameters + "' USING CqlNativeStorage();";
- pig.registerQuery(initialQuery);
- Iterator<Tuple> it = pig.openIterator("rows");
- if (it.hasNext()) {
- Tuple t = it.next();
- Assert.assertEquals(t.get(0), 11);
- Assert.assertEquals(t.get(1), 12);
- Assert.assertEquals(t.get(2), 13);
- Assert.assertEquals(t.get(3), 14);
- Assert.assertEquals(t.get(4), 15);
- Assert.assertEquals(t.get(5), "value1");
- }
- else
- {
- Assert.fail("Failed to get data for query " + initialQuery);
- }
- }
-
- @Test
- public void defaultMixedCaseCqlQueryTest() throws Exception
- {
- String initialQuery = "rows = LOAD 'cql://cql3ks/MixedCaseCqlTable?" + defaultParameters + nativeParameters + "' USING CqlNativeStorage();";
- pig.registerQuery(initialQuery);
- Iterator<Tuple> it = pig.openIterator("rows");
- if (it.hasNext()) {
- Tuple t = it.next();
- Assert.assertEquals(t.get(0), 11);
- Assert.assertEquals(t.get(1), 12);
- Assert.assertEquals(t.get(2), 13);
- Assert.assertEquals(t.get(3), 14);
- Assert.assertEquals(t.get(4), 15);
- Assert.assertEquals(t.get(5), "value1");
- }
- else
- {
- Assert.fail("Failed to get data for query " + initialQuery);
- }
- }
-
- @Test
- public void selectColumnsTest() throws Exception
- {
- String initialQuery = "rows = LOAD 'cql://cql3ks/cqltable?" + defaultParameters + nativeParameters + "&columns=ck1%2Cck2%2Cdata' USING CqlNativeStorage();";
- pig.registerQuery(initialQuery);
- Iterator<Tuple> it = pig.openIterator("rows");
- if (it.hasNext()) {
- Tuple t = it.next();
- Assert.assertEquals(t.get(0), 11);
- Assert.assertEquals(t.get(1), 12);
- Assert.assertEquals(t.get(2), 13);
- Assert.assertEquals(t.get(3), 14);
- Assert.assertEquals(t.get(4), 15);
- Assert.assertEquals(t.get(5), "value1");
- }
- else
- {
- Assert.fail("Failed to get data for query " + initialQuery);
- }
- }
-
- @Test
- public void whereClauseTest() throws Exception
- {
- String initialQuery = "rows = LOAD 'cql://cql3ks/cqltable?" + defaultParameters + nativeParameters + "&where_clause=ck1%3d14' USING CqlNativeStorage();";
- pig.registerQuery(initialQuery);
- Iterator<Tuple> it = pig.openIterator("rows");
- if (it.hasNext()) {
- Tuple t = it.next();
- Assert.assertEquals(t.get(0), 11);
- Assert.assertEquals(t.get(1), 12);
- Assert.assertEquals(t.get(2), 13);
- Assert.assertEquals(t.get(3), 14);
- Assert.assertEquals(t.get(4), 15);
- Assert.assertEquals(t.get(5), "value1");
- }
- else
- {
- Assert.fail("Failed to get data for query " + initialQuery);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/56cfc6ea/test/pig/org/apache/cassandra/pig/CqlTableDataTypeTest.java
----------------------------------------------------------------------
diff --git a/test/pig/org/apache/cassandra/pig/CqlTableDataTypeTest.java b/test/pig/org/apache/cassandra/pig/CqlTableDataTypeTest.java
deleted file mode 100644
index ca01901..0000000
--- a/test/pig/org/apache/cassandra/pig/CqlTableDataTypeTest.java
+++ /dev/null
@@ -1,479 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.pig;
-
-import java.io.IOException;
-import java.util.Iterator;
-
-import org.apache.cassandra.db.marshal.TimeUUIDType;
-import org.apache.cassandra.db.marshal.UUIDType;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.utils.Hex;
-import org.apache.pig.data.DataByteArray;
-import org.apache.pig.data.Tuple;
-import org.apache.thrift.TException;
-
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class CqlTableDataTypeTest extends PigTestBase
-{
- //ASCII (AsciiType.instance),
- //BIGINT (LongType.instance),
- //BLOB (BytesType.instance),
- //BOOLEAN (BooleanType.instance),
- //COUNTER (CounterColumnType.instance),
- //DECIMAL (DecimalType.instance),
- //DOUBLE (DoubleType.instance),
- //FLOAT (FloatType.instance),
- //INET (InetAddressType.instance),
- //INT (Int32Type.instance),
- //TEXT (UTF8Type.instance),
- //TIMESTAMP(DateType.instance),
- //UUID (UUIDType.instance),
- //VARCHAR (UTF8Type.instance),
- //VARINT (IntegerType.instance),
- //TIMEUUID (TimeUUIDType.instance);
- //SET
- //LIST
- //MAP
- //Create table to test the above data types
- private static String[] statements = {
- "DROP KEYSPACE IF EXISTS cql3ks",
- "CREATE KEYSPACE cql3ks WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 1}",
- "USE cql3ks;",
-
- "CREATE TABLE cqltable (" +
- "key int primary key," +
- "col_ascii ascii," +
- "col_bigint bigint," +
- "col_blob blob," +
- "col_boolean boolean," +
- "col_decimal decimal," +
- "col_double double," +
- "col_float float," +
- "col_inet inet," +
- "col_int int," +
- "col_text text," +
- "col_timestamp timestamp," +
- "col_uuid uuid," +
- "col_varchar varchar," +
- "col_varint varint," +
- "col_timeuuid timeuuid);",
-
- "CREATE TABLE settable (" +
- "key int primary key," +
- "col_set_ascii set<ascii>," +
- "col_set_bigint set<bigint>," +
- "col_set_blob set<blob>," +
- "col_set_boolean set<boolean>," +
- "col_set_decimal set<decimal>," +
- "col_set_double set<double>," +
- "col_set_float set<float>," +
- "col_set_inet set<inet>," +
- "col_set_int set<int>," +
- "col_set_text set<text>," +
- "col_set_timestamp set<timestamp>," +
- "col_set_uuid set<uuid>," +
- "col_set_varchar set<varchar>," +
- "col_set_varint set<varint>," +
- "col_set_timeuuid set<timeuuid>);",
-
- "CREATE TABLE listtable (" +
- "key int primary key," +
- "col_list_ascii list<ascii>," +
- "col_list_bigint list<bigint>," +
- "col_list_blob list<blob>," +
- "col_list_boolean list<boolean>," +
- "col_list_decimal list<decimal>," +
- "col_list_double list<double>," +
- "col_list_float list<float>," +
- "col_list_inet list<inet>," +
- "col_list_int list<int>," +
- "col_list_text list<text>," +
- "col_list_timestamp list<timestamp>," +
- "col_list_uuid list<uuid>," +
- "col_list_varchar list<varchar>," +
- "col_list_varint list<varint>," +
- "col_list_timeuuid list<timeuuid>);",
-
- "CREATE TABLE maptable (" +
- "key int primary key," +
- "col_map_ascii map<ascii, ascii>," +
- "col_map_bigint map<bigint, bigint>," +
- "col_map_blob map<blob, blob>," +
- "col_map_boolean map<boolean, boolean>," +
- "col_map_decimal map<decimal, decimal>," +
- "col_map_double map<double, double>," +
- "col_map_float map<float, float>," +
- "col_map_inet map<inet, inet>," +
- "col_map_int map<int, int>," +
- "col_map_text map<text, text>," +
- "col_map_timestamp map<timestamp, timestamp>," +
- "col_map_uuid map<uuid, uuid>," +
- "col_map_varchar map<varchar, varchar>," +
- "col_map_varint map<varint, varint>," +
- "col_map_timeuuid map<timeuuid, timeuuid>);",
-
- "INSERT INTO cqltable(key, col_ascii) VALUES (1, 'ascii');",
- "INSERT INTO cqltable(key, col_bigint) VALUES (1, 12345678);",
- "INSERT INTO cqltable(key, col_blob) VALUES (1, 0x23446c6c6f);",
- "INSERT INTO cqltable(key, col_boolean) VALUES (1, false);",
- "INSERT INTO cqltable(key, col_decimal) VALUES (1, 23.4567);",
- "INSERT INTO cqltable(key, col_double) VALUES (1, 12345678.12345678);",
- "INSERT INTO cqltable(key, col_float) VALUES (1, 123.12);",
- "INSERT INTO cqltable(key, col_inet) VALUES (1, '127.0.0.1');",
- "INSERT INTO cqltable(key, col_int) VALUES (1, 123);",
- "INSERT INTO cqltable(key, col_text) VALUES (1, 'text');",
- "INSERT INTO cqltable(key, col_timestamp) VALUES (1, '2011-02-03T04:05:00+0000');",
- "INSERT INTO cqltable(key, col_timeuuid) VALUES (1, maxTimeuuid('2013-01-01 00:05+0000'));",
- "INSERT INTO cqltable(key, col_uuid) VALUES (1, 550e8400-e29b-41d4-a716-446655440000);",
- "INSERT INTO cqltable(key, col_varchar) VALUES (1, 'varchar');",
- "INSERT INTO cqltable(key, col_varint) VALUES (1, 123);",
-
- "INSERT INTO settable(key, col_set_ascii) VALUES (1, {'ascii1', 'ascii2'});",
- "INSERT INTO settable(key, col_set_bigint) VALUES (1, {12345678, 12345679});",
- "INSERT INTO settable(key, col_set_blob) VALUES (1, {0x68656c6c6f, 0x68656c6c6e});",
- "INSERT INTO settable(key, col_set_boolean) VALUES (1, {false, true});",
- "INSERT INTO settable(key, col_set_decimal) VALUES (1, {23.4567, 23.4568});",
- "INSERT INTO settable(key, col_set_double) VALUES (1, {12345678.12345678, 12345678.12345679});",
- "INSERT INTO settable(key, col_set_float) VALUES (1, {123.12, 123.13});",
- "INSERT INTO settable(key, col_set_inet) VALUES (1, {'127.0.0.1', '127.0.0.2'});",
- "INSERT INTO settable(key, col_set_int) VALUES (1, {123, 124});",
- "INSERT INTO settable(key, col_set_text) VALUES (1, {'text1', 'text2'});",
- "INSERT INTO settable(key, col_set_timestamp) VALUES (1, {'2011-02-03T04:05:00+0000', '2011-02-04T04:05:00+0000'});",
- "INSERT INTO settable(key, col_set_timeuuid) VALUES (1, {e23f450f-53a6-11e2-7f7f-7f7f7f7f7f7f, e23f450f-53a6-11e2-7f7f-7f7f7f7f7f77});",
- "INSERT INTO settable(key, col_set_uuid) VALUES (1, {550e8400-e29b-41d4-a716-446655440000, 550e8400-e29b-41d4-a716-446655440001});",
- "INSERT INTO settable(key, col_set_varchar) VALUES (1, {'varchar1', 'varchar2'});",
- "INSERT INTO settable(key, col_set_varint) VALUES (1, {123, 124});",
-
- "INSERT INTO listtable(key, col_list_ascii) VALUES (1, ['ascii2', 'ascii1']);",
- "INSERT INTO listtable(key, col_list_bigint) VALUES (1, [12345679, 12345678]);",
- "INSERT INTO listtable(key, col_list_blob) VALUES (1, [0x68656c6c6e, 0x68656c6c6f]);",
- "INSERT INTO listtable(key, col_list_boolean) VALUES (1, [true, false]);",
- "INSERT INTO listtable(key, col_list_decimal) VALUES (1, [23.4568, 23.4567]);",
- "INSERT INTO listtable(key, col_list_double) VALUES (1, [12345678.12345679, 12345678.12345678]);",
- "INSERT INTO listtable(key, col_list_float) VALUES (1, [123.13, 123.12]);",
- "INSERT INTO listtable(key, col_list_inet) VALUES (1, ['127.0.0.2', '127.0.0.1']);",
- "INSERT INTO listtable(key, col_list_int) VALUES (1, [124, 123]);",
- "INSERT INTO listtable(key, col_list_text) VALUES (1, ['text2', 'text1']);",
- "INSERT INTO listtable(key, col_list_timestamp) VALUES (1, ['2011-02-04T04:05:00+0000', '2011-02-03T04:05:00+0000']);",
- "INSERT INTO listtable(key, col_list_timeuuid) VALUES (1, [e23f450f-53a6-11e2-7f7f-7f7f7f7f7f77, e23f450f-53a6-11e2-7f7f-7f7f7f7f7f7f]);",
- "INSERT INTO listtable(key, col_list_uuid) VALUES (1, [550e8400-e29b-41d4-a716-446655440001, 550e8400-e29b-41d4-a716-446655440000]);",
- "INSERT INTO listtable(key, col_list_varchar) VALUES (1, ['varchar2', 'varchar1']);",
- "INSERT INTO listtable(key, col_list_varint) VALUES (1, [124, 123]);",
-
- "INSERT INTO maptable(key, col_map_ascii) VALUES (1, {'ascii1' : 'ascii2'});",
- "INSERT INTO maptable(key, col_map_bigint) VALUES (1, {12345678 : 12345679});",
- "INSERT INTO maptable(key, col_map_blob) VALUES (1, {0x68656c6c6f : 0x68656c6c6e});",
- "INSERT INTO maptable(key, col_map_boolean) VALUES (1, {false : true});",
- "INSERT INTO maptable(key, col_map_decimal) VALUES (1, {23.4567 : 23.4568});",
- "INSERT INTO maptable(key, col_map_double) VALUES (1, {12345678.12345678 : 12345678.12345679});",
- "INSERT INTO maptable(key, col_map_float) VALUES (1, {123.12 : 123.13});",
- "INSERT INTO maptable(key, col_map_inet) VALUES (1, {'127.0.0.1' : '127.0.0.2'});",
- "INSERT INTO maptable(key, col_map_int) VALUES (1, {123 : 124});",
- "INSERT INTO maptable(key, col_map_text) VALUES (1, {'text1' : 'text2'});",
- "INSERT INTO maptable(key, col_map_timestamp) VALUES (1, {'2011-02-03T04:05:00+0000' : '2011-02-04T04:05:00+0000'});",
- "INSERT INTO maptable(key, col_map_timeuuid) VALUES (1, {e23f450f-53a6-11e2-7f7f-7f7f7f7f7f7f : e23f450f-53a6-11e2-7f7f-7f7f7f7f7f77});",
- "INSERT INTO maptable(key, col_map_uuid) VALUES (1, {550e8400-e29b-41d4-a716-446655440000 : 550e8400-e29b-41d4-a716-446655440001});",
- "INSERT INTO maptable(key, col_map_varchar) VALUES (1, {'varchar1' : 'varchar2'});",
- "INSERT INTO maptable(key, col_map_varint) VALUES (1, {123 : 124});",
-
- "CREATE TABLE countertable (key int primary key, col_counter counter);",
- "UPDATE countertable SET col_counter = col_counter + 3 WHERE key = 1;",
- };
-
- @BeforeClass
- public static void setup() throws IOException, ConfigurationException, TException
- {
- startCassandra();
- executeCQLStatements(statements);
- startHadoopCluster();
- }
-
- @Test
- public void testCqlNativeStorageRegularType() throws IOException
- {
- //input_cql=select * from cqltable where token(key) > ? and token(key) <= ?
- cqlTableTest("rows = LOAD 'cql://cql3ks/cqltable?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20cqltable%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' USING CqlNativeStorage();");
-
- //input_cql=select * from countertable where token(key) > ? and token(key) <= ?
- counterTableTest("cc_rows = LOAD 'cql://cql3ks/countertable?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20countertable%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' USING CqlNativeStorage();");
- }
-
- private void cqlTableTest(String initialQuery) throws IOException
- {
- pig.registerQuery(initialQuery);
- Iterator<Tuple> it = pig.openIterator("rows");
- //{key: int,
- //col_ascii: chararray,
- //col_bigint: long,
- //col_blob: bytearray,
- //col_boolean: bytearray,
- //col_decimal: chararray,
- //col_double: double,
- //col_float: float,
- //col_inet: chararray,
- //col_int: int,
- //col_text: chararray,
- //col_timestamp: long,
- //col_timeuuid: bytearray,
- //col_uuid: chararray,
- //col_varchar: chararray,
- //col_varint: int}
- if (it.hasNext()) {
- Tuple t = it.next();
- Assert.assertEquals(t.get(0), 1);
- Assert.assertEquals(t.get(1), "ascii");
- Assert.assertEquals(t.get(2), 12345678L);
- Assert.assertEquals(t.get(3), new DataByteArray(Hex.hexToBytes("23446c6c6f")));
- Assert.assertEquals(t.get(4), false);
- Assert.assertEquals(t.get(5), "23.4567");
- Assert.assertEquals(t.get(6), 12345678.12345678d);
- Assert.assertEquals(t.get(7), 123.12f);
- Assert.assertEquals(t.get(8), "127.0.0.1");
- Assert.assertEquals(t.get(9), 123);
- Assert.assertEquals(t.get(10), "text");
- Assert.assertEquals(t.get(11), 1296705900000L);
- Assert.assertEquals(t.get(12), new DataByteArray((TimeUUIDType.instance.fromString("e23f450f-53a6-11e2-7f7f-7f7f7f7f7f7f").array())));
- Assert.assertEquals(t.get(13), new DataByteArray((UUIDType.instance.fromString("550e8400-e29b-41d4-a716-446655440000").array())));
- Assert.assertEquals(t.get(14), "varchar");
- Assert.assertEquals(t.get(15), 123);
- }
- else
- {
- Assert.fail("Failed to get data for query " + initialQuery);
- }
- }
-
- private void counterTableTest(String initialQuery) throws IOException
- {
- pig.registerQuery(initialQuery);
- Iterator<Tuple> it = pig.openIterator("cc_rows");
- if (it.hasNext()) {
- Tuple t = it.next();
- Assert.assertEquals(t.get(0), 1);
- Assert.assertEquals(t.get(1), 3L);
- }
- else
- {
- Assert.fail("Failed to get data for query " + initialQuery);
- }
- }
-
- @Test
- public void testCqlNativeStorageSetType() throws IOException
- {
- //input_cql=select * from settable where token(key) > ? and token(key) <= ?
- settableTest("set_rows = LOAD 'cql://cql3ks/settable?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20settable%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' USING CqlNativeStorage();");
- }
-
- private void settableTest(String initialQuery) throws IOException
- {
- pig.registerQuery(initialQuery);
- Iterator<Tuple> it = pig.openIterator("set_rows");
- if (it.hasNext()) {
- Tuple t = it.next();
- Assert.assertEquals(t.get(0), 1);
- Tuple innerTuple = (Tuple) t.get(1);
- Assert.assertEquals(innerTuple.get(0), "ascii1");
- Assert.assertEquals(innerTuple.get(1), "ascii2");
- innerTuple = (Tuple) t.get(2);
- Assert.assertEquals(innerTuple.get(0), 12345678L);
- Assert.assertEquals(innerTuple.get(1), 12345679L);
- innerTuple = (Tuple) t.get(3);
- Assert.assertEquals(innerTuple.get(0), new DataByteArray(Hex.hexToBytes("68656c6c6e")));
- Assert.assertEquals(innerTuple.get(1), new DataByteArray(Hex.hexToBytes("68656c6c6f")));
- innerTuple = (Tuple) t.get(4);
- Assert.assertEquals(innerTuple.get(0), false);
- Assert.assertEquals(innerTuple.get(1), true);
- innerTuple = (Tuple) t.get(5);
- Assert.assertEquals(innerTuple.get(0), "23.4567");
- Assert.assertEquals(innerTuple.get(1), "23.4568");
- innerTuple = (Tuple) t.get(6);
- Assert.assertEquals(innerTuple.get(0), 12345678.12345678d);
- Assert.assertEquals(innerTuple.get(1), 12345678.12345679d);
- innerTuple = (Tuple) t.get(7);
- Assert.assertEquals(innerTuple.get(0), 123.12f);
- Assert.assertEquals(innerTuple.get(1), 123.13f);
- innerTuple = (Tuple) t.get(8);
- Assert.assertEquals(innerTuple.get(0), "127.0.0.1");
- Assert.assertEquals(innerTuple.get(1), "127.0.0.2");
- innerTuple = (Tuple) t.get(9);
- Assert.assertEquals(innerTuple.get(0), 123);
- Assert.assertEquals(innerTuple.get(1), 124);
- innerTuple = (Tuple) t.get(10);
- Assert.assertEquals(innerTuple.get(0), "text1");
- Assert.assertEquals(innerTuple.get(1), "text2");
- innerTuple = (Tuple) t.get(11);
- Assert.assertEquals(innerTuple.get(0), 1296705900000L);
- Assert.assertEquals(innerTuple.get(1), 1296792300000L);
- innerTuple = (Tuple) t.get(12);
- Assert.assertEquals(innerTuple.get(0), new DataByteArray((TimeUUIDType.instance.fromString("e23f450f-53a6-11e2-7f7f-7f7f7f7f7f77").array())));
- Assert.assertEquals(innerTuple.get(1), new DataByteArray((TimeUUIDType.instance.fromString("e23f450f-53a6-11e2-7f7f-7f7f7f7f7f7f").array())));
- innerTuple = (Tuple) t.get(13);
- Assert.assertEquals(innerTuple.get(0), new DataByteArray((UUIDType.instance.fromString("550e8400-e29b-41d4-a716-446655440000").array())));
- Assert.assertEquals(innerTuple.get(1), new DataByteArray((UUIDType.instance.fromString("550e8400-e29b-41d4-a716-446655440001").array())));
- innerTuple = (Tuple) t.get(14);
- Assert.assertEquals(innerTuple.get(0), "varchar1");
- Assert.assertEquals(innerTuple.get(1), "varchar2");
- innerTuple = (Tuple) t.get(15);
- Assert.assertEquals(innerTuple.get(0), 123);
- Assert.assertEquals(innerTuple.get(1), 124);
- }
- else
- {
- Assert.fail("Failed to get data for query " + initialQuery);
- }
- }
-
- @Test
- public void testCqlNativeStorageListType() throws IOException
- {
- //input_cql=select * from listtable where token(key) > ? and token(key) <= ?
- listtableTest("list_rows = LOAD 'cql://cql3ks/listtable?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20listtable%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' USING CqlNativeStorage();");
- }
-
- private void listtableTest(String initialQuery) throws IOException
- {
- pig.registerQuery(initialQuery);
- Iterator<Tuple> it = pig.openIterator("list_rows");
- if (it.hasNext()) {
- Tuple t = it.next();
- Assert.assertEquals(t.get(0), 1);
- Tuple innerTuple = (Tuple) t.get(1);
- Assert.assertEquals(innerTuple.get(1), "ascii1");
- Assert.assertEquals(innerTuple.get(0), "ascii2");
- innerTuple = (Tuple) t.get(2);
- Assert.assertEquals(innerTuple.get(1), 12345678L);
- Assert.assertEquals(innerTuple.get(0), 12345679L);
- innerTuple = (Tuple) t.get(3);
- Assert.assertEquals(innerTuple.get(1), new DataByteArray(Hex.hexToBytes("68656c6c6f")));
- Assert.assertEquals(innerTuple.get(0), new DataByteArray(Hex.hexToBytes("68656c6c6e")));
- innerTuple = (Tuple) t.get(4);
- Assert.assertEquals(innerTuple.get(1), false);
- Assert.assertEquals(innerTuple.get(0), true);
- innerTuple = (Tuple) t.get(5);
- Assert.assertEquals(innerTuple.get(1), "23.4567");
- Assert.assertEquals(innerTuple.get(0), "23.4568");
- innerTuple = (Tuple) t.get(6);
- Assert.assertEquals(innerTuple.get(1), 12345678.12345678d);
- Assert.assertEquals(innerTuple.get(0), 12345678.12345679d);
- innerTuple = (Tuple) t.get(7);
- Assert.assertEquals(innerTuple.get(1), 123.12f);
- Assert.assertEquals(innerTuple.get(0), 123.13f);
- innerTuple = (Tuple) t.get(8);
- Assert.assertEquals(innerTuple.get(1), "127.0.0.1");
- Assert.assertEquals(innerTuple.get(0), "127.0.0.2");
- innerTuple = (Tuple) t.get(9);
- Assert.assertEquals(innerTuple.get(1), 123);
- Assert.assertEquals(innerTuple.get(0), 124);
- innerTuple = (Tuple) t.get(10);
- Assert.assertEquals(innerTuple.get(1), "text1");
- Assert.assertEquals(innerTuple.get(0), "text2");
- innerTuple = (Tuple) t.get(11);
- Assert.assertEquals(innerTuple.get(1), 1296705900000L);
- Assert.assertEquals(innerTuple.get(0), 1296792300000L);
- innerTuple = (Tuple) t.get(12);
- Assert.assertEquals(innerTuple.get(1), new DataByteArray((TimeUUIDType.instance.fromString("e23f450f-53a6-11e2-7f7f-7f7f7f7f7f7f").array())));
- Assert.assertEquals(innerTuple.get(0), new DataByteArray((TimeUUIDType.instance.fromString("e23f450f-53a6-11e2-7f7f-7f7f7f7f7f77").array())));
- innerTuple = (Tuple) t.get(13);
- Assert.assertEquals(innerTuple.get(1), new DataByteArray((UUIDType.instance.fromString("550e8400-e29b-41d4-a716-446655440000").array())));
- Assert.assertEquals(innerTuple.get(0), new DataByteArray((UUIDType.instance.fromString("550e8400-e29b-41d4-a716-446655440001").array())));
- innerTuple = (Tuple) t.get(14);
- Assert.assertEquals(innerTuple.get(1), "varchar1");
- Assert.assertEquals(innerTuple.get(0), "varchar2");
- innerTuple = (Tuple) t.get(15);
- Assert.assertEquals(innerTuple.get(1), 123);
- Assert.assertEquals(innerTuple.get(0), 124);
- }
- else
- {
- Assert.fail("Failed to get data for query " + initialQuery);
- }
- }
-
- @Test
- public void testCqlNativeStorageMapType() throws IOException
- {
- //input_cql=select * from maptable where token(key) > ? and token(key) <= ?
- maptableTest("map_rows = LOAD 'cql://cql3ks/maptable?" + defaultParameters + nativeParameters + "&input_cql=select%20*%20from%20maptable%20where%20token(key)%20%3E%20%3F%20and%20token(key)%20%3C%3D%20%3F' USING CqlNativeStorage();");
- }
-
- private void maptableTest(String initialQuery) throws IOException
- {
- pig.registerQuery(initialQuery);
- Iterator<Tuple> it = pig.openIterator("map_rows");
- if (it.hasNext()) {
- Tuple t = it.next();
- Assert.assertEquals(t.get(0), 1);
- Tuple innerTuple = (Tuple) ((Tuple) t.get(1)).get(0);
- Assert.assertEquals(innerTuple.get(0), "ascii1");
- Assert.assertEquals(innerTuple.get(1), "ascii2");
- innerTuple = (Tuple) ((Tuple) t.get(2)).get(0);
- Assert.assertEquals(innerTuple.get(0), 12345678L);
- Assert.assertEquals(innerTuple.get(1), 12345679L);
- innerTuple = (Tuple) ((Tuple) t.get(3)).get(0);
- Assert.assertEquals(innerTuple.get(0), new DataByteArray(Hex.hexToBytes("68656c6c6f")));
- Assert.assertEquals(innerTuple.get(1), new DataByteArray(Hex.hexToBytes("68656c6c6e")));
- innerTuple = (Tuple) ((Tuple) t.get(4)).get(0);
- Assert.assertEquals(innerTuple.get(0), false);
- Assert.assertEquals(innerTuple.get(1), true);
- innerTuple = (Tuple) ((Tuple) t.get(5)).get(0);
- Assert.assertEquals(innerTuple.get(0), "23.4567");
- Assert.assertEquals(innerTuple.get(1), "23.4568");
- innerTuple = (Tuple) ((Tuple) t.get(6)).get(0);
- Assert.assertEquals(innerTuple.get(0), 12345678.12345678d);
- Assert.assertEquals(innerTuple.get(1), 12345678.12345679d);
- innerTuple = (Tuple) ((Tuple) t.get(7)).get(0);
- Assert.assertEquals(innerTuple.get(0), 123.12f);
- Assert.assertEquals(innerTuple.get(1), 123.13f);
- innerTuple = (Tuple) ((Tuple) t.get(8)).get(0);
- Assert.assertEquals(innerTuple.get(0), "127.0.0.1");
- Assert.assertEquals(innerTuple.get(1), "127.0.0.2");
- innerTuple = (Tuple) ((Tuple) t.get(9)).get(0);
- Assert.assertEquals(innerTuple.get(0), 123);
- Assert.assertEquals(innerTuple.get(1), 124);
- innerTuple = (Tuple) ((Tuple) t.get(10)).get(0);
- Assert.assertEquals(innerTuple.get(0), "text1");
- Assert.assertEquals(innerTuple.get(1), "text2");
- innerTuple = (Tuple) ((Tuple) t.get(11)).get(0);
- Assert.assertEquals(innerTuple.get(0), 1296705900000L);
- Assert.assertEquals(innerTuple.get(1), 1296792300000L);
- innerTuple = (Tuple) ((Tuple) t.get(12)).get(0);
- Assert.assertEquals(innerTuple.get(0), new DataByteArray((TimeUUIDType.instance.fromString("e23f450f-53a6-11e2-7f7f-7f7f7f7f7f7f").array())));
- Assert.assertEquals(innerTuple.get(1), new DataByteArray((TimeUUIDType.instance.fromString("e23f450f-53a6-11e2-7f7f-7f7f7f7f7f77").array())));
- innerTuple = (Tuple) ((Tuple) t.get(13)).get(0);
- Assert.assertEquals(innerTuple.get(0), new DataByteArray((UUIDType.instance.fromString("550e8400-e29b-41d4-a716-446655440000").array())));
- Assert.assertEquals(innerTuple.get(1), new DataByteArray((UUIDType.instance.fromString("550e8400-e29b-41d4-a716-446655440001").array())));
- innerTuple = (Tuple) ((Tuple) t.get(14)).get(0);
- Assert.assertEquals(innerTuple.get(0), "varchar1");
- Assert.assertEquals(innerTuple.get(1), "varchar2");
- innerTuple = (Tuple) ((Tuple) t.get(15)).get(0);
- Assert.assertEquals(innerTuple.get(0), 123);
- Assert.assertEquals(innerTuple.get(1), 124);
- }
- else
- {
- Assert.fail("Failed to get data for query " + initialQuery);
- }
- }
-
-}
[2/5] cassandra git commit: Merge branch 'cassandra-2.2' into
cassandra-3.0
Posted by al...@apache.org.
Merge branch 'cassandra-2.2' into cassandra-3.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/56d83513
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/56d83513
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/56d83513
Branch: refs/heads/cassandra-3.0
Commit: 56d835135bc4fd75aa715d9afd24746d52fb564a
Parents: e1fb18a def5803
Author: Aleksey Yeschenko <al...@apache.org>
Authored: Fri Oct 16 13:18:37 2015 +0100
Committer: Aleksey Yeschenko <al...@apache.org>
Committed: Fri Oct 16 13:18:37 2015 +0100
----------------------------------------------------------------------
NEWS.txt | 5 +++++
src/java/org/apache/cassandra/hadoop/pig/CqlNativeStorage.java | 2 +-
src/java/org/apache/cassandra/hadoop/pig/StorageHelper.java | 1 +
3 files changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/56d83513/NEWS.txt
----------------------------------------------------------------------