You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ga...@apache.org on 2009/10/15 22:38:13 UTC
svn commit: r825641 [1/3] - in /hadoop/pig/trunk: ./ contrib/zebra/
contrib/zebra/src/java/org/apache/hadoop/zebra/io/
contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/
contrib/zebra/src/java/org/apache/hadoop/zebra/pig/
contrib/zebra/src/java/org...
Author: gates
Date: Thu Oct 15 20:38:08 2009
New Revision: 825641
URL: http://svn.apache.org/viewvc?rev=825641&view=rev
Log:
PIG-992 Separate schema related files into a schema package.
Added:
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/schema/
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/schema/ColumnType.java
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/schema/Schema.java
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/schema/SchemaParser.jjt
Removed:
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/ColumnType.java
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/ParseException.java
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/Schema.java
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/SchemaParser.jjt
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/SimpleCharStream.java
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/TableSchemaParser.java
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/TableSchemaParserConstants.java
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/TableSchemaParserTokenManager.java
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/TableStorageParser.java
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/TableStorageParserConstants.java
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/TableStorageParserTokenManager.java
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/Token.java
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/TokenMgrError.java
Modified:
hadoop/pig/trunk/build.xml
hadoop/pig/trunk/contrib/zebra/CHANGES.txt
hadoop/pig/trunk/contrib/zebra/build-contrib.xml
hadoop/pig/trunk/contrib/zebra/build.xml
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/io/BasicTable.java
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/io/ColumnGroup.java
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/io/TableInserter.java
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/io/TableScanner.java
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/BasicTableExpr.java
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/BasicTableOutputFormat.java
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/CachedTableScanner.java
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/CompositeTableExpr.java
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/NullScanner.java
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/TableExpr.java
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/TableInputFormat.java
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/TableUnionExpr.java
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/SchemaConverter.java
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/TableLoader.java
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/TableStorer.java
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/CGSchema.java
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/Partition.java
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/Projection.java
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/SubColumnExtraction.java
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/TableStorageParser.jjt
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/TypesUtils.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestBasicTable.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestBasicTableMapSplits.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestBasicTableProjections.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestBasicTableSplits.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestCollection.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroup.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupInserters.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName1.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName2.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName3.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName4.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName5.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName6.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupOpen.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupProjections.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupReaders.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupSchemas.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupSplits.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestMap.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestMapOfRecord.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestMixedType1.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestNegative.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestNonDefaultWholeMapSplit.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestRecord.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestRecord2Map.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestRecord3Map.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestRecordMap.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestSchema.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestSimple.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestWrite.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TableMRSample.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TableMRSample1.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TableMRSample2.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TableMapReduceExample.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestBasicTableIOFormatLocalFS.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestBasicTableUnionLoader.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestBasicUnion.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestCollection.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestCollectionTableLoader.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestCollectionTableStorer.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestMapTableLoader.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestMapTableStorer.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestMapType.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestMixedType1.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestRealCluster.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestSimpleType.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableLoader.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableStorer.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestUnionMixedTypes.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestColumnGroupName.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestSchemaCollection.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestSchemaMap.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestSchemaPrimitive.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestSchemaRecord.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageCollection.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageMap.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageMisc1.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageMisc2.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageMisc3.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageRecord.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorePrimitive.java
Modified: hadoop/pig/trunk/build.xml
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/build.xml?rev=825641&r1=825640&r2=825641&view=diff
==============================================================================
--- hadoop/pig/trunk/build.xml (original)
+++ hadoop/pig/trunk/build.xml Thu Oct 15 20:38:08 2009
@@ -241,8 +241,6 @@
<javacc target="${src.gen.dot.parser.dir}/DOTParser.jj" outputdirectory="${src.gen.dot.parser.dir}" javacchome="${javacc.home}" />
<jjtree target="${src.dir}/org/apache/pig/data/parser/TextDataParser.jjt" outputdirectory="${src.gen.textdata.parser.dir}/" javacchome="${javacc.home}" />
<javacc target="${src.gen.textdata.parser.dir}/TextDataParser.jj" outputdirectory="${src.gen.textdata.parser.dir}" javacchome="${javacc.home}" />
- <javacc target="contrib/zebra/src/java/org/apache/hadoop/zebra/types/SchemaParser.jjt" outputdirectory="contrib/zebra/src/java/org/apache/hadoop/zebra/types/" javacchome="${javacc.home}" />
- <javacc target="contrib/zebra/src/java/org/apache/hadoop/zebra/types/TableStorageParser.jjt" outputdirectory="contrib/zebra/src/java/org/apache/hadoop/zebra/types/" javacchome="${javacc.home}" />
</target>
<!-- ================================================================== -->
Modified: hadoop/pig/trunk/contrib/zebra/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/CHANGES.txt?rev=825641&r1=825640&r2=825641&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/CHANGES.txt (original)
+++ hadoop/pig/trunk/contrib/zebra/CHANGES.txt Thu Oct 15 20:38:08 2009
@@ -2,6 +2,15 @@
Trunk (unreleased changes)
+ INCOMPATIBLE CHANGES
+
+ IMPROVEMENTS
+
+ PIG-992 Separate schema related files into a schema package (yanz via
+ gates)
+
+ OPTIMIZATIONS
+
BUG FIXES
PIG-918. Fix infinite loop only columns in first column group are
Modified: hadoop/pig/trunk/contrib/zebra/build-contrib.xml
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/build-contrib.xml?rev=825641&r1=825640&r2=825641&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/build-contrib.xml (original)
+++ hadoop/pig/trunk/contrib/zebra/build-contrib.xml Thu Oct 15 20:38:08 2009
@@ -56,6 +56,15 @@
<property name="hbase.jarfile" value="hbase-0.18.1.jar" />
<property name="hbase.test.jarfile" value="hbase-0.18.1-test.jar" />
+ <!-- IVY properteis set here -->
+ <property name="build.ivy.dir" location="${pig.root}/build/ivy" />
+ <property name="build.ivy.lib.dir" location="${build.ivy.dir}/lib" />
+ <property name="ivy.lib.dir" location="${build.ivy.lib.dir}/Pig"/>
+
+ <!-- javacc properties -->
+ <property name="src.gen.dir" value="${basedir}/src-gen/" />
+ <property name="src.gen.zebra.parser.dir" value="${src.gen.dir}/org/apache/hadoop/zebra/parser" />
+
<!-- convert spaces to _ so that mac os doesn't break things -->
<exec executable="sed" inputstring="${os.name}"
outputproperty="nonspace.os">
@@ -111,6 +120,7 @@
<!-- ====================================================== -->
<target name="init" depends="check-contrib" unless="skip.contrib">
<echo message="contrib: ${name}"/>
+ <mkdir dir="${src.gen.zebra.parser.dir}"/>
<mkdir dir="${build.dir}"/>
<mkdir dir="${build.classes}"/>
<mkdir dir="${build.test}"/>
@@ -123,11 +133,11 @@
<!-- ====================================================== -->
<!-- Compile a Pig contrib's files -->
<!-- ====================================================== -->
- <target name="compile" depends="init, hadoop-jar-exists" unless="skip.contrib">
+ <target name="compile" depends="init, cc-compile, hadoop-jar-exists" unless="skip.contrib">
<echo message="contrib: ${name}"/>
<javac
encoding="${build.encoding}"
- srcdir="${src.dir}"
+ srcdir="${src.dir};${src.gen.dir}"
includes="**/*.java"
excludes="**/doc-files/examples/*.java"
destdir="${build.classes}"
@@ -138,7 +148,24 @@
</javac>
</target>
+ <property name="javacc.home" location="${ivy.lib.dir}" />
+ <target name="cc-compile" depends="init, javacc-exists" description="Create and Compile Parser">
+ <javacc target="${root}/src/java/org/apache/hadoop/zebra/schema/SchemaParser.jjt" outputdirectory="${src.gen.zebra.parser.dir}" javacchome="${javacc.home}" />
+ <javacc target="${root}/src/java/org/apache/hadoop/zebra/types/TableStorageParser.jjt" outputdirectory="${src.gen.zebra.parser.dir}" javacchome="${javacc.home}" />
+ </target>
+
+ <target name="javacc-exists">
+ <fail>
+ <condition >
+ <not>
+ <available file="${javacc.home}/javacc.jar" />
+ </not>
+ </condition>
+ Not found: ${javacc.home}/javacc.jar
+ Please run the target "cc-compile" in the main build file
+ </fail>
+ </target>
<!-- ======================================================= -->
<!-- Compile a Pig contrib's example files (if available) -->
<!-- ======================================================= -->
@@ -257,6 +284,7 @@
<target name="clean">
<echo message="contrib: ${name}"/>
<delete dir="${build.dir}"/>
+ <delete dir="${src.gen.dir}" />
</target>
</project>
Modified: hadoop/pig/trunk/contrib/zebra/build.xml
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/build.xml?rev=825641&r1=825640&r2=825641&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/build.xml (original)
+++ hadoop/pig/trunk/contrib/zebra/build.xml Thu Oct 15 20:38:08 2009
@@ -38,6 +38,15 @@
</jar>
</target>
+ <target name="schema-jar" depends="compile">
+ <jar
+ jarfile="${build.dir}/schema-${zebraVersion}.jar"
+ basedir="${build.classes}"
+ includes="**/schema/**"
+ >
+ </jar>
+ </target>
+
<!-- Temp check for hadoop jar file -->
<target name="hadoop-jar-exists">
<fail message="${hadoop.jarfile} is missing. ${line.separator}
Modified: hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/io/BasicTable.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/io/BasicTable.java?rev=825641&r1=825640&r2=825641&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/io/BasicTable.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/io/BasicTable.java Thu Oct 15 20:38:08 2009
@@ -43,11 +43,11 @@
import org.apache.hadoop.io.file.tfile.Utils.Version;
import org.apache.hadoop.zebra.io.ColumnGroup.Reader.CGRangeSplit;
import org.apache.hadoop.zebra.types.CGSchema;
-import org.apache.hadoop.zebra.types.ParseException;
+import org.apache.hadoop.zebra.parser.ParseException;
import org.apache.hadoop.zebra.types.Partition;
import org.apache.hadoop.zebra.types.Projection;
-import org.apache.hadoop.zebra.types.Schema;
-import org.apache.hadoop.zebra.types.TableSchemaParser;
+import org.apache.hadoop.zebra.schema.Schema;
+import org.apache.hadoop.zebra.parser.TableSchemaParser;
import org.apache.hadoop.zebra.types.TypesUtils;
import org.apache.pig.data.Tuple;
Modified: hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/io/ColumnGroup.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/io/ColumnGroup.java?rev=825641&r1=825640&r2=825641&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/io/ColumnGroup.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/io/ColumnGroup.java Thu Oct 15 20:38:08 2009
@@ -53,10 +53,10 @@
import org.apache.hadoop.io.file.tfile.ByteArray;
import org.apache.hadoop.io.file.tfile.RawComparable;
import org.apache.hadoop.zebra.types.CGSchema;
-import org.apache.hadoop.zebra.types.ParseException;
+import org.apache.hadoop.zebra.parser.ParseException;
import org.apache.hadoop.zebra.types.Partition;
import org.apache.hadoop.zebra.types.Projection;
-import org.apache.hadoop.zebra.types.Schema;
+import org.apache.hadoop.zebra.schema.Schema;
import org.apache.hadoop.zebra.types.TypesUtils;
import org.apache.hadoop.zebra.types.TypesUtils.TupleReader;
import org.apache.hadoop.zebra.types.TypesUtils.TupleWriter;
@@ -241,12 +241,12 @@
* @throws IOException
*/
public Reader(Path path, Configuration conf) throws IOException,
- ParseException {
+ ParseException {
this(path, true, conf);
}
Reader(Path path, boolean dirty, Configuration conf) throws IOException,
- ParseException {
+ ParseException {
this.path = path;
this.conf = conf;
@@ -786,7 +786,7 @@
private boolean scannerClosed = true;
CGScanner(CGRangeSplit split, boolean closeReader) throws IOException,
- ParseException {
+ ParseException {
if (split == null) {
beginIndex = 0;
endIndex = cgindex.size();
Modified: hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/io/TableInserter.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/io/TableInserter.java?rev=825641&r1=825640&r2=825641&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/io/TableInserter.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/io/TableInserter.java Thu Oct 15 20:38:08 2009
@@ -22,7 +22,7 @@
import java.io.IOException;
import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.zebra.types.Schema;
+import org.apache.hadoop.zebra.schema.Schema;
import org.apache.pig.data.Tuple;
/**
Modified: hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/io/TableScanner.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/io/TableScanner.java?rev=825641&r1=825640&r2=825641&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/io/TableScanner.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/io/TableScanner.java Thu Oct 15 20:38:08 2009
@@ -23,7 +23,7 @@
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.zebra.types.Projection;
-import org.apache.hadoop.zebra.types.Schema;
+import org.apache.hadoop.zebra.schema.Schema;
import org.apache.pig.data.Tuple;
/**
Modified: hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/BasicTableExpr.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/BasicTableExpr.java?rev=825641&r1=825640&r2=825641&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/BasicTableExpr.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/BasicTableExpr.java Thu Oct 15 20:38:08 2009
@@ -27,8 +27,8 @@
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.zebra.io.BasicTable;
import org.apache.hadoop.zebra.io.TableScanner;
-import org.apache.hadoop.zebra.types.ParseException;
-import org.apache.hadoop.zebra.types.Schema;
+import org.apache.hadoop.zebra.parser.ParseException;
+import org.apache.hadoop.zebra.schema.Schema;
/**
* Table expression for reading a BasicTable.
@@ -111,9 +111,9 @@
reader.setProjection(projection);
} catch (ParseException e) {
throw new IOException("Projection parsing failed : "+e.getMessage());
- }
+ }
return reader.getScanner(begin, end, true);
- }
+ }
@Override
public Schema getSchema(Configuration conf) throws IOException {
Modified: hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/BasicTableOutputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/BasicTableOutputFormat.java?rev=825641&r1=825640&r2=825641&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/BasicTableOutputFormat.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/BasicTableOutputFormat.java Thu Oct 15 20:38:08 2009
@@ -28,9 +28,9 @@
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.zebra.io.BasicTable;
import org.apache.hadoop.zebra.io.TableInserter;
-import org.apache.hadoop.zebra.types.ParseException;
+import org.apache.hadoop.zebra.parser.ParseException;
import org.apache.hadoop.zebra.types.Partition;
-import org.apache.hadoop.zebra.types.Schema;
+import org.apache.hadoop.zebra.schema.Schema;
import org.apache.pig.data.Tuple;
/**
Modified: hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/CachedTableScanner.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/CachedTableScanner.java?rev=825641&r1=825640&r2=825641&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/CachedTableScanner.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/CachedTableScanner.java Thu Oct 15 20:38:08 2009
@@ -22,7 +22,7 @@
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.zebra.io.TableScanner;
import org.apache.hadoop.zebra.types.Projection;
-import org.apache.hadoop.zebra.types.Schema;
+import org.apache.hadoop.zebra.schema.Schema;
import org.apache.hadoop.zebra.types.TypesUtils;
import org.apache.pig.data.Tuple;
Modified: hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/CompositeTableExpr.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/CompositeTableExpr.java?rev=825641&r1=825640&r2=825641&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/CompositeTableExpr.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/CompositeTableExpr.java Thu Oct 15 20:38:08 2009
@@ -31,8 +31,8 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.zebra.io.BasicTable;
-import org.apache.hadoop.zebra.types.ParseException;
-import org.apache.hadoop.zebra.types.Schema;
+import org.apache.hadoop.zebra.parser.ParseException;
+import org.apache.hadoop.zebra.schema.Schema;
/**
Modified: hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/NullScanner.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/NullScanner.java?rev=825641&r1=825640&r2=825641&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/NullScanner.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/NullScanner.java Thu Oct 15 20:38:08 2009
@@ -21,9 +21,9 @@
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.zebra.io.TableScanner;
-import org.apache.hadoop.zebra.types.ParseException;
+import org.apache.hadoop.zebra.parser.ParseException;
import org.apache.hadoop.zebra.types.Projection;
-import org.apache.hadoop.zebra.types.Schema;
+import org.apache.hadoop.zebra.schema.Schema;
import org.apache.pig.data.Tuple;
/**
Modified: hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/TableExpr.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/TableExpr.java?rev=825641&r1=825640&r2=825641&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/TableExpr.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/TableExpr.java Thu Oct 15 20:38:08 2009
@@ -26,9 +26,9 @@
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.zebra.io.BasicTable;
import org.apache.hadoop.zebra.io.TableScanner;
-import org.apache.hadoop.zebra.types.ParseException;
+import org.apache.hadoop.zebra.parser.ParseException;
import org.apache.hadoop.zebra.types.Projection;
-import org.apache.hadoop.zebra.types.Schema;
+import org.apache.hadoop.zebra.schema.Schema;
/**
* Table Expression - expression to describe an input table.
Modified: hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/TableInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/TableInputFormat.java?rev=825641&r1=825640&r2=825641&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/TableInputFormat.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/TableInputFormat.java Thu Oct 15 20:38:08 2009
@@ -42,9 +42,9 @@
import org.apache.hadoop.zebra.io.BasicTable.Reader;
import org.apache.hadoop.zebra.io.BasicTable.Reader.RangeSplit;
import org.apache.hadoop.zebra.mapred.TableExpr.LeafTableInfo;
-import org.apache.hadoop.zebra.types.ParseException;
+import org.apache.hadoop.zebra.parser.ParseException;
import org.apache.hadoop.zebra.types.Projection;
-import org.apache.hadoop.zebra.types.Schema;
+import org.apache.hadoop.zebra.schema.Schema;
import org.apache.hadoop.zebra.types.TypesUtils;
import org.apache.pig.data.DataBag;
import org.apache.pig.data.DataByteArray;
Modified: hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/TableUnionExpr.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/TableUnionExpr.java?rev=825641&r1=825640&r2=825641&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/TableUnionExpr.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/mapred/TableUnionExpr.java Thu Oct 15 20:38:08 2009
@@ -31,9 +31,9 @@
import org.apache.hadoop.zebra.io.BasicTable;
import org.apache.hadoop.zebra.io.BasicTableStatus;
import org.apache.hadoop.zebra.io.TableScanner;
-import org.apache.hadoop.zebra.types.ParseException;
+import org.apache.hadoop.zebra.parser.ParseException;
import org.apache.hadoop.zebra.types.Projection;
-import org.apache.hadoop.zebra.types.Schema;
+import org.apache.hadoop.zebra.schema.Schema;
import org.apache.pig.data.Tuple;
/**
Modified: hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/SchemaConverter.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/SchemaConverter.java?rev=825641&r1=825640&r2=825641&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/SchemaConverter.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/SchemaConverter.java Thu Oct 15 20:38:08 2009
@@ -20,7 +20,7 @@
import java.util.Iterator;
-import org.apache.hadoop.zebra.types.ParseException;
+import org.apache.hadoop.zebra.parser.ParseException;
import org.apache.pig.data.DataType;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
@@ -90,14 +90,14 @@
}
public static Schema toPigSchema(
- org.apache.hadoop.zebra.types.Schema tschema)
+ org.apache.hadoop.zebra.schema.Schema tschema)
throws FrontendException {
Schema ret = new Schema();
for (String col : tschema.getColumns()) {
- org.apache.hadoop.zebra.types.Schema.ColumnSchema columnSchema =
+ org.apache.hadoop.zebra.schema.Schema.ColumnSchema columnSchema =
tschema.getColumn(col);
if (columnSchema != null) {
- ret.add(new FieldSchema(col, columnSchema.type.pigDataType()));
+ ret.add(new FieldSchema(col, columnSchema.getType().pigDataType()));
} else {
ret.add(new FieldSchema(null, null));
}
@@ -106,7 +106,7 @@
return ret;
}
- public static org.apache.hadoop.zebra.types.Schema fromPigSchema(
+ public static org.apache.hadoop.zebra.schema.Schema fromPigSchema(
Schema pschema) throws FrontendException, ParseException {
String[] colnames = new String[pschema.size()];
int i = 0;
@@ -115,6 +115,6 @@
FieldSchema fs = it.next();
colnames[i] = FieldSchemaMaker.makeColumnName(fs);
}
- return new org.apache.hadoop.zebra.types.Schema(colnames);
+ return new org.apache.hadoop.zebra.schema.Schema(colnames);
}
}
Modified: hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/TableLoader.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/TableLoader.java?rev=825641&r1=825640&r2=825641&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/TableLoader.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/TableLoader.java Thu Oct 15 20:38:08 2009
@@ -43,7 +43,7 @@
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.zebra.io.BasicTable;
import org.apache.hadoop.zebra.mapred.TableInputFormat;
-import org.apache.hadoop.zebra.types.ParseException;
+import org.apache.hadoop.zebra.parser.ParseException;
import org.apache.hadoop.zebra.types.Projection;
import org.apache.hadoop.zebra.types.TypesUtils;
import org.apache.pig.ExecType;
@@ -154,10 +154,10 @@
checkConf(storage, fileName);
Projection projection;
- org.apache.hadoop.zebra.types.Schema projectionSchema;
+ org.apache.hadoop.zebra.schema.Schema projectionSchema;
if (!fileName.contains(",")) { // one table;
- org.apache.hadoop.zebra.types.Schema tschema = BasicTable.Reader.getSchema(new Path(fileName), jobConf);
+ org.apache.hadoop.zebra.schema.Schema tschema = BasicTable.Reader.getSchema(new Path(fileName), jobConf);
try {
projection = new org.apache.hadoop.zebra.types.Projection(tschema, TableInputFormat.getProjection(jobConf));
projectionSchema = projection.getProjectionSchema();
@@ -165,9 +165,9 @@
throw new IOException("Schema parsing failed : "+e.getMessage());
}
} else { // table union;
- org.apache.hadoop.zebra.types.Schema unionSchema = new org.apache.hadoop.zebra.types.Schema();
+ org.apache.hadoop.zebra.schema.Schema unionSchema = new org.apache.hadoop.zebra.schema.Schema();
for (Path p : paths) {
- org.apache.hadoop.zebra.types.Schema schema = BasicTable.Reader.getSchema(p, jobConf);
+ org.apache.hadoop.zebra.schema.Schema schema = BasicTable.Reader.getSchema(p, jobConf);
try {
unionSchema.unionSchema(schema);
} catch (ParseException e) {
Modified: hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/TableStorer.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/TableStorer.java?rev=825641&r1=825640&r2=825641&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/TableStorer.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/TableStorer.java Thu Oct 15 20:38:08 2009
@@ -32,7 +32,7 @@
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.zebra.io.BasicTable;
import org.apache.hadoop.zebra.io.TableInserter;
-import org.apache.hadoop.zebra.types.ParseException;
+import org.apache.hadoop.zebra.parser.ParseException;
import org.apache.pig.StoreConfig;
import org.apache.pig.StoreFunc;
import org.apache.pig.backend.hadoop.executionengine.util.MapRedUtil;
@@ -141,4 +141,4 @@
}
inserter.insert(key, value);
}
-}
\ No newline at end of file
+}
Added: hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/schema/ColumnType.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/schema/ColumnType.java?rev=825641&view=auto
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/schema/ColumnType.java (added)
+++ hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/schema/ColumnType.java Thu Oct 15 20:38:08 2009
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.zebra.schema;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.file.tfile.Utils;
+import org.apache.pig.data.DataType;
+
+public enum ColumnType {
+ ANY("any") {
+ public byte pigDataType() {
+ return DataType.UNKNOWN;
+ }
+ },
+ INT("int") {
+ public byte pigDataType() {
+ return DataType.INTEGER;
+ }
+ },
+ LONG("long") {
+ public byte pigDataType() {
+ return DataType.LONG;
+ }
+ },
+ FLOAT("float") {
+ public byte pigDataType() {
+ return DataType.FLOAT;
+ }
+ },
+ DOUBLE("double") {
+ public byte pigDataType() {
+ return DataType.DOUBLE;
+ }
+ },
+ BOOL("bool") {
+ public byte pigDataType() {
+ return DataType.BOOLEAN;
+ }
+ },
+ COLLECTION("collection") {
+ public byte pigDataType() {
+ return DataType.BAG;
+ }
+ },
+ MAP("map") {
+ public byte pigDataType() {
+ return DataType.MAP;
+ }
+ },
+ RECORD("record") {
+ public byte pigDataType() {
+ return DataType.TUPLE;
+ }
+ },
+ STRING("string") {
+ public byte pigDataType() {
+ return DataType.CHARARRAY;
+ }
+ },
+ BYTES("bytes") {
+ public byte pigDataType() {
+ return DataType.BYTEARRAY;
+ }
+ };
+
+ private String name;
+
+ private ColumnType(String name) {
+ this.name = name;
+ }
+
+ public abstract byte pigDataType();
+
+ /**
+ * To get the type based on the type name string.
+ *
+ * @param name name of the type
+ *
+ * @return ColumnType Enum for the type
+ */
+ public static ColumnType getTypeByName(String name) {
+ return ColumnType.valueOf(name.toUpperCase());
+ }
+
+ public static ColumnType getTypeByPigDataType(byte dt) {
+ for (ColumnType ct : ColumnType.values()) {
+ if (ct.pigDataType() == dt) {
+ return ct;
+ }
+ }
+ return null;
+ }
+
+ public static String findTypeName(ColumnType columntype) {
+ return columntype.getName();
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public String toString() {
+ return name;
+ }
+
+ public static boolean isSchemaType(ColumnType columnType) {
+ return ((columnType == RECORD) || (columnType == MAP) || (columnType == COLLECTION));
+ }
+}
Added: hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/schema/Schema.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/schema/Schema.java?rev=825641&view=auto
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/schema/Schema.java (added)
+++ hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/schema/Schema.java Thu Oct 15 20:38:08 2009
@@ -0,0 +1,964 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.zebra.schema;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.io.StringReader;
+import java.util.HashSet;
+
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.zebra.parser.ParseException;
+import org.apache.hadoop.zebra.parser.TableSchemaParser;
+
+/**
+ * Logical schema of tabular data.
+ */
+public class Schema implements Comparable<Schema>, Writable {
+ public final static String COLUMN_DELIMITER = ",";
+
+ private static final long schemaVersion = 1L;
+
+ public static class ColumnSchema {
+ private String name;
+ private ColumnType type;
+ private Schema schema;
+ private int index; // field index in schema
+
+ /**
+ * construct a ColumnSchema for a native type
+ *
+ * @param a
+ * column name
+ * @param t
+ * native column type
+ */
+ public ColumnSchema(String a, ColumnType t) {
+ name = a;
+ type = t;
+ schema = null;
+ }
+
+ /**
+ * construct a Column schema for a RECORD column type
+ *
+ * @param a
+ * column name
+ * @param s
+ * column schema
+ */
+ public ColumnSchema(String a, Schema s) {
+ name = a;
+ type = ColumnType.RECORD;
+ schema = s;
+ }
+
+ /**
+ * access function to get the column name
+ */
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * access function to get the column type
+ */
+ public ColumnType getType() {
+ return type;
+ }
+
+ /**
+ * access function to get the column name
+ */
+ public int getIndex() {
+ return index;
+ }
+
+ /**
+ * construct a column schema for a complex column type
+ *
+ * @param a
+ * column name
+ * @param s
+ * column schema
+ * @param t
+ * complex column type
+ * @throws ParseException
+ */
+ public ColumnSchema(String a, Schema s, ColumnType t) throws ParseException {
+ if ((null != s) && !(ColumnType.isSchemaType(t))) {
+ throw new ParseException(
+ "Only a COLLECTION or RECORD or MAP can have schemas.");
+ }
+ name = a;
+ schema = s;
+ type = t;
+ }
+
+ /**
+ * copy ctor
+ *
+ * @param cs
+ * source column schema
+ */
+ public ColumnSchema(ColumnSchema cs) {
+ name = cs.name;
+ type = cs.type;
+ schema = cs.schema;
+ }
+
+ /**
+ * Compare two field schema for equality
+ *
+ * @param fschema
+ * @param fother
+ * @return true if ColumnSchema are equal, false otherwise
+ */
+ public static boolean equals(ColumnSchema fschema, ColumnSchema fother) {
+ if (fschema == null) {
+ return false;
+ }
+
+ if (fother == null) {
+ return false;
+ }
+
+ if (fschema.type != fother.type) {
+ return false;
+ }
+
+ if ((fschema.name == null) && (fother.name == null)) {
+ // good
+ }
+ else if ((fschema.name != null) && (fother.name == null)) {
+ return false;
+ }
+ else if ((fschema.name == null) && (fother.name != null)) {
+ return false;
+ }
+ else if (!fschema.name.equals(fother.name)) {
+ return false;
+ }
+
+ if (ColumnType.isSchemaType(fschema.type)) {
+ // Don't do the comparison if both embedded schemas are
+ // null. That will cause Schema.equals to return false,
+ // even though we want to view that as true.
+ if (!(fschema.schema == null && fother.schema == null)) {
+ // compare recursively using schema
+ if (!fschema.schema.equals(fother.schema)) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ /**
+ * string representation of the schema
+ */
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ if (name != null) {
+ sb.append(name);
+ }
+ sb.append(ColumnType.findTypeName(type));
+
+ if (schema != null) {
+ sb.append("(");
+ sb.append(schema.toString());
+ sb.append(")");
+ }
+
+ return sb.toString();
+ }
+
+ /*
+ * Returns the schema for the next level structure, in record, collection
+ * and map.
+ */
+ public Schema getSchema() {
+ return schema;
+ }
+ }
+
+ /*
+ * helper class to parse a column name string one section at a time and find
+ * the required type for the parsed part.
+ */
+ public static class ParsedName {
+ public String mName;
+ int mKeyOffset; // the offset where the keysstring starts
+ public ColumnType mDT = ColumnType.ANY; // parent's type
+
+ public ParsedName() {
+ }
+
+ public void setName(String name) {
+ mName = name;
+ }
+
+ public void setName(String name, ColumnType pdt) {
+ mName = name;
+ mDT = pdt;
+ }
+
+ void setName(String name, ColumnType pdt, int keyStrOffset) {
+ this.setName(name, pdt);
+ mKeyOffset = keyStrOffset;
+ }
+
+ void setDT(ColumnType dt) {
+ mDT = dt;
+ }
+
+ ColumnType getDT() {
+ return mDT;
+ }
+
+ String getName() {
+ return mName;
+ }
+
+ public String parseName(Schema.ColumnSchema fs) throws ParseException {
+ // parse one sector of a fq name, also checks sanity of MAP and RECORD
+ // fields
+ int fieldIndex, hashIndex;
+ fieldIndex = mName.indexOf('.');
+ hashIndex = mName.indexOf('#');
+ String prefix;
+ if (hashIndex != -1 && fieldIndex == -1) {
+ if (fs.type != ColumnType.MAP)
+ throw new ParseException(mName + " : is not of type MAP");
+ prefix = mName.substring(0, hashIndex);
+ setName(mName.substring(hashIndex + 1), ColumnType.MAP);
+ }
+ else if (hashIndex == -1 && fieldIndex != -1) {
+ if (fs.type != ColumnType.RECORD)
+ throw new ParseException(mName + " : is not of type RECORD");
+ prefix = mName.substring(0, fieldIndex);
+ setName(mName.substring(fieldIndex + 1), ColumnType.RECORD);
+ }
+ else if (hashIndex != -1 && fieldIndex != -1) {
+ if (hashIndex < fieldIndex) {
+ if (fs.type != ColumnType.MAP)
+ throw new ParseException(mName + " : is not of type MAP");
+ prefix = mName.substring(0, hashIndex);
+ setName(mName.substring(hashIndex + 1), ColumnType.MAP);
+ }
+ else {
+ if (fs.type != ColumnType.RECORD)
+ throw new ParseException(mName + " : is not of type RECORD");
+ prefix = mName.substring(0, fieldIndex);
+ setName(mName.substring(fieldIndex + 1), ColumnType.RECORD);
+ }
+ }
+ else {
+ prefix = mName; // no hash or subfield contained
+ mDT = ColumnType.ANY;
+ }
+ return prefix;
+ }
+ }
+
+ private ArrayList<ColumnSchema> mFields;
+ private HashMap<String, ColumnSchema> mNames;
+ private boolean dupColNameAllowed;
+
+ /**
+ * Constructor - schema for empty schema (zero-column) .
+ */
+ public Schema() {
+ init();
+ }
+
+ /**
+ * Constructor - create a schema from a string representation.
+ *
+ * @param schema
+ * A string representation of the schema. For this version, the
+ * schema string is simply a comma separated list of column names. Of
+ * course, comma (,) and space characters are illegal in column
+ * names. To maintain forward compatibility, please use only
+ * alpha-numeric characters in column names.
+ */
+ public Schema(String schema) throws ParseException {
+ init(schema);
+ }
+
+ public Schema(String schema, boolean dupAllowed) throws ParseException {
+ dupColNameAllowed = dupAllowed;
+ init(schema);
+ }
+
+ public Schema(ColumnSchema fs) throws ParseException {
+ init();
+ add(fs);
+ }
+
+ /**
+ * Constructor - create a schema from an array of column names.
+ *
+ * @param columns
+ * An array of column names. To maintain forward compatibility,
+ * please use only alpha-numeric characters in column names.
+ */
+ public Schema(String[] columns) throws ParseException {
+ init(columns);
+ }
+
+ /**
+ * add a column
+ *
+ * @param f
+ * Column to be added to the schema
+ */
+ public void add(ColumnSchema f) throws ParseException {
+ add(f, false);
+ }
+
+ private void add(ColumnSchema f, boolean dupAllowed) throws ParseException {
+ if (f == null) {
+ mFields.add(null);
+ return;
+ }
+ f.index = mFields.size();
+ mFields.add(f);
+ if (null != f && null != f.name) {
+ if (mNames.put(f.name, f) != null && !dupAllowed && !dupColNameAllowed)
+ throw new ParseException("Duplicate field name: " + f.name);
+ }
+ }
+
+ /**
+ * get a column by name
+ */
+ public ColumnSchema getColumn(String name) {
+ return mNames.get(name);
+ }
+
+ /**
+ * Get the names of the individual columns.
+ *
+ * @return An array of the column names.
+ */
+ public String[] getColumns() {
+ String[] result = new String[mFields.size()];
+ for (int i = 0; i < mFields.size(); i++) {
+ ColumnSchema cs = mFields.get(i);
+ if (cs != null) {
+ result[i] = mFields.get(i).name;
+ }
+ else {
+ result[i] = null;
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Get a particular column's schema
+ */
+ public ColumnSchema getColumn(int index) {
+ return mFields.get(index);
+ }
+
+ public String getColumnName(int index) {
+ if (mFields.get(index) == null) return null;
+ return mFields.get(index).name;
+ }
+
+ /**
+ * Get the names and types of the individual columns.
+ *
+ * @return An array of the column names.
+ */
+ public String[] getTypedColumns() {
+ String[] result = new String[mFields.size()];
+ for (int i = 0; i < mFields.size(); i++)
+ result[i] = mFields.get(i).name + ":" + mFields.get(i).toString();
+ return result;
+ }
+
+ /**
+ * Get the index of the column for the input column name.
+ *
+ * @param name
+ * input column name.
+ * @return The column index if the name is valid; -1 otherwise.
+ */
+ public int getColumnIndex(String name) {
+ ColumnSchema fs;
+ if ((fs = mNames.get(name)) != null) return fs.index;
+ return -1;
+ }
+
+ /**
+ * Get the number of columns as defined in the schema.
+ *
+ * @return The number of columns as defined in the schema.
+ */
+ public int getNumColumns() {
+ return mFields.size();
+ }
+
+ /**
+ * Parse a schema string and create a schema object.
+ *
+ * @param schema
+ * comma separated schema string.
+ * @return Schema object
+ */
+ public static Schema parse(String schema) throws ParseException {
+ Schema s = new Schema();
+ s.init(schema);
+ return s;
+ }
+
+ /**
+ * Convert the schema to a String.
+ *
+ * @return the string representation of the schema.
+ */
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ try {
+ stringifySchema(sb, this, ColumnType.COLLECTION, true);
+ }
+ catch (Exception fee) {
+ throw new RuntimeException("PROBLEM PRINTING SCHEMA");
+ }
+ return sb.toString();
+ }
+
+ /**
+ * This is used for building up output string type can only be COLLECTION or
+ * RECORD or MAP
+ */
+ private void stringifySchema(StringBuilder sb, Schema schema,
+ ColumnType type, boolean top) {
+ if (!top) {
+ if (type == ColumnType.RECORD) {
+ sb.append("(");
+ }
+ else if (type == ColumnType.COLLECTION) {
+ sb.append("(");
+ }
+ else if (type == ColumnType.MAP) {
+ sb.append("(");
+ }
+ }
+ boolean isFirst = true;
+ for (int i = 0; i < schema.getNumColumns(); i++) {
+
+ if (!isFirst) {
+ sb.append(",");
+ }
+ else {
+ isFirst = false;
+ }
+
+ ColumnSchema fs = schema.getColumn(i);
+
+ if (fs == null) {
+ continue;
+ }
+
+ if (fs.name != null && !fs.name.equals("")) {
+ sb.append(fs.name);
+ sb.append(":");
+ }
+
+ sb.append(ColumnType.findTypeName(fs.type));
+ if ((fs.type == ColumnType.RECORD) || (fs.type == ColumnType.MAP)
+ || (fs.type == ColumnType.COLLECTION)) {
+ // safety net
+ if (this != fs.schema) {
+ stringifySchema(sb, fs.schema, fs.type, false);
+ }
+ else {
+ throw new IllegalArgumentException("Schema refers to itself "
+ + "as inner schema");
+ }
+ }
+ }
+ if (!top) {
+ if (type == ColumnType.RECORD) {
+ sb.append(")");
+ }
+ else if (type == ColumnType.COLLECTION) {
+ sb.append(")");
+ }
+ else if (type == ColumnType.MAP) {
+ sb.append(")");
+ }
+ }
+ }
+
+ /**
+ * Normalize the schema string.
+ *
+ * @param value
+ * the input string representation of the schema.
+ * @return the normalized string representation.
+ */
+ public static String normalize(String value) {
+ String result = new String();
+
+ if (value == null || value.trim().isEmpty()) return result;
+
+ StringBuilder sb = new StringBuilder();
+ String[] parts = value.trim().split(COLUMN_DELIMITER);
+ for (int nx = 0; nx < parts.length; nx++) {
+ if (nx > 0) sb.append(COLUMN_DELIMITER);
+ sb.append(parts[nx].trim());
+ }
+ return sb.toString();
+ }
+
+ /**
+ * @see Comparable#compareTo(Object)
+ */
+ @Override
+ public int compareTo(Schema other) {
+ if (this.mFields.size() != other.mFields.size()) {
+ return this.mFields.size() - other.mFields.size();
+ }
+ int ret = 0;
+ for (int nx = 0; nx < this.mFields.size(); nx++) {
+ if (mFields.get(nx).schema == null
+ && other.mFields.get(nx).schema != null) return -1;
+ else if (mFields.get(nx).schema != null
+ && other.mFields.get(nx).schema == null) return 1;
+ else if (mFields.get(nx).schema == null
+ && other.mFields.get(nx).schema == null) return 0;
+ ret = mFields.get(nx).schema.compareTo(other.mFields.get(nx).schema);
+ if (ret != 0) {
+ return ret;
+ }
+ }
+ return 0;
+ }
+
+ /**
+ * @see Object#equals(Object)
+ */
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) return true;
+ if (obj == null) return false;
+ if (getClass() != obj.getClass()) return false;
+ Schema other = (Schema) obj;
+
+ if (mFields.size() != other.mFields.size()) return false;
+
+ Iterator<ColumnSchema> i = mFields.iterator();
+ Iterator<ColumnSchema> j = other.mFields.iterator();
+
+ while (i.hasNext()) {
+
+ ColumnSchema myFs = i.next();
+ ColumnSchema otherFs = j.next();
+
+ if ((myFs.name == null) && (otherFs.name == null)) {
+ // good
+ }
+ else if ((myFs.name != null) && (otherFs.name == null)) {
+ return false;
+ }
+ else if ((myFs.name == null) && (otherFs.name != null)) {
+ return false;
+ }
+ else if (!myFs.name.equals(otherFs.name)) {
+ return false;
+ }
+
+ if (myFs.type != otherFs.type) {
+ return false;
+ }
+
+ // Compare recursively using field schema
+ if (!ColumnSchema.equals(myFs, otherFs)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * @see Writable#readFields(DataInput)
+ */
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ long version = org.apache.hadoop.io.file.tfile.Utils.readVLong(in);
+
+ if (version > schemaVersion)
+ throw new IOException("Schema version is newer than that in software.");
+
+ // check-ups are needed for future versions for backward-compatibility
+ String strSchema = org.apache.hadoop.io.file.tfile.Utils.readString(in);
+ try {
+ init(strSchema);
+ }
+ catch (Exception e) {
+ throw new IOException(e.getMessage());
+ }
+ }
+
+ /**
+ * @see Writable#write(DataOutput)
+ */
+ @Override
+ public void write(DataOutput out) throws IOException {
+ org.apache.hadoop.io.file.tfile.Utils.writeVLong(out, schemaVersion);
+ org.apache.hadoop.io.file.tfile.Utils.writeString(out, toString());
+ }
+
+ private void init(String[] columnNames) throws ParseException {
+ // the arg must be of type or they will be treated as the default type
+ mFields = new ArrayList<ColumnSchema>();
+ mNames = new HashMap<String, ColumnSchema>();
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < columnNames.length; i++) {
+ if (columnNames[i].contains(COLUMN_DELIMITER))
+ throw new ParseException("Column name should not contain "
+ + COLUMN_DELIMITER);
+ if (i > 0) sb.append(",");
+ sb.append(columnNames[i]);
+ }
+ TableSchemaParser parser =
+ new TableSchemaParser(new StringReader(sb.toString()));
+ parser.RecordSchema(this);
+ }
+
+ private void init() {
+ mFields = new ArrayList<ColumnSchema>();
+ mNames = new HashMap<String, ColumnSchema>();
+ }
+
+ private void init(String columnString) throws ParseException {
+ String trimmedColumnStr;
+ if (columnString == null || (trimmedColumnStr = columnString.trim()).isEmpty()) {
+ init();
+ return;
+ }
+
+ String[] parts = trimmedColumnStr.split(COLUMN_DELIMITER);
+ for (int nx = 0; nx < parts.length; nx++) {
+ parts[nx] = parts[nx].trim();
+ }
+ init(parts);
+ }
+
+ /**
+ * Get a projection's schema
+ */
+ public Schema getProjectionSchema(String[] projcols,
+ HashMap<Schema.ColumnSchema, HashSet<String>> keysmap)
+ throws ParseException {
+ int ncols = projcols.length;
+ Schema result = new Schema();
+ ColumnSchema cs, mycs;
+ String keysStr;
+ String[] keys;
+ ParsedName pn = new ParsedName();
+ HashSet<String> keyentries;
+ for (int i = 0; i < ncols; i++) {
+ pn.setName(projcols[i]);
+ if ((cs = getColumnSchemaOnParsedName(pn)) != null) {
+ mycs = new ColumnSchema(pn.mName, cs.schema, cs.type);
+ result.add(mycs, true);
+ if (pn.mDT == ColumnType.MAP) {
+ keysStr = projcols[i].substring(pn.mKeyOffset);
+ if (!keysStr.startsWith("{") || !keysStr.endsWith("}"))
+ throw new ParseException("Invalid map key specification in "
+ + projcols[i]);
+ keysStr = keysStr.substring(1, keysStr.length() - 1);
+ if ((keysStr.indexOf('{') != -1) || (keysStr.indexOf('}') != -1)
+ || (keysStr.indexOf('.') != -1) || (keysStr.indexOf('#') != -1))
+ throw new ParseException("Invalid map key specification in "
+ + projcols[i]);
+ keys = keysStr.split("\\|");
+ if ((keyentries = keysmap.get(mycs)) == null) {
+ keyentries = new HashSet<String>();
+ keysmap.put(mycs, keyentries);
+ }
+ for (int j = 0; j < keys.length; j++) {
+ keyentries.add(keys[j]);
+ }
+ }
+ }
+ else result.add(null);
+ }
+ return result;
+ }
+
+ /**
+ * Get a column's schema
+ *
+ * @param name
+ * column name
+ * @return Column schema for the named column
+ */
+ public ColumnSchema getColumnSchema(String name) throws ParseException {
+ int hashIndex, fieldIndex;
+ String currentName = name, prefix;
+ Schema currentSchema = this;
+ ColumnSchema fs = null;
+
+ while (true) {
+ /**
+ * this loop is necessary because the top columns in schema may contain .s
+ * and #s, particularly for column group schemas
+ */
+ fieldIndex = currentName.lastIndexOf('.');
+ hashIndex = currentName.lastIndexOf('#');
+ if (fieldIndex == -1 && hashIndex == -1) {
+ break;
+ }
+ else if (hashIndex != -1 && (fieldIndex == -1 || hashIndex > fieldIndex)) {
+ currentName = currentName.substring(0, hashIndex);
+ if (getColumn(currentName) != null) break;
+ }
+ else {
+ currentName = currentName.substring(0, fieldIndex);
+ if (getColumn(currentName) != null) break;
+ }
+ }
+
+ currentName = name;
+ while (true) {
+ if (fieldIndex == -1 && hashIndex == -1) {
+ fs = currentSchema.getColumn(currentName);
+ return fs;
+ }
+ else if (hashIndex != -1 && (fieldIndex == -1 || hashIndex < fieldIndex)) {
+ prefix = currentName.substring(0, hashIndex);
+ fs = currentSchema.getColumn(prefix);
+ if (fs == null)
+ throw new ParseException("Column " + name + " does not exist");
+ currentSchema = fs.schema;
+ if (fs.type != ColumnType.MAP)
+ throw new ParseException(name + " is not of type MAP");
+ fs = currentSchema.getColumn(0);
+ return fs;
+ }
+ else {
+ prefix = currentName.substring(0, fieldIndex);
+ if ((fs = currentSchema.getColumn(prefix)) == null)
+ throw new ParseException("Column " + name + " does not exist");
+ currentSchema = fs.schema;
+ if (fs.type != ColumnType.RECORD && fs.type != ColumnType.COLLECTION)
+ throw new ParseException(name
+ + " is not of type RECORD or COLLECTION");
+ currentName = currentName.substring(fieldIndex + 1);
+ if (currentName.length() == 0)
+ throw new ParseException("Column " + name
+ + " does not have field after the record field separator '.'");
+ }
+ fieldIndex = currentName.indexOf('.');
+ hashIndex = currentName.indexOf('#');
+ }
+ }
+
+ /**
+ * Get a subcolumn's schema and move the name just parsed into the next subtype
+ *
+ * @param pn
+ * The name of subcolumn to be parsed. On return it contains the
+ * subcolumn at the next level after parsing
+ *
+ * @return the discovered Column Schema for the subcolumn
+ */
+ public ColumnSchema getColumnSchemaOnParsedName(ParsedName pn)
+ throws ParseException {
+ int hashIndex, fieldIndex, offset = 0;
+ String name = pn.mName;
+
+ /**
+ * strip of any possible type specs
+ */
+ String currentName = name, prefix;
+ Schema currentSchema = this;
+ ColumnSchema fs = null;
+
+ while (true) {
+ /**
+ * this loop is necessary because the top columns in schema may contain .s
+ * and #s, particularly for column group schemas
+ */
+ fieldIndex = currentName.lastIndexOf('.');
+ hashIndex = currentName.lastIndexOf('#');
+ if (fieldIndex == -1 && hashIndex == -1) {
+ break;
+ }
+ else if (hashIndex != -1 && (fieldIndex == -1 || hashIndex > fieldIndex)) {
+ currentName = currentName.substring(0, hashIndex);
+ if ((fs = getColumn(currentName)) != null) {
+ if (fs.type != ColumnType.MAP)
+ throw new ParseException(name + " is not of type MAP");
+ offset += hashIndex;
+ pn.setName(name.substring(0, hashIndex), ColumnType.MAP,
+ hashIndex + 1);
+ return fs;
+ }
+ }
+ else {
+ currentName = currentName.substring(0, fieldIndex);
+ if (getColumn(currentName) != null) break;
+ }
+ }
+
+ currentName = name;
+ ColumnType ct = ColumnType.ANY;
+
+ while (true) {
+ if (fieldIndex == -1 && hashIndex == -1) {
+ offset += currentName.length();
+ pn.setName(name.substring(0, offset), ct);
+ fs = currentSchema.getColumn(currentName);
+ return fs;
+ }
+ else if (hashIndex != -1 && (fieldIndex == -1 || hashIndex < fieldIndex)) {
+ prefix = currentName.substring(0, hashIndex);
+ fs = currentSchema.getColumn(prefix);
+ if (fs == null)
+ throw new ParseException("Column " + name + " does not exist");
+ currentSchema = fs.schema;
+ if (fs.type != ColumnType.MAP)
+ throw new ParseException(name + " is not of type MAP");
+ offset += hashIndex;
+ pn.setName(name.substring(0, offset), ColumnType.MAP, offset + 1);
+ return fs;
+ }
+ else {
+ prefix = currentName.substring(0, fieldIndex);
+ if ((fs = currentSchema.getColumn(prefix)) == null)
+ throw new ParseException("Column " + name + " does not exist");
+ currentSchema = fs.schema;
+ if (fs.type != ColumnType.RECORD && fs.type != ColumnType.COLLECTION)
+ throw new ParseException(name
+ + " is not of type RECORD or COLLECTION");
+ currentName = currentName.substring(fieldIndex + 1);
+ if (currentName.length() == 0)
+ throw new ParseException("Column " + name
+ + " does not have field after the record field separator '.'");
+ offset += fieldIndex + 1;
+ ct = fs.type;
+ }
+ fieldIndex = currentName.indexOf('.');
+ hashIndex = currentName.indexOf('#');
+ }
+ }
+
+ /**
+ * find the most fitting subcolumn containing the name: the parsed name is set
+ * after the field name plus any possible separator of '.' or '#'.
+ *
+ * This is used to help discover the most fitting column schema in multiple
+ * CG schemas.
+ *
+ * For instance, if pn contains a name of r.r1.f11 and current schema has
+ * r.r1:record(f11:int, f12), it will return f11's column schema, and pn
+ * is set at "f12".
+ *
+ * @param pn
+ * The name of subcolumn to be parsed. On return it contains the
+ * subcolumn at the next level after parsing
+ *
+ * @return the discovered Column Schema for the subcolumn
+ */
+ public ColumnSchema getColumnSchema(ParsedName pn) throws ParseException {
+ int maxlen = 0, size = getNumColumns(), len;
+ Schema.ColumnSchema fs = null;
+ String name = pn.getName(), fname;
+ boolean whole = false, record = false, map = false, tmprecord = false, tmpmap =
+ false;
+ for (int i = 0; i < size; i++) {
+ fname = getColumnName(i);
+ if ((whole = name.equals(fname))
+ || (tmprecord = name.startsWith(fname + "."))
+ || (tmpmap = name.startsWith(fname + "#"))) {
+ len = fname.length();
+ if (len > maxlen) {
+ maxlen = len;
+ record = tmprecord;
+ map = tmpmap;
+ fs = getColumn(i);
+ if (whole) break;
+ }
+ }
+ }
+ if (fs != null) {
+ name = name.substring(maxlen);
+ if (record) {
+ if (fs.type != ColumnType.RECORD && fs.type != ColumnType.COLLECTION)
+ throw new ParseException(name
+ + " is not of type RECORD or COLLECTION");
+ name = name.substring(1);
+ pn.setName(name, fs.type);
+ }
+ else if (map) {
+ if (fs.type != ColumnType.MAP)
+ throw new ParseException(name + " is not of type MAP");
+ name = name.substring(1);
+ pn.setName(name, ColumnType.MAP);
+ }
+ else pn.setName(name, ColumnType.ANY);
+ return fs;
+ }
+ else return null;
+ }
+
+ /**
+ * union compatible schemas. Exception will be thrown if a name appears in
+ * multiple schemas but the types are different.
+ */
+ public void unionSchema(Schema other) throws ParseException {
+ int size = other.getNumColumns();
+ ColumnSchema fs, otherfs;
+ for (int i = 0; i < size; i++) {
+ otherfs = other.getColumn(i);
+ if (otherfs == null) continue;
+ fs = getColumn(otherfs.name);
+ if (fs == null) add(otherfs);
+ else {
+ if (!ColumnSchema.equals(fs, otherfs))
+ throw new ParseException("Different types of column " + fs.name
+ + " in uioned tables");
+ }
+ }
+ }
+
+ /**
+ * return untyped schema string for projection
+ */
+ public String toProjectionString() {
+ String result = new String();
+ ColumnSchema fs;
+ for (int i = 0; i < mFields.size(); i++) {
+ if (i > 0) result += ",";
+ if ((fs = mFields.get(i)) != null) result += fs.name;
+ }
+ return result;
+ }
+}
Added: hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/schema/SchemaParser.jjt
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/schema/SchemaParser.jjt?rev=825641&view=auto
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/schema/SchemaParser.jjt (added)
+++ hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/schema/SchemaParser.jjt Thu Oct 15 20:38:08 2009
@@ -0,0 +1,331 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+options {
+ STATIC = false ;
+ IGNORE_CASE = true;
+}
+
+PARSER_BEGIN(TableSchemaParser)
+package org.apache.hadoop.zebra.parser;
+import java.io.*;
+import org.apache.hadoop.zebra.schema.Schema;
+import org.apache.hadoop.zebra.schema.ColumnType;
+
+public class TableSchemaParser {
+ public static void main( String[] args )
+ throws ParseException, TokenMgrError , java.io.FileNotFoundException, java.io.IOException {
+ TableSchemaParser parser = new TableSchemaParser( System.in ) ;
+ Schema schema = parser.RecordSchema(null);
+ System.out.println("OK");
+ FileOutputStream output = new FileOutputStream("/tmp/schema");
+ ObjectOutputStream objout = new ObjectOutputStream(output);
+ objout.writeObject(schema);
+ objout.close();
+ System.out.println(schema.toString());
+ }
+}
+PARSER_END(TableSchemaParser)
+
+// Skip all the new lines, tabs and spaces
+SKIP : { " " | "\r" | "\t" | "\n" }
+
+// Skip comments(single line and multiline)
+SKIP : {
+ <"--"(~["\r","\n"])*>
+| <"/*" (~["*"])* "*" ("*" | (~["*","/"] (~["*"])* "*"))* "/">
+}
+
+TOKEN : { <INT : "int"> }
+TOKEN : { <BOOL : "bool"> }
+TOKEN : { <LONG : "long"> }
+TOKEN : { <FLOAT : "float"> }
+TOKEN : { <DOUBLE : "double"> }
+TOKEN : { <STRING : "string"> }
+TOKEN : { <BYTES : "bytes"> }
+TOKEN : { <COLLECTION : "collection"> }
+TOKEN : { <RECORD : "record"> }
+TOKEN : { <MAP : "map"> }
+
+TOKEN:
+{
+ <#LETTER : ["a"-"z", "A"-"Z"] >
+| <#DIGIT : ["0"-"9"] >
+| <#SPECIALCHAR : ["_", ".", "#"] >
+| <IDENTIFIER: ( <LETTER> )+ ( <DIGIT> | <LETTER> | <SPECIALCHAR> )* >
+}
+
+ColumnType Type() :
+{
+ ColumnType type;
+}
+{
+ (type = BasicType() | type = CompositeType())
+ {
+ return type;
+ }
+}
+
+ColumnType CompositeType() :
+{
+ Token t = null;
+ ColumnType type;
+}
+{
+ (t = <MAP>| t = <COLLECTION>| t = <RECORD>)
+ {
+ type = ColumnType.getTypeByName(t.image);
+ return type;
+ }
+}
+
+ColumnType BasicType() :
+{
+ Token t = null;
+ ColumnType type;
+}
+{
+ (t = <INT>| t = <LONG>| t = <FLOAT>| t = <DOUBLE>| t = <STRING>| t = <BYTES>| t = <BOOL>)
+ {
+ String typeName = t.image;
+ type = ColumnType.getTypeByName(typeName);
+ return type;
+ }
+}
+
+Schema.ColumnSchema ColumnSchema() throws ParseException:
+{
+ Token t1;
+ Schema item = null;
+ Schema.ColumnSchema fs = null;
+}
+{
+ (
+ LOOKAHEAD(SchemaRecord()) fs = SchemaRecord()
+| LOOKAHEAD(SchemaCollection()) fs = SchemaCollection()
+| LOOKAHEAD(SchemaMap()) fs = SchemaMap()
+| LOOKAHEAD(AtomSchema()) fs = AtomSchema()
+ )
+ {
+ return fs;
+ }
+}
+
+Schema.ColumnSchema AtomSchema() throws ParseException :
+{
+ Token t1 = null;
+ ColumnType type = ColumnType.BYTES;
+ Schema.ColumnSchema fs;
+}
+{
+ ( ( t1 = <IDENTIFIER> [":" type = BasicType() ] )
+ {
+ fs = new Schema.ColumnSchema(t1.image, type);
+ return fs;
+ }
+|
+ {
+ return null;
+ }
+ )
+}
+
+Schema.ColumnSchema SchemaMap() throws ParseException :
+{
+ Token t1 = null;
+ Schema s;
+ Schema.ColumnSchema fs;
+}
+{
+ t1 = <IDENTIFIER> ":" <MAP> s = MapSchema()
+ {
+ fs = new Schema.ColumnSchema(t1.image, s, ColumnType.MAP);
+ return fs;
+ }
+}
+
+Schema.ColumnSchema SchemaRecord() throws ParseException :
+{
+ Token t1 = null;
+ Schema s;
+ Schema.ColumnSchema fs;
+}
+{
+ t1 = <IDENTIFIER> ":" <RECORD> "(" s = RecordSchemaInternal(null) ")"
+ {
+ fs = new Schema.ColumnSchema(t1.image, s, ColumnType.RECORD);
+ return fs;
+ }
+}
+
+Schema.ColumnSchema SchemaCollection() throws ParseException :
+{
+ Token t1 = null;
+ Schema.ColumnSchema fs = null;
+}
+{
+ t1 = <IDENTIFIER> ":" <COLLECTION> "(" fs = SchemaCollectionEntry(t1.image) ")"
+ {
+ return fs;
+ }
+}
+
+Schema.ColumnSchema SchemaCollectionEntry(String id) throws ParseException :
+{
+ Schema s = null;
+ Schema.ColumnSchema fs = null;
+}
+{
+ (
+ LOOKAHEAD(RecordSchemaInternal(null)) s= RecordSchemaInternal(null)
+|
+ fs = AnonymousColumnSchema()
+ )
+ { if (s == null) s = new Schema(fs); fs = new Schema.ColumnSchema(id, s, ColumnType.COLLECTION); return fs; }
+}
+
+Schema.ColumnSchema AnonymousColumnSchema() throws ParseException :
+{
+ Token t1;
+ Schema item = null;
+ Schema.ColumnSchema fs = null;
+}
+{
+ (
+ LOOKAHEAD(AnonymousSchemaRecord()) fs = AnonymousSchemaRecord()
+| LOOKAHEAD(AnonymousSchemaCollection()) fs = AnonymousSchemaCollection()
+| LOOKAHEAD(AnonymousSchemaMap()) fs = AnonymousSchemaMap()
+| LOOKAHEAD(AnonymousAtomSchema()) fs = AnonymousAtomSchema()
+ )
+ {
+ return fs;
+ }
+}
+
+Schema.ColumnSchema AnonymousAtomSchema() throws ParseException :
+{
+ ColumnType type = ColumnType.BYTES;
+ Schema.ColumnSchema fs;
+}
+{
+ ( ( [ type = BasicType() ] )
+ {
+ fs = new Schema.ColumnSchema(null, type);
+ return fs;
+ }
+ )
+}
+
+Schema.ColumnSchema AnonymousSchemaMap() throws ParseException :
+{
+ Schema s;
+ Schema.ColumnSchema fs;
+}
+{
+ <MAP> s = MapSchema()
+ {
+ fs = new Schema.ColumnSchema(null, s, ColumnType.MAP);
+ return fs;
+ }
+}
+
+Schema.ColumnSchema AnonymousSchemaRecord() throws ParseException :
+{
+ Token t1 = null;
+ Schema s;
+ Schema.ColumnSchema fs;
+}
+{
+ <RECORD> "(" s = RecordSchemaInternal(null) ")"
+ {
+ fs = new Schema.ColumnSchema(null, s, ColumnType.RECORD);
+ return fs;
+ }
+}
+
+Schema.ColumnSchema AnonymousSchemaCollection() throws ParseException :
+{
+ Schema s;
+ Schema.ColumnSchema fs;
+}
+{
+ ( <COLLECTION> "(" s = RecordSchemaInternal(null) ")" )
+ {
+ fs = new Schema.ColumnSchema(null, s, ColumnType.COLLECTION);
+ return fs;
+ }
+}
+
+Schema RecordSchemaInternal(Schema list) throws ParseException :
+{
+ Schema item = null;
+ if (list == null)
+ list = new Schema();
+ Schema.ColumnSchema fs = null;
+}
+{
+ (
+ (
+ fs = ColumnSchema() {list.add(fs);}
+ ( "," fs = ColumnSchema() {list.add(fs);})*
+ )
+ )
+ { return list; }
+}
+
+Schema RecordSchema(Schema list) throws ParseException :
+{
+ Schema item = null;
+ if (list == null)
+ list = new Schema();
+ Schema.ColumnSchema fs = null;
+}
+{
+ (
+ (
+ fs = ColumnSchema() {list.add(fs);}
+ ( "," fs = ColumnSchema() {list.add(fs);})* <EOF>
+ )
+ )
+ { return (list.getNumColumns() == 0 || (list.getNumColumns() == 1 && list.getColumn(0) == null) ? null : list); }
+}
+
+Schema MapSchema() throws ParseException :
+{
+ Schema item = null;
+ Schema list = new Schema();
+ Schema.ColumnSchema fs = null;
+}
+{
+ (
+ LOOKAHEAD(3)
+ (
+ "(" fs = AnonymousColumnSchema() ")"
+ )
+| {}
+ )
+ {
+ if (fs == null)
+ {
+ list.add(new Schema.ColumnSchema("", ColumnType.BYTES));
+ } else {
+ list.add(fs);
+ }
+ return list;
+ }
+}
Modified: hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/CGSchema.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/CGSchema.java?rev=825641&r1=825640&r2=825641&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/CGSchema.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/CGSchema.java Thu Oct 15 20:38:08 2009
@@ -28,7 +28,8 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.io.file.tfile.Utils.Version;
-import org.apache.hadoop.zebra.types.Schema;
+import org.apache.hadoop.zebra.schema.Schema;
+import org.apache.hadoop.zebra.parser.ParseException;
/**
* ColumnGroup Schema. This object is first written to a schema file when the
Modified: hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/Partition.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/Partition.java?rev=825641&r1=825640&r2=825641&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/Partition.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/Partition.java Thu Oct 15 20:38:08 2009
@@ -29,6 +29,11 @@
import org.apache.hadoop.io.BytesWritable;
import java.io.IOException;
import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.hadoop.zebra.schema.Schema;
+import org.apache.hadoop.zebra.parser.TableSchemaParser;
+import org.apache.hadoop.zebra.parser.TableStorageParser;
+import org.apache.hadoop.zebra.parser.ParseException;
+import org.apache.hadoop.zebra.schema.ColumnType;
/**
* Partition is a column group management class. Its responsibilities include
@@ -44,7 +49,7 @@
* This class holds the column group information as generated by
* TableStorageParser.
*/
- class PartitionInfo {
+ public class PartitionInfo {
private Map<Schema.ColumnSchema, PartitionFieldInfo> fieldMap =
new HashMap<Schema.ColumnSchema, PartitionFieldInfo>();
private Map<String, HashSet<ColumnMappingEntry>> mColMap =
@@ -207,12 +212,14 @@
Map<Schema.ColumnSchema, PartitionFieldInfo> fmap)
throws ParseException {
if (schema == null) throw new AssertionError("Interal Logic Error.");
- if (prefix == null) prefix = fs0.name;
- else if (fs0.name != null && !fs0.name.isEmpty())
- prefix += "." + fs0.name;
+ String name = fs0.getName();
+ if (prefix == null) prefix = name;
+ else if (name != null && !name.isEmpty())
+ prefix += "." + name;
Schema.ColumnSchema fs;
- for (int i = 0; i < fs0.schema.getNumColumns(); i++) {
- fs = fs0.schema.getColumn(i);
+ Schema schema0 = fs0.getSchema();
+ for (int i = 0; i < schema0.getNumColumns(); i++) {
+ fs = schema0.getColumn(i);
PartitionFieldInfo pi;
if ((pi = fmap.get(fs)) == null)
fmap.put(fs, pi = new PartitionFieldInfo());
@@ -242,19 +249,20 @@
}
}
} else {
- HashSet<ColumnMappingEntry> cms = colmap.get(prefix+"."+fs.name);
+ String name0 = fs.getName();
+ HashSet<ColumnMappingEntry> cms = colmap.get(prefix+"."+name0);
if (cms == null)
{
cms = new HashSet<ColumnMappingEntry>();
- colmap.put(prefix+"."+fs.name, cms);
+ colmap.put(prefix+"."+name0, cms);
}
pi.mCGIndex = new ColumnMappingEntry(defaultCGIndex,
schema.getNumColumns(), fs);
- pi.mCGName = prefix+"."+fs.name;
+ pi.mCGName = prefix+"."+name0;
cms.add(pi.mCGIndex);
- schema.add(new Schema.ColumnSchema(prefix + "." + fs.name,
- fs.schema, fs.type));
+ schema.add(new Schema.ColumnSchema(prefix + "." + name0,
+ fs.getSchema(), fs.getType()));
if (!pi.mSplitMaps.isEmpty()) {
for (Iterator<ColumnMappingEntry> it = pi.mSplitMaps.iterator();
it.hasNext(); )
@@ -270,20 +278,20 @@
/**
* set a MAP key split (sub)column
*/
- boolean setKeyCGIndex(Schema.ColumnSchema fs, int ri, int fi, String name, HashSet<String> keys) {
+ public boolean setKeyCGIndex(Schema.ColumnSchema fs, int ri, int fi, String name, HashSet<String> keys) {
PartitionFieldInfo pi;
if ((pi = fieldMap.get(fs)) == null) {
pi = new PartitionFieldInfo();
fieldMap.put(fs, pi);
}
- return pi.setKeyCGIndex(ri, fi, name, fs.schema.getColumn(0), keys);
+ return pi.setKeyCGIndex(ri, fi, name, fs.getSchema().getColumn(0), keys);
}
/**
* set a record field split (sub)column
*/
- boolean setCGIndex(Schema.ColumnSchema fs, int ri, int fi, String name) {
+ public boolean setCGIndex(Schema.ColumnSchema fs, int ri, int fi, String name) {
PartitionFieldInfo pi;
if ((pi = fieldMap.get(fs)) == null) {
pi = new PartitionFieldInfo();
@@ -342,16 +350,16 @@
}
}
else {
- HashSet<ColumnMappingEntry> cms = mColMap.get(fs.name);
+ HashSet<ColumnMappingEntry> cms = mColMap.get(fs.getName());
if (cms == null)
{
cms = new HashSet<ColumnMappingEntry>();
- mColMap.put(fs.name, cms);
+ mColMap.put(fs.getName(), cms);
}
ColumnMappingEntry cme = new ColumnMappingEntry(defaultCGIndex,
schema.getNumColumns(), fs);
cms.add(cme);
- setCGIndex(fs, defaultCGIndex, schema.getNumColumns(), fs.name);
+ setCGIndex(fs, defaultCGIndex, schema.getNumColumns(), fs.getName());
schema.add(fs);
for (Iterator<ColumnMappingEntry> it = pi.mSplitMaps.iterator();
it.hasNext(); )
@@ -653,7 +661,7 @@
{
if (projSchema.getColumn(i) == null)
continue;
- name = projSchema.getColumn(i).name;
+ name = projSchema.getColumn(i).getName();
pname.setName(name);
projectedKeys = (projection.getKeys() == null ? null :
projection.getKeys().get(projSchema.getColumn(i)));
@@ -673,8 +681,7 @@
if (mapentry.getValue() != null)
throw new AssertionError( "Internal Logical Error: RECORD should not have a split key map.");
cgentry = getCGEntry(cgindex.getCGIndex());
- parCol = new PartitionedColumn(i, true);
- mPCNeedTmpTuple.add(parCol);
+ parCol = new PartitionedColumn(i, false);
cgentry.addUser(parCol, name);
curCol.addChild(parCol);
} else {
@@ -939,13 +946,14 @@
CGEntry cgentry = null;
PartitionedColumn parCol;
- if (fs.schema != null && fs.schema.getNumColumns() > 0) {
+ Schema schema = fs.getSchema();
+ if (schema != null && schema.getNumColumns() > 0) {
Schema.ColumnSchema child;
pn.parseName(fs);
Schema.ParsedName oripn = new Schema.ParsedName();
- for (int i = 0; i < fs.schema.getNumColumns(); i++) {
+ for (int i = 0; i < schema.getNumColumns(); i++) {
oripn.setName(new String(pn.mName), pn.mDT);
- child = fs.schema.getColumn(i);
+ child = schema.getColumn(i);
if (getCGIndex(child) == null) {
// not a CG: go one level lower
parCol = new PartitionedColumn(i, true);
@@ -959,8 +967,7 @@
if (getSplitMap(child).isEmpty()) {
// this subtype is not MAP-split
cgentry = getCGEntry(getCGIndex(child).getCGIndex());
- parCol = new PartitionedColumn(i, true);
- mPCNeedTmpTuple.add(parCol);
+ parCol = new PartitionedColumn(i, false);
cgentry.addUser(parCol, getCGName(child));
parent.addChild(parCol);
}
@@ -1111,10 +1118,11 @@
CGEntry cgentry;
PartitionedColumn parCol;
- if (fs.schema != null && fs.schema.getNumColumns() > 0) {
+ Schema schema = fs.getSchema();
+ if (schema != null && schema.getNumColumns() > 0) {
Schema.ColumnSchema child;
- for (int i = 0; i < fs.schema.getNumColumns(); i++) {
- child = fs.schema.getColumn(i);
+ for (int i = 0; i < schema.getNumColumns(); i++) {
+ child = schema.getColumn(i);
PartitionInfo.ColumnMappingEntry cgindex = getCGIndex(child);
if (cgindex == null) {
// not a CG: go one level lower
@@ -1274,11 +1282,11 @@
defaultCGIndex);
}
- void setSplit(Schema.ColumnSchema fs, SplitType st, SplitType cst, String name, String childName, boolean splitChild) throws ParseException {
+ public void setSplit(Schema.ColumnSchema fs, SplitType st, SplitType cst, String name, String childName, boolean splitChild) throws ParseException {
mPartitionInfo.setSplit(fs, st, cst, name, childName, splitChild);
}
- boolean setCGIndex(Schema.ColumnSchema fs, int ri, int fi, String name) {
+ public boolean setCGIndex(Schema.ColumnSchema fs, int ri, int fi, String name) {
return mPartitionInfo.setCGIndex(fs, ri, fi, name);
}
Modified: hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/Projection.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/Projection.java?rev=825641&r1=825640&r2=825641&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/Projection.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/Projection.java Thu Oct 15 20:38:08 2009
@@ -19,6 +19,8 @@
import java.util.HashMap;
import java.util.HashSet;
+import org.apache.hadoop.zebra.schema.Schema;
+import org.apache.hadoop.zebra.parser.ParseException;
/**
* Projection for Table and Column Group
Modified: hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/SubColumnExtraction.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/SubColumnExtraction.java?rev=825641&r1=825640&r2=825641&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/SubColumnExtraction.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/SubColumnExtraction.java Thu Oct 15 20:38:08 2009
@@ -25,6 +25,9 @@
import java.util.HashMap;
import java.io.IOException;
import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.hadoop.zebra.schema.Schema;
+import org.apache.hadoop.zebra.schema.ColumnType;
+import org.apache.hadoop.zebra.parser.ParseException;
public class SubColumnExtraction {
@@ -64,7 +67,7 @@
fs = projection.getColumnSchema(i);
if (fs == null)
continue;
- name = fs.name;
+ name = fs.getName();
if (name == null)
continue;
if (projection.getKeys() != null)
@@ -78,7 +81,7 @@
if (fs == null)
continue; // skip non-existing field
- j = fs.index;
+ j = fs.getIndex();
if (pn.mDT == ColumnType.MAP || pn.mDT == ColumnType.RECORD || pn.mDT == ColumnType.COLLECTION)
{
// record/map subfield is expected
@@ -105,8 +108,8 @@
Schema.ParsedName pn, final int projIndex, HashSet<String> keys) throws ParseException, ExecException
{
// recursive call to get the next level schema
- if (pn.mDT != fs.type)
- throw new ParseException(fs.name+" is not of proper type.");
+ if (pn.mDT != fs.getType())
+ throw new ParseException(fs.getName()+" is not of proper type.");
String prefix;
int fieldIndex;
@@ -116,16 +119,17 @@
(pn.mDT == ColumnType.COLLECTION ? Partition.SplitType.COLLECTION :
Partition.SplitType.NONE)));
prefix = pn.parseName(fs);
+ Schema schema = fs.getSchema();
if (callerDT == Partition.SplitType.RECORD || callerDT == Partition.SplitType.COLLECTION)
{
if (keys != null)
throw new AssertionError("Internal Logical Error: empty key map expected.");
- if ((fieldIndex = fs.schema.getColumnIndex(prefix)) == -1)
+ if ((fieldIndex = schema.getColumnIndex(prefix)) == -1)
return; // skip non-existing fields
- fs = fs.schema.getColumn(fieldIndex);
+ fs = schema.getColumn(fieldIndex);
} else {
parent.setKeys(keys); // map key is set at parent which is of type MAP
- fs = fs.schema.getColumn(0); // MAP value is a singleton type!
+ fs = schema.getColumn(0); // MAP value is a singleton type!
fieldIndex = 0;
}