You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by bi...@apache.org on 2013/04/02 05:32:42 UTC
svn commit: r1463367 [2/11] - in /pig/site:
author/src/documentation/content/xdocs/ publish/ publish/docs/r0.11.1/
publish/docs/r0.11.1/images/ publish/docs/r0.11.1/skin/
publish/docs/r0.11.1/skin/css/ publish/docs/r0.11.1/skin/images/
publish/docs/r0....
Added: pig/site/publish/docs/r0.11.1/basic.html
URL: http://svn.apache.org/viewvc/pig/site/publish/docs/r0.11.1/basic.html?rev=1463367&view=auto
==============================================================================
--- pig/site/publish/docs/r0.11.1/basic.html (added)
+++ pig/site/publish/docs/r0.11.1/basic.html Tue Apr 2 03:32:39 2013
@@ -0,0 +1,12416 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<meta content="Apache Forrest" name="Generator">
+<meta name="Forrest-version" content="0.9">
+<meta name="Forrest-skin-name" content="pelt">
+<title>Pig Latin Basics</title>
+<link type="text/css" href="skin/basic.css" rel="stylesheet">
+<link media="screen" type="text/css" href="skin/screen.css" rel="stylesheet">
+<link media="print" type="text/css" href="skin/print.css" rel="stylesheet">
+<link type="text/css" href="skin/profile.css" rel="stylesheet">
+<script src="skin/getBlank.js" language="javascript" type="text/javascript"></script><script src="skin/getMenu.js" language="javascript" type="text/javascript"></script><script src="skin/fontsize.js" language="javascript" type="text/javascript"></script>
+<link rel="shortcut icon" href="">
+</head>
+<body onload="init()">
+<script type="text/javascript">ndeSetTextSize();</script>
+<div id="top">
+<!--+
+ |breadtrail
+ +-->
+<div class="breadtrail">
+<a href="http://www.apache.org/">Apache</a> > <a href="http://hadoop.apache.org/">Hadoop</a> > <a href="http://hadoop.apache.org/pig/">Pig</a><script src="skin/breadcrumbs.js" language="JavaScript" type="text/javascript"></script>
+</div>
+<!--+
+ |header
+ +-->
+<div class="header">
+<!--+
+ |start group logo
+ +-->
+<div class="grouplogo">
+<a href="http://hadoop.apache.org/"><img class="logoImage" alt="Hadoop" src="images/hadoop-logo.jpg" title="Apache Hadoop"></a>
+</div>
+<!--+
+ |end group logo
+ +-->
+<!--+
+ |start Project Logo
+ +-->
+<div class="projectlogo">
+<a href="http://hadoop.apache.org/pig/"><img class="logoImage" alt="Pig" src="images/pig-logo.gif" title="A platform for analyzing large datasets."></a>
+</div>
+<!--+
+ |end Project Logo
+ +-->
+<!--+
+ |start Search
+ +-->
+<div class="searchbox">
+<form action="http://www.google.com/search" method="get" class="roundtopsmall">
+<input value="" name="sitesearch" type="hidden"><input onFocus="getBlank (this, 'Search the site with google');" size="25" name="q" id="query" type="text" value="Search the site with google">
+ <input name="Search" value="Search" type="submit">
+</form>
+</div>
+<!--+
+ |end search
+ +-->
+<!--+
+ |start Tabs
+ +-->
+<ul id="tabs">
+<li>
+<a class="unselected" href="http://hadoop.apache.org/pig/">Project</a>
+</li>
+<li>
+<a class="unselected" href="http://wiki.apache.org/pig/">Wiki</a>
+</li>
+<li class="current">
+<a class="selected" href="index.html">Pig 0.11.1 Documentation</a>
+</li>
+</ul>
+<!--+
+ |end Tabs
+ +-->
+</div>
+</div>
+<div id="main">
+<div id="publishedStrip">
+<!--+
+ |start Subtabs
+ +-->
+<div id="level2tabs"></div>
+<!--+
+ |end Endtabs
+ +-->
+<script type="text/javascript"><!--
+document.write("Last Published: " + document.lastModified);
+// --></script>
+</div>
+<!--+
+ |breadtrail
+ +-->
+<div class="breadtrail">
+
+
+ </div>
+<!--+
+ |start Menu, mainarea
+ +-->
+<!--+
+ |start Menu
+ +-->
+<div id="menu">
+<div onclick="SwitchMenu('menu_selected_1.1', 'skin/')" id="menu_selected_1.1Title" class="menutitle" style="background-image: url('skin/images/chapter_open.gif');">Pig</div>
+<div id="menu_selected_1.1" class="selectedmenuitemgroup" style="display: block;">
+<div class="menuitem">
+<a href="index.html">Overview</a>
+</div>
+<div class="menuitem">
+<a href="start.html">Getting Started</a>
+</div>
+<div class="menupage">
+<div class="menupagetitle">Pig Latin Basics</div>
+</div>
+<div class="menuitem">
+<a href="func.html">Built In Functions</a>
+</div>
+<div class="menuitem">
+<a href="udf.html">User Defined Functions</a>
+</div>
+<div class="menuitem">
+<a href="cont.html">Control Structures</a>
+</div>
+<div class="menuitem">
+<a href="cmds.html">Shell and Utililty Commands</a>
+</div>
+<div class="menuitem">
+<a href="perf.html">Performance and Efficiency</a>
+</div>
+<div class="menuitem">
+<a href="test.html">Testing and Diagnostics</a>
+</div>
+<div class="menuitem">
+<a href="pig-index.html">Index</a>
+</div>
+</div>
+<div onclick="SwitchMenu('menu_1.2', 'skin/')" id="menu_1.2Title" class="menutitle">Miscellaneous</div>
+<div id="menu_1.2" class="menuitemgroup">
+<div class="menuitem">
+<a href="api/">API Docs</a>
+</div>
+<div class="menuitem">
+<a href="jdiff/changes.html">API Changes</a>
+</div>
+<div class="menuitem">
+<a href="https://cwiki.apache.org/confluence/display/PIG">Wiki</a>
+</div>
+<div class="menuitem">
+<a href="https://cwiki.apache.org/confluence/display/PIG/FAQ">FAQ</a>
+</div>
+<div class="menuitem">
+<a href="http://hadoop.apache.org/pig/releases.html">Release Notes</a>
+</div>
+</div>
+<div id="credit"></div>
+<div id="roundbottom">
+<img style="display: none" class="corner" height="15" width="15" alt="" src="skin/images/rc-b-l-15-1body-2menu-3menu.png"></div>
+<!--+
+ |alternative credits
+ +-->
+<div id="credit2"></div>
+</div>
+<!--+
+ |end Menu
+ +-->
+<!--+
+ |start content
+ +-->
+<div id="content">
+<div title="Portable Document Format" class="pdflink">
+<a class="dida" href="basic.pdf"><img alt="PDF -icon" src="skin/images/pdfdoc.gif" class="skin"><br>
+ PDF</a>
+</div>
+<h1>Pig Latin Basics</h1>
+<div id="front-matter">
+<div id="minitoc-area">
+<ul class="minitoc">
+<li>
+<a href="#Conventions">Conventions</a>
+</li>
+<li>
+<a href="#reserved-keywords">Reserved Keywords</a>
+</li>
+<li>
+<a href="#case-sensitivity">Case Sensitivity</a>
+</li>
+<li>
+<a href="#Data+Types+and+More">Data Types and More</a>
+<ul class="minitoc">
+<li>
+<a href="#identifiers">Identifiers</a>
+</li>
+<li>
+<a href="#relations">Relations, Bags, Tuples, Fields</a>
+</li>
+<li>
+<a href="#data-types">Data Types</a>
+</li>
+<li>
+<a href="#nulls">Nulls and Pig Latin</a>
+</li>
+<li>
+<a href="#constants">Constants</a>
+</li>
+<li>
+<a href="#expressions">Expressions</a>
+</li>
+<li>
+<a href="#Schemas">Schemas</a>
+</li>
+</ul>
+</li>
+<li>
+<a href="#artichmetic-ops">Arithmetic Operators and More</a>
+<ul class="minitoc">
+<li>
+<a href="#arithmetic">Arithmetic Operators</a>
+</li>
+<li>
+<a href="#boolops">Boolean Operators</a>
+</li>
+<li>
+<a href="#cast">Cast Operators</a>
+</li>
+<li>
+<a href="#comparison">Comparison Operators</a>
+</li>
+<li>
+<a href="#type-construction">Type Construction Operators</a>
+</li>
+<li>
+<a href="#deref">Dereference Operators</a>
+</li>
+<li>
+<a href="#disambiguate">Disambiguate Operator</a>
+</li>
+<li>
+<a href="#flatten">Flatten Operator</a>
+</li>
+<li>
+<a href="#null_operators">Null Operators</a>
+</li>
+<li>
+<a href="#sign">Sign Operators</a>
+</li>
+</ul>
+</li>
+<li>
+<a href="#Relational+Operators">Relational Operators</a>
+<ul class="minitoc">
+<li>
+<a href="#COGROUP">COGROUP</a>
+</li>
+<li>
+<a href="#cross">CROSS</a>
+</li>
+<li>
+<a href="#cube">CUBE</a>
+</li>
+<li>
+<a href="#define">DEFINE</a>
+</li>
+<li>
+<a href="#distinct">DISTINCT </a>
+</li>
+<li>
+<a href="#filter">FILTER </a>
+</li>
+<li>
+<a href="#foreach">FOREACH</a>
+</li>
+<li>
+<a href="#GROUP">GROUP</a>
+</li>
+<li>
+<a href="#import">IMPORT</a>
+</li>
+<li>
+<a href="#join-inner">JOIN (inner) </a>
+</li>
+<li>
+<a href="#join-outer">JOIN (outer) </a>
+</li>
+<li>
+<a href="#limit">LIMIT </a>
+</li>
+<li>
+<a href="#load">LOAD </a>
+</li>
+<li>
+<a href="#mapreduce">MAPREDUCE</a>
+</li>
+<li>
+<a href="#order-by">ORDER BY</a>
+</li>
+<li>
+<a href="#rank">RANK</a>
+</li>
+<li>
+<a href="#sample">SAMPLE</a>
+</li>
+<li>
+<a href="#SPLIT">SPLIT</a>
+</li>
+<li>
+<a href="#store">STORE </a>
+</li>
+<li>
+<a href="#stream">STREAM</a>
+</li>
+<li>
+<a href="#union">UNION</a>
+</li>
+</ul>
+</li>
+<li>
+<a href="#udf-statements">UDF Statements</a>
+<ul class="minitoc">
+<li>
+<a href="#define-udfs">DEFINE (UDFs, streaming)</a>
+</li>
+<li>
+<a href="#register">REGISTER</a>
+</li>
+</ul>
+</li>
+</ul>
+</div>
+</div>
+
+<!-- CONVENTIONS -->
+
+<a name="Conventions"></a>
+<h2 class="h3">Conventions</h2>
+<div class="section">
+<p>Conventions for the syntax and code examples in the Pig Latin Reference Manual are described here.</p>
+<table class="ForrestTable" cellspacing="1" cellpadding="4">
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>Convention</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>Description</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>Example</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>( )</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>Parentheses enclose one or more items.</p>
+
+<p>Parentheses are also used to indicate the tuple data type.</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>Multiple items:</p>
+
+<p>(1, abc, (2,4,6) )</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>[ ]</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>Straight brackets enclose one or more optional items.</p>
+
+<p>Straight brackets are also used to indicate the map data type. In this case <> is used to indicate optional items.</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>Optional items:</p>
+
+<p>[INNER | OUTER]</p>
+
+<p></p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>{ }</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>Curly brackets enclose two or more items, one of which is required. </p>
+
+<p>Curly brackets also used to indicate the bag data type. In this case <> is used to indicate required items.</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>Two items, one required:</p>
+
+<p>{ block | nested_block }</p>
+
+<p></p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>…</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>Horizontal ellipsis points indicate that you can repeat a portion of the code.</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>Pig Latin syntax statement:</p>
+
+<p>cat path [path …]</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>UPPERCASE</p>
+
+<p></p>
+
+<p>lowercase</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>In general, uppercase type indicates elements the system supplies.</p>
+
+<p>In general, lowercase type indicates elements that you supply.</p>
+
+<p>(These conventions are not strictly adherered to in all examples.)</p>
+
+<p>See <a href="#Case-Sensitivity">Case Sensitivity</a>
+</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>Pig Latin statement:</p>
+
+<p>a = LOAD 'data' AS (f1:int);</p>
+
+<p></p>
+
+<ul>
+
+<li>
+
+<p>LOAD, AS - Pig keywords</p>
+
+</li>
+
+<li>
+
+<p>a, f1 - aliases you supply</p>
+
+</li>
+
+<li>
+
+<p>'data' - data source you supply</p>
+
+</li>
+
+</ul>
+
+</td>
+
+</tr>
+
+</table>
+</div>
+
+
+ <!-- KEYWORDS -->
+
+<a name="reserved-keywords"></a>
+<h2 class="h3">Reserved Keywords</h2>
+<div class="section">
+<p>Pig reserved keywords are listed here.</p>
+<table class="ForrestTable" cellspacing="1" cellpadding="4">
+
+<tr>
+
+<td colspan="1" rowspan="1">
+<p>-- A </p>
+</td>
+ <td colspan="1" rowspan="1">
+<p>and, any, all, arrange, as, asc, AVG</p>
+</td>
+
+</tr>
+
+
+<tr>
+
+<td colspan="1" rowspan="1">
+<p>-- B </p>
+</td>
+ <td colspan="1" rowspan="1">
+<p>bag, BinStorage, by, bytearray </p>
+</td>
+
+</tr>
+
+
+<tr>
+
+<td colspan="1" rowspan="1">
+<p>-- C </p>
+</td>
+ <td colspan="1" rowspan="1">
+<p>cache, cat, cd, chararray, cogroup, CONCAT, copyFromLocal, copyToLocal, COUNT, cp, cross</p>
+</td>
+
+</tr>
+
+
+<tr>
+
+<td colspan="1" rowspan="1">
+<p>-- D </p>
+</td>
+ <td colspan="1" rowspan="1">
+<p>datetime, %declare, %default, define, desc, describe, DIFF, distinct, double, du, dump</p>
+</td>
+
+</tr>
+
+
+<tr>
+
+<td colspan="1" rowspan="1">
+<p>-- E </p>
+</td>
+ <td colspan="1" rowspan="1">
+<p>e, E, eval, exec, explain</p>
+</td>
+
+</tr>
+
+
+<tr>
+
+<td colspan="1" rowspan="1">
+<p>-- F </p>
+</td>
+ <td colspan="1" rowspan="1">
+<p>f, F, filter, flatten, float, foreach, full</p>
+</td>
+
+</tr>
+
+
+<tr>
+
+<td colspan="1" rowspan="1">
+<p>-- G </p>
+</td>
+ <td colspan="1" rowspan="1">
+<p>generate, group</p>
+</td>
+
+</tr>
+
+
+<tr>
+
+<td colspan="1" rowspan="1">
+<p>-- H </p>
+</td>
+ <td colspan="1" rowspan="1">
+<p>help</p>
+</td>
+
+</tr>
+
+
+<tr>
+
+<td colspan="1" rowspan="1">
+<p>-- I </p>
+</td>
+ <td colspan="1" rowspan="1">
+<p>if, illustrate, import, inner, input, int, into, is</p>
+</td>
+
+</tr>
+
+
+<tr>
+
+<td colspan="1" rowspan="1">
+<p>-- J </p>
+</td>
+ <td colspan="1" rowspan="1">
+<p>join</p>
+</td>
+
+</tr>
+
+
+<tr>
+
+<td colspan="1" rowspan="1">
+<p>-- K </p>
+</td>
+ <td colspan="1" rowspan="1">
+<p>kill</p>
+</td>
+
+</tr>
+
+
+<tr>
+
+<td colspan="1" rowspan="1">
+<p>-- L </p>
+</td>
+ <td colspan="1" rowspan="1">
+<p>l, L, left, limit, load, long, ls</p>
+</td>
+
+</tr>
+
+
+<tr>
+
+<td colspan="1" rowspan="1">
+<p>-- M </p>
+</td>
+ <td colspan="1" rowspan="1">
+<p>map, matches, MAX, MIN, mkdir, mv </p>
+</td>
+
+</tr>
+
+
+<tr>
+
+<td colspan="1" rowspan="1">
+<p>-- N </p>
+</td>
+ <td colspan="1" rowspan="1">
+<p>not, null</p>
+</td>
+
+</tr>
+
+
+<tr>
+
+<td colspan="1" rowspan="1">
+<p>-- O </p>
+</td>
+ <td colspan="1" rowspan="1">
+<p>onschema, or, order, outer, output</p>
+</td>
+
+</tr>
+
+
+<tr>
+
+<td colspan="1" rowspan="1">
+<p>-- P </p>
+</td>
+ <td colspan="1" rowspan="1">
+<p>parallel, pig, PigDump, PigStorage, pwd</p>
+</td>
+
+</tr>
+
+
+<tr>
+
+<td colspan="1" rowspan="1">
+<p>-- Q </p>
+</td>
+ <td colspan="1" rowspan="1">
+<p>quit</p>
+</td>
+
+</tr>
+
+
+<tr>
+
+<td colspan="1" rowspan="1">
+<p>-- R </p>
+</td>
+ <td colspan="1" rowspan="1">
+<p>register, right, rm, rmf, run</p>
+</td>
+
+</tr>
+
+
+<tr>
+
+<td colspan="1" rowspan="1">
+<p>-- S </p>
+</td>
+ <td colspan="1" rowspan="1">
+<p>sample, set, ship, SIZE, split, stderr, stdin, stdout, store, stream, SUM</p>
+</td>
+
+</tr>
+
+
+<tr>
+
+<td colspan="1" rowspan="1">
+<p>-- T </p>
+</td>
+ <td colspan="1" rowspan="1">
+<p>TextLoader, TOKENIZE, through, tuple</p>
+</td>
+
+</tr>
+
+
+<tr>
+
+<td colspan="1" rowspan="1">
+<p>-- U </p>
+</td>
+ <td colspan="1" rowspan="1">
+<p>union, using</p>
+</td>
+
+</tr>
+
+
+<tr>
+
+<td colspan="1" rowspan="1">
+<p>-- V, W, X, Y, Z </p>
+</td>
+ <td colspan="1" rowspan="1">
+<p>
+</p>
+</td>
+
+</tr>
+
+
+</table>
+</div>
+
+
+ <!-- ++++++++++++++++++++++++++++++++++ -->
+ <!-- CASE SENSITIVITY -->
+
+<a name="case-sensitivity"></a>
+<h2 class="h3">Case Sensitivity</h2>
+<div class="section">
+<p>The names (aliases) of relations and fields are case sensitive. The names of Pig Latin functions are case sensitive.
+ The names of parameters (see <a href="cont.html#Parameter-Sub">Parameter Substitution</a>) and all other Pig Latin keywords (see <a href="#Reserved-Keywords">Reserved Keywords</a>) are case insensitive.</p>
+<p>In the example below, note the following:</p>
+<ul>
+
+<li>
+
+<p>The names (aliases) of relations A, B, and C are case sensitive.</p>
+
+</li>
+
+<li>
+
+<p>The names (aliases) of fields f1, f2, and f3 are case sensitive.</p>
+
+</li>
+
+<li>
+
+<p>Function names PigStorage and COUNT are case sensitive.</p>
+
+</li>
+
+<li>
+
+<p>Keywords LOAD, USING, AS, GROUP, BY, FOREACH, GENERATE, and DUMP are case insensitive.
+ They can also be written as load, using, as, group, by, etc.</p>
+
+</li>
+
+<li>
+
+<p>In the FOREACH statement, the field in relation B is referred to by positional notation ($0).</p>
+
+</li>
+
+</ul>
+<p></p>
+<pre class="code">
+grunt> A = LOAD 'data' USING PigStorage() AS (f1:int, f2:int, f3:int);
+grunt> B = GROUP A BY f1;
+grunt> C = FOREACH B GENERATE COUNT ($0);
+grunt> DUMP C;
+</pre>
+</div>
+
+ <!-- ++++++++++++++++++++++++++++++++++ -->
+<!-- DATA TYPES AND MORE-->
+
+<a name="Data+Types+and+More"></a>
+<h2 class="h3">Data Types and More</h2>
+<div class="section">
+<a name="identifiers"></a>
+<h3 class="h4">Identifiers</h3>
+<p>Identifiers include the names of relations (aliases), fields, variables, and so on.
+In Pig, identifiers start with a letter and can be followed by any number of letters, digits, or underscores.</p>
+<p>Valid identifiers:</p>
+<pre class="code">
+A
+A123
+abc_123_BeX_
+</pre>
+<p></p>
+<p>Invalid identifiers: </p>
+<pre class="code">
+_A123
+abc_$
+A!B
+</pre>
+<a name="relations"></a>
+<h3 class="h4">Relations, Bags, Tuples, Fields</h3>
+<p>
+<a href="start.html#pl-Statements">Pig Latin statements</a> work with relations. A relation can be defined as follows:</p>
+<ul>
+
+<li>
+
+<p>A relation is a bag (more specifically, an outer bag).</p>
+
+</li>
+
+<li>
+
+<p>A bag is a collection of tuples. </p>
+
+</li>
+
+<li>
+
+<p>A tuple is an ordered set of fields.</p>
+
+</li>
+
+<li>
+
+<p>A field is a piece of data.</p>
+
+</li>
+
+</ul>
+<p></p>
+<p>A Pig relation is a bag of tuples. A Pig relation is similar to a table in a relational database, where the tuples in the bag correspond to the rows in a table. Unlike a relational table, however, Pig relations don't require that every tuple contain the same number of fields or that the fields in the same position (column) have the same type.</p>
+<p>Also note that relations are unordered which means there is no guarantee that tuples are processed in any particular order. Furthermore, processing may be parallelized in which case tuples are not processed according to any total ordering.</p>
+<a name="ref-relation"></a>
+<h4>Referencing Relations</h4>
+<p>Relations are referred to by name (or alias). Names are assigned by you as part of the Pig Latin statement. In this example the name (alias) of the relation is A.</p>
+<pre class="code">
+A = LOAD 'student' USING PigStorage() AS (name:chararray, age:int, gpa:float);
+DUMP A;
+(John,18,4.0F)
+(Mary,19,3.8F)
+(Bill,20,3.9F)
+(Joe,18,3.8F)
+</pre>
+<p>You an assign an alias to another alias. The new alias can be used in the place of the original alias to refer the original relation. </p>
+<pre class="code">
+ A = LOAD 'student' USING PigStorage() AS (name:chararray, age:int, gpa:float);
+ B = A;
+ DUMP B;
+ </pre>
+<a name="ref-field"></a>
+<h4>Referencing Fields</h4>
+<p>Fields are referred to by positional notation or by name (alias). </p>
+<ul>
+
+<li>
+
+<p>Positional notation is generated by the system. Positional notation is indicated with the dollar sign ($) and begins with zero (0); for example, $0, $1, $2. </p>
+
+</li>
+
+<li>
+
+<p>Names are assigned by you using schemas (or, in the case of the GROUP operator and some functions, by the system). You can use any name that is not a Pig keyword (see <a href="#identifiers">Identifiers</a> for valid name examples).</p>
+
+</li>
+
+</ul>
+<p>Given relation A above, the three fields are separated out in this table. </p>
+<table class="ForrestTable" cellspacing="1" cellpadding="4">
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>First Field</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>Second Field</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>Third Field </p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>Data type</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>chararray</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>int</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>float</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>Positional notation (generated by system) </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>$0</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>$1</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>$2</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>Possible name (assigned by you using a schema)</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>name</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>age</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>gpa</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>Field value (for the first tuple)</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>John</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>18</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>4.0</p>
+
+</td>
+
+</tr>
+
+</table>
+<p>As shown in this example when you assign names to fields (using the AS schema clause) you can still refer to the fields using positional notation. However, for debugging purposes and ease of comprehension, it is better to use field names.</p>
+<pre class="code">
+A = LOAD 'student' USING PigStorage() AS (name:chararray, age:int, gpa:float);
+X = FOREACH A GENERATE name,$2;
+DUMP X;
+(John,4.0F)
+(Mary,3.8F)
+(Bill,3.9F)
+(Joe,3.8F)
+</pre>
+<p>In this example an error is generated because the requested column ($3) is outside of the declared schema (positional notation begins with $0). Note that the error is caught before the statements are executed.</p>
+<pre class="code">
+A = LOAD 'data' AS (f1:int,f2:int,f3:int);
+B = FOREACH A GENERATE $3;
+DUMP B;
+2009-01-21 23:03:46,715 [main] ERROR org.apache.pig.tools.grunt.GruntParser - java.io.IOException:
+Out of bound access. Trying to access non-existent : 3. Schema {f1: bytearray,f2: bytearray,f3: bytearray} has 3 column(s).
+<em>etc ... </em>
+</pre>
+<a name="ref-field-complex"></a>
+<h4>Referencing Fields that are Complex Data Types</h4>
+<p>As noted, the fields in a tuple can be any data type, including the complex data types: bags, tuples, and maps. </p>
+<ul>
+
+<li>
+
+<p>Use the schemas for complex data types to name fields that are complex data types. </p>
+
+</li>
+
+<li>
+
+<p>Use the dereference operators to reference and work with fields that are complex data types.</p>
+
+</li>
+
+</ul>
+<p>In this example the data file contains tuples. A schema for complex data types (in this case, tuples) is used to load the data. Then, dereference operators (the dot in t1.t1a and t2.$0) are used to access the fields in the tuples. Note that when you assign names to fields you can still refer to these fields using positional notation.</p>
+<pre class="code">
+cat data;
+(3,8,9) (4,5,6)
+(1,4,7) (3,7,5)
+(2,5,8) (9,5,8)
+
+A = LOAD 'data' AS (t1:tuple(t1a:int, t1b:int,t1c:int),t2:tuple(t2a:int,t2b:int,t2c:int));
+
+DUMP A;
+((3,8,9),(4,5,6))
+((1,4,7),(3,7,5))
+((2,5,8),(9,5,8))
+
+X = FOREACH A GENERATE t1.t1a,t2.$0;
+
+DUMP X;
+(3,4)
+(1,3)
+(2,9)
+</pre>
+<a name="data-types"></a>
+<h3 class="h4">Data Types</h3>
+<a name="Simple+and+Complex"></a>
+<h4>Simple and Complex</h4>
+<p></p>
+<table class="ForrestTable" cellspacing="1" cellpadding="4">
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>
+<strong>Simple Types</strong>
+</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>Description</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>Example </p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>int</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>Signed 32-bit integer</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>10</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>long</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>Signed 64-bit integer</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>Data: 10L or 10l </p>
+
+<p>Display: 10L </p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>float</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>32-bit floating point</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>Data: 10.5F or 10.5f or 10.5e2f or 10.5E2F</p>
+
+<p>Display: 10.5F or 1050.0F</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>double</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>64-bit floating point</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>Data: 10.5 or 10.5e2 or 10.5E2</p>
+
+<p>Display: 10.5 or 1050.0</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>chararray</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>Character array (string) in Unicode UTF-8 format</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>hello world</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>bytearray</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>Byte array (blob)</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>boolean</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>boolean</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>true/false (case insensitive)</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>datetime</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>datetime</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>1970-01-01T00:00:00.000+00:00</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>
+<strong>Complex Types</strong>
+</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>tuple</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>An ordered set of fields.</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>(19,2)</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>bag</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>An collection of tuples.</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>{(19,2), (18,1)}</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>map</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>A set of key value pairs.</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>[open#apache]</p>
+
+</td>
+
+</tr>
+
+</table>
+<p>Note the following general observations about data types:</p>
+<ul>
+
+<li>
+
+<p>Use schemas to assign types to fields. If you don't assign types, fields default to type bytearray and implicit conversions are applied to the data depending on the context in which that data is used. For example, in relation B, f1 is converted to integer because 5 is integer. In relation C, f1 and f2 are converted to double because we don't know the type of either f1 or f2.</p>
+
+<pre class="code">
+A = LOAD 'data' AS (f1,f2,f3);
+B = FOREACH A GENERATE f1 + 5;
+C = FOREACH A generate f1 + f2;
+</pre>
+
+</li>
+
+</ul>
+<ul>
+
+<li>
+
+<p>If a schema is defined as part of a load statement, the load function will attempt to enforce the schema. If the data does not conform to the schema, the loader will generate a null value or an error.</p>
+
+<pre class="code">
+A = LOAD 'data' AS (name:chararray, age:int, gpa:float);
+</pre>
+
+</li>
+
+</ul>
+<p></p>
+<ul>
+
+<li>
+
+<p>If an explicit cast is not supported, an error will occur. For example, you cannot cast a chararray to int.</p>
+
+<pre class="code">
+A = LOAD 'data' AS (name:chararray, age:int, gpa:float);
+B = FOREACH A GENERATE (int)name;
+
+This will cause an error …</pre>
+
+</li>
+
+</ul>
+<p></p>
+<ul>
+
+<li>
+
+<p>If Pig cannot resolve incompatible types through implicit casts, an error will occur. For example, you cannot add chararray and float (see the Types Table for addition and subtraction).</p>
+
+<pre class="code">
+A = LOAD 'data' AS (name:chararray, age:int, gpa:float);
+B = FOREACH A GENERATE name + gpa;
+
+This will cause an error …</pre>
+
+</li>
+
+</ul>
+<p></p>
+<p>All data types have corresponding <a href="#schemas">schemas</a>.</p>
+<a name="tuple"></a>
+<h4>Tuple</h4>
+<p>A tuple is an ordered set of fields.</p>
+<a name="Syntax"></a>
+<h5>Syntax </h5>
+<table class="ForrestTable" cellspacing="1" cellpadding="4">
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>( field [, field …] ) </p>
+
+</td>
+
+</tr>
+
+</table>
+<a name="Terms"></a>
+<h5>Terms</h5>
+<table class="ForrestTable" cellspacing="1" cellpadding="4">
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>( )</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>A tuple is enclosed in parentheses ( ).</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>field</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>A piece of data. A field can be any data type (including tuple and bag).</p>
+
+</td>
+
+</tr>
+
+</table>
+<a name="Usage"></a>
+<h5>Usage</h5>
+<p>You can think of a tuple as a row with one or more fields, where each field can be any data type and any field may or may not have data. If a field has no data, then the following happens:</p>
+<ul>
+
+<li>
+
+<p>In a load statement, the loader will inject null into the tuple. The actual value that is substituted for null is loader specific; for example, PigStorage substitutes an empty field for null.</p>
+
+</li>
+
+<li>
+
+<p>In a non-load statement, if a requested field is missing from a tuple, Pig will inject null.</p>
+
+</li>
+
+</ul>
+<p></p>
+<p>Also see <a href="#tuple-schema">tuple schemas</a>.</p>
+<a name="Example"></a>
+<h5>Example</h5>
+<p>In this example the tuple contains three fields.</p>
+<pre class="code">(John,18,4.0F)</pre>
+<a name="bag"></a>
+<h4>Bag</h4>
+<p>A bag is a collection of tuples.</p>
+<a name="Syntax%3A+Inner+bag"></a>
+<h5>Syntax: Inner bag</h5>
+<table class="ForrestTable" cellspacing="1" cellpadding="4">
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>{ tuple [, tuple …] }</p>
+
+</td>
+
+</tr>
+
+</table>
+<a name="Terms-N106C1"></a>
+<h5>Terms</h5>
+<table class="ForrestTable" cellspacing="1" cellpadding="4">
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>{ }</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>An inner bag is enclosed in curly brackets { }.</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>tuple</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>A tuple.</p>
+
+</td>
+
+</tr>
+
+</table>
+<a name="Usage-N106F1"></a>
+<h5>Usage </h5>
+<p>Note the following about bags:</p>
+<ul>
+
+<li>
+
+<p>A bag can have duplicate tuples.</p>
+
+</li>
+
+<li>
+
+<p>A bag can have tuples with differing numbers of fields. However, if Pig tries to access a field that does not exist, a null value is substituted.</p>
+
+</li>
+
+<li>
+
+<p>A bag can have tuples with fields that have different data types. However, for Pig to effectively process bags, the schemas of the tuples within those bags should be the same. For example, if half of the tuples include chararray fields and while the other half include float fields, only half of the tuples will participate in any kind of computation because the chararray fields will be converted to null.</p>
+
+<p></p>
+
+<p>Bags have two forms: outer bag (or relation) and inner bag.</p>
+
+</li>
+
+</ul>
+<p></p>
+<p>Also see <a href="#bag-schema">bag schemas</a>.</p>
+<a name="Example%3A+Outer+Bag"></a>
+<h5>Example: Outer Bag</h5>
+<p>In this example A is a relation or bag of tuples. You can think of this bag as an outer bag.</p>
+<pre class="code">
+A = LOAD 'data' as (f1:int, f2:int, f3;int);
+DUMP A;
+(1,2,3)
+(4,2,1)
+(8,3,4)
+(4,3,3)
+</pre>
+<a name="Example%3A+Inner+Bag"></a>
+<h5>Example: Inner Bag</h5>
+<p>Now, suppose we group relation A by the first field to form relation X. </p>
+<p>In this example X is a relation or bag of tuples. The tuples in relation X have two fields. The first field is type int. The second field is type bag; you can think of this bag as an inner bag.</p>
+<pre class="code">
+X = GROUP A BY f1;
+DUMP X;
+(1,{(1,2,3)})
+(4,{(4,2,1),(4,3,3)})
+(8,{(8,3,4)})
+</pre>
+<a name="map"></a>
+<h4>Map</h4>
+<p>A map is a set of key/value pairs.</p>
+<a name="Syntax+%28%3C%3E+denotes+optional%29"></a>
+<h5>Syntax (<> denotes optional)</h5>
+<table class="ForrestTable" cellspacing="1" cellpadding="4">
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>[ key#value <, key#value …> ]</p>
+
+</td>
+
+</tr>
+
+</table>
+<a name="Terms-N1075C"></a>
+<h5>Terms</h5>
+<table class="ForrestTable" cellspacing="1" cellpadding="4">
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>[ ]</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>Maps are enclosed in straight brackets [ ].</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>#</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>Key value pairs are separated by the pound sign #.</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>key</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>Must be chararray data type. Must be a unique value.</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>value</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>Any data type (the defaults to bytearray).</p>
+
+</td>
+
+</tr>
+
+</table>
+<a name="Usage-N107B2"></a>
+<h5>Usage</h5>
+<p>Key values within a relation must be unique.</p>
+<p>Also see <a href="#map-schema">map schemas</a>.</p>
+<a name="Example-N107C3"></a>
+<h5>Example</h5>
+<p>In this example the map includes two key value pairs.</p>
+<pre class="code">[name#John,phone#5551212]</pre>
+<a name="nulls"></a>
+<h3 class="h4">Nulls and Pig Latin</h3>
+<p>In Pig Latin, nulls are implemented using the SQL definition of null as unknown or non-existent. Nulls can occur naturally in data or can be the result of an operation. </p>
+<a name="nulls-ops"></a>
+<h4>Nulls, Operators, and Functions</h4>
+<p>Pig Latin operators and functions interact with nulls as shown in this table.</p>
+<table class="ForrestTable" cellspacing="1" cellpadding="4">
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>Operator </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>Interaction </p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>Comparison operators:</p>
+
+<p>==, !=</p>
+
+<p>>, <</p>
+
+<p>>=, <=</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>If either subexpression is null, the result is null.</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>Comparison operator:</p>
+
+<p>matches </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>If either the string being matched against or the string defining the match is null, the result is null.</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>Arithmetic operators:</p>
+
+<p> + , -, *, /</p>
+
+<p>% modulo</p>
+
+<p>? : bincond</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>If either subexpression is null, the resulting expression is null.</p>
+
+<p></p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>Null operator:</p>
+
+<p>is null </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>If the tested value is null, returns true; otherwise, returns false (see <a href="#null_operators">Null Operators</a>).</p>
+
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>Null operator:</p>
+
+<p>is not null</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>If the tested value is not null, returns true; otherwise, returns false (see <a href="#null_operators">Null Operators</a>).</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>Dereference operators:</p>
+
+<p>tuple (.) or map (#)</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>If the de-referenced tuple or map is null, returns null.</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>Operators:</p>
+
+<p>COGROUP, GROUP, JOIN</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>These operators handle nulls differently (see examples below).</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>Function:</p>
+
+<p>COUNT_STAR</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>This function counts all values, including nulls.</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>Cast operator</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>Casting a null from one type to another type results in a null.</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>Functions:</p>
+
+<p>AVG, MIN, MAX, SUM, COUNT</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>These functions ignore nulls. </p>
+
+</td>
+
+</tr>
+
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>Function:</p>
+
+<p>CONCAT</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>If either subexpression is null, the resulting expression is null.</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>Function:</p>
+
+<p>SIZE</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>If the tested object is null, returns null.</p>
+
+</td>
+
+</tr>
+
+</table>
+<p>For Boolean subexpressions, note the results when nulls are used with these operators:</p>
+<ul>
+
+<li>
+
+<p>FILTER operator – If a filter expression results in null value, the filter does not pass them through (if X is null, !X is also null, and the filter will reject both).</p>
+
+</li>
+
+<li>
+
+<p>Bincond operator – If a Boolean subexpression results in null value, the resulting expression is null (see the interactions above for Arithmetic operators)</p>
+
+</li>
+
+</ul>
+<a name="nulls-constants"></a>
+<h4>Nulls and Constants</h4>
+<p>Nulls can be used as constant expressions in place of expressions of any type.</p>
+<p>In this example a and null are projected.</p>
+<pre class="code">
+A = LOAD 'data' AS (a, b, c).
+B = FOREACH A GENERATE a, null;
+</pre>
+<p>In this example of an outer join, if the join key is missing from a table it is replaced by null.</p>
+<pre class="code">
+A = LOAD 'student' AS (name: chararray, age: int, gpa: float);
+B = LOAD 'votertab10k' AS (name: chararray, age: int, registration: chararray, donation: float);
+C = COGROUP A BY name, B BY name;
+D = FOREACH C GENERATE FLATTEN((IsEmpty(A) ? null : A)), FLATTEN((IsEmpty(B) ? null : B));
+</pre>
+<p>Like any other expression, null constants can be implicitly or explicitly cast. </p>
+<p>In this example both a and null will be implicitly cast to double.</p>
+<pre class="code">
+A = LOAD 'data' AS (a, b, c).
+B = FOREACH A GENERATE a + null;
+</pre>
+<p>In this example both a and null will be cast to int, a implicitly, and null explicitly.</p>
+<pre class="code">
+A = LOAD 'data' AS (a, b, c).
+B = FOREACH A GENERATE a + (int)null;
+</pre>
+<a name="nulls-ops-produce"></a>
+<h4>Operations That Produce Nulls</h4>
+<p>As noted, nulls can be the result of an operation. These operations can produce null values: </p>
+<ul>
+
+<li>
+
+<p>Division by zero</p>
+
+</li>
+
+<li>
+
+<p>Returns from user defined functions (UDFs) </p>
+
+</li>
+
+<li>
+
+<p>Dereferencing a field that does not exist.</p>
+
+</li>
+
+<li>
+
+<p>Dereferencing a key that does not exist in a map. For example, given a map, info, containing [name#john, phone#5551212] if a user tries to use info#address a null is returned.</p>
+
+</li>
+
+<li>
+
+<p>Accessing a field that does not exist in a tuple.</p>
+
+</li>
+
+</ul>
+<a name="Example%3A+Accessing+a+field+that+does+not+exist+in+a+tuple"></a>
+<h5>Example: Accessing a field that does not exist in a tuple</h5>
+<p>In this example nulls are injected if fields do not have data.</p>
+<pre class="code">
+cat data;
+ 2 3
+4
+7 8 9
+
+A = LOAD 'data' AS (f1:int,f2:int,f3:int)
+
+DUMP A;
+(,2,3)
+(4,,)
+(7,8,9)
+
+B = FOREACH A GENERATE f1,f2;
+
+DUMP B;
+(,2)
+(4,)
+(7,8)
+</pre>
+<a name="nulls-load"></a>
+<h4>Nulls and Load Functions</h4>
+<p>As noted, nulls can occur naturally in the data. If nulls are part of the data, it is the responsibility of the load function to handle them correctly. Keep in mind that what is considered a null value is loader-specific; however, the load function should always communicate null values to Pig by producing Java nulls.</p>
+<p>The Pig Latin load functions (for example, PigStorage and TextLoader) produce null values wherever data is missing. For example, empty strings (chararrays) are not loaded; instead, they are replaced by nulls.</p>
+<p>PigStorage is the default load function for the LOAD operator. In this example the is not null operator is used to filter names with null values.</p>
+<pre class="code">
+A = LOAD 'student' AS (name, age, gpa);
+B = FILTER A BY name is not null;
+</pre>
+<a name="nulls_group"></a>
+<h4>Nulls and GROUP/COGROUP Operators</h4>
+<p>When using the GROUP operator with a single relation, records with a null group key are grouped together.</p>
+<pre class="code">
+A = load 'student' as (name:chararray, age:int, gpa:float);
+dump A;
+(joe,18,2.5)
+(sam,,3.0)
+(bob,,3.5)
+
+X = group A by age;
+dump X;
+(18,{(joe,18,2.5)})
+(,{(sam,,3.0),(bob,,3.5)})
+ </pre>
+<p>When using the GROUP (COGROUP) operator with multiple relations, records with a null group key from different relations are considered different and are grouped separately. In the example below note that there are two tuples in the output corresponding to the null group key: one that contains tuples from relation A (but not relation B) and one that contains tuples from relation B (but not relation A).</p>
+<pre class="code">
+A = load 'student' as (name:chararray, age:int, gpa:float);
+B = load 'student' as (name:chararray, age:int, gpa:float);
+dump B;
+(joe,18,2.5)
+(sam,,3.0)
+(bob,,3.5)
+
+X = cogroup A by age, B by age;
+dump X;
+(18,{(joe,18,2.5)},{(joe,18,2.5)})
+(,{(sam,,3.0),(bob,,3.5)},{})
+(,{},{(sam,,3.0),(bob,,3.5)})
+</pre>
+<a name="nulls_join"></a>
+<h4>Nulls and JOIN Operator</h4>
+<p>The JOIN operator - when performing inner joins - adheres to the SQL standard and disregards (filters out) null values.
+ (See also <a href="perf.html#nulls">Drop Nulls Before a Join</a>.)</p>
+<pre class="code">
+A = load 'student' as (name:chararray, age:int, gpa:float);
+B = load 'student' as (name:chararray, age:int, gpa:float);
+dump B;
+(joe,18,2.5)
+(sam,,3.0)
+(bob,,3.5)
+
+X = join A by age, B by age;
+dump X;
+(joe,18,2.5,joe,18,2.5)
+</pre>
+<a name="constants"></a>
+<h3 class="h4">Constants</h3>
+<p>Pig provides constant representations for all data types except bytearrays.</p>
+<table class="ForrestTable" cellspacing="1" cellpadding="4">
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>
+<strong>Constant Example</strong>
+</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>
+<strong>Notes</strong>
+</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>
+<strong>Simple Data Types</strong>
+</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>int</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>19</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>long</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>19L</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>float</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>19.2F or 1.92e2f</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>double</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>19.2 or 1.92e2</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>chararray</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>'hello world'</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>bytearray</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>Not applicable.</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>boolean</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>true/false</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>Case insensitive.</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>
+<strong>Complex Data Types</strong>
+</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>tuple</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>(19, 2, 1)</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>A constant in this form creates a tuple.</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>bag</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>{ (19, 2), (1, 2) }</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>A constant in this form creates a bag.</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>map</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>[ 'name' # 'John', 'ext' # 5555 ]</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>A constant in this form creates a map.</p>
+
+</td>
+
+</tr>
+
+</table>
+<p></p>
+<p>Please note the following:</p>
+<ul>
+
+<li>
+
+<p>On UTF-8 systems you can specify string constants consisting of printable ASCII characters such as 'abc'; you can specify control characters such as '\t'; and, you can specify a character in Unicode by starting it with '\u', for instance, '\u0001' represents Ctrl-A in hexadecimal (see Wikipedia <a href="http://en.wikipedia.org/wiki/ASCII">ASCII</a>, <a href="http://en.wikipedia.org/wiki/Unicode">Unicode</a>, and <a href="http://en.wikipedia.org/wiki/UTF-8">UTF-8</a>). In theory, you should be able to specify non-UTF-8 constants on non-UTF-8 systems but as far as we know this has not been tested.</p>
+
+</li>
+
+<li>
+
+<p>To specify a long constant, l or L must be appended to the number (for example, 12345678L). If the l or L is not specified, but the number is too large to fit into an int, the problem will be detected at parse time and the processing is terminated. </p>
+
+</li>
+
+<li>
+
+<p>Any numeric constant with decimal point (for example, 1.5) and/or exponent (for example, 5e+1) is treated as double unless it ends with f or F in which case it is assigned type float (for example, 1.5f). </p>
+
+</li>
+
+<li>
+
+<p>There is no native constant type for datetime field. You can use a ToDate udf with chararray constant as argument to generate a datetime value. </p>
+
+</li>
+
+
+</ul>
+<p></p>
+<p>The data type definitions for tuples, bags, and maps apply to constants:</p>
+<ul>
+
+<li>
+
+<p>A tuple can contain fields of any data type</p>
+
+</li>
+
+<li>
+
+<p>A bag is a collection of tuples</p>
+
+</li>
+
+<li>
+
+<p>A map key must be a chararray; a map value can be any data type</p>
+
+</li>
+
+</ul>
+<p></p>
+<p>Complex constants (either with or without values) can be used in the same places scalar constants can be used; that is, in FILTER and GENERATE statements.</p>
+<pre class="code">
+A = LOAD 'data' USING MyStorage() AS (T: tuple(name:chararray, age: int));
+B = FILTER A BY T == ('john', 25);
+D = FOREACH B GENERATE T.name, [25#5.6], {(1, 5, 18)};
+</pre>
+<a name="expressions"></a>
+<h3 class="h4">Expressions</h3>
+<p>In Pig Latin, expressions are language constructs used with the FILTER, FOREACH, GROUP, and SPLIT operators as well as the eval functions.</p>
+<p>Expressions are written in conventional mathematical infix notation and are adapted to the UTF-8 character set. Depending on the context, expressions can include:</p>
+<ul>
+
+<li>
+
+<p>Any Pig data type (simple data types, complex data types)</p>
+
+</li>
+
+<li>
+
+<p>Any Pig operator (arithmetic, comparison, null, boolean, dereference, sign, and cast)</p>
+
+</li>
+
+<li>
+
+<p>Any Pig built in function.</p>
+
+</li>
+
+<li>
+
+<p>Any user defined function (UDF) written in Java. </p>
+
+</li>
+
+</ul>
+<p></p>
+<p>In Pig Latin,</p>
+<ul>
+
+<li>
+
+<p>An arithmetic expression could look like this:</p>
+
+<pre class="code">
+X = GROUP A BY f2*f3;
+</pre>
+
+</li>
+
+
+<li>
+
+<p></p>
+
+<p>A string expression could look like this, where a and b are both chararrays:</p>
+
+<pre class="code">
+X = FOREACH A GENERATE CONCAT(a,b);
+</pre>
+
+</li>
+
+
+<li>
+
+<p></p>
+
+<p>A boolean expression could look like this:</p>
+
+<pre class="code">
+X = FILTER A BY (f1==8) OR (NOT (f2+f3 > f1));
+</pre>
+
+</li>
+
+</ul>
+<a name="fexp"></a>
+<h4>Field Expressions</h4>
+<p>Field expressions represent a field or a <a href="#deref">dereference operator</a> applied to a field.</p>
+<a name="sexp"></a>
+<h4>Star Expressions</h4>
+<p>Star expressions ( * ) can be used to represent all the fields of a tuple. It is equivalent to writing out the fields explicitly. In the following example the definition of B and C are exactly the same, and MyUDF will be invoked with exactly the same arguments in both cases.</p>
+<pre class="code">
+A = LOAD 'data' USING MyStorage() AS (name:chararray, age: int);
+B = FOREACH A GENERATE *, MyUDF(name, age);
+C = FOREACH A GENERATE name, age, MyUDF(*);
+ </pre>
+<p>A common error when using the star expression is shown below. In this example, the programmer really wants to count the number of elements in the bag in the second field: COUNT($1).</p>
+<pre class="code">
+G = GROUP A BY $0;
+C = FOREACH G GENERATE COUNT(*)
+ </pre>
+<p>There are some restrictions on use of the star expression when the input schema is unknown (null):</p>
+<ul>
+
+<li>For GROUP/COGROUP, you can't include a star expression in a GROUP BY column. </li>
+
+<li>For ORDER BY, if you have project-star as ORDER BY column, you can’t have any other ORDER BY column in that statement. </li>
+
+</ul>
+<a name="prexp"></a>
+<h4>Project-Range Expressions</h4>
+<p>Project-range ( .. ) expressions can be used to project a range of columns from input. For example:</p>
+<ul>
+
+<li>.. $x : projects columns $0 through $x, inclusive </li>
+
+<li>$x .. : projects columns through end, inclusive </li>
+
+<li>$x .. $y : projects columns through $y, inclusive </li>
+
+</ul>
+<p></p>
+<p>If the input relation has a schema, you can refer to columns by alias rather than by column position. You can also combine aliases and column positions in an expression; for example, "col1 .. $5" is valid. </p>
+<p>Project-range can be used in all cases where the <a href="#sexp">star expression</a> ( * ) is allowed.</p>
+<p>Project-range can be used in the following statements:
+<a href="#FOREACH">FOREACH</a>,
+<a href="#JOIN+%28inner%29">JOIN</a>,
+<a href="#GROUP">GROUP</a>,
+<a href="#COGROUP">COGROUP</a>, and
+<a href="#ORDER+BY">ORDER BY</a> (also when ORDER BY is used within a nested FOREACH block).</p>
+<p>A few examples are shown here:</p>
+<pre class="code">
+.....
+grunt> F = foreach IN generate (int)col0, col1 .. col3;
+grunt> describe F;
+F: {col0: int,col1: bytearray,col2: bytearray,col3: bytearray}
+.....
+.....
+grunt> SORT = order IN by col2 .. col3, col0, col4 ..;
+.....
+.....
+J = join IN1 by $0 .. $3, IN2 by $0 .. $3;
+.....
+.....
+g = group l1 by b .. c;
+.....
+</pre>
+<p>There are some restrictions on the use of project-to-end form of project-range (eg "x .. ") when the input schema is unknown (null): </p>
+<ul>
+
+<li>For GROUP/COGROUP, the project-to-end form of project-range is not allowed.</li>
+
+<li>For ORDER BY, the project-to-end form of project-range is supported only as the last sort column.
+<pre class="code">
+.....
+grunt> describe IN;
+Schema for IN unknown.
+
+/* This statement is supported */
+SORT = order IN by $2 .. $3, $6 ..;
+
+/* This statement is NOT supported */
+SORT = order IN by $2 .. $3, $6 ..;
+.....
+</pre>
+
+
+
+</li>
+
+</ul>
+<a name="bexp"></a>
+<h4>Boolean Expressions</h4>
+<p>Boolean expressions can be made up of UDFs that return a boolean value or boolean operators
+ (see <a href="#boolops">Boolean Operators</a>).
+ </p>
+<a name="texp"></a>
+<h4>Tuple Expressions</h4>
+<p>Tuple expressions form subexpressions into tuples. The tuple expression has the form (expression [, expression …]), where expression is a general expression. The simplest tuple expression is the star expression, which represents all fields.
+ </p>
+<a name="gexp"></a>
+<h4>General Expressions</h4>
+<p>General expressions can be made up of UDFs and almost any operator. Since Pig does not consider boolean a base type, the result of a general expression cannot be a boolean. Field expressions are the simpliest general expressions.
+ </p>
+<a name="Schemas"></a>
+<h3 class="h4">Schemas</h3>
+<p>Schemas enable you to assign names to fields and declare types for fields. Schemas are optional but we encourage you to use them whenever possible; type declarations result in better parse-time error checking and more efficient code execution.</p>
+<p>Schemas for <a href="#schema-simple">simple types</a> and <a href="#schema-complex">complex types</a> can be used anywhere a schema definition is appropriate.</p>
+<p>Schemas are defined with the <a href="#LOAD">LOAD</a>, <a href="#STREAM">STREAM</a>, and <a href="#FOREACH">FOREACH</a> operators using the AS clause. If you define a schema using the LOAD operator, then it is the load function that enforces the schema
+ (see <a href="#LOAD">LOAD</a> and <a href="udf.html">User Defined Functions</a> for more information).</p>
+<p></p>
+<p>
+<strong>Known Schema Handling</strong>
+</p>
+<p>Note the following:</p>
+<ul>
+
+<li>You can define a schema that includes both the field name and field type.</li>
+
+<li>You can define a schema that includes the field name only; in this case, the field type defaults to bytearray.</li>
+
+<li>You can choose not to define a schema; in this case, the field is un-named and the field type defaults to bytearray.</li>
+
+</ul>
+<p>If you assign a name to a field, you can refer to that field using the name or by positional notation. If you don't assign a name to a field (the field is un-named) you can only refer to the field using positional notation.</p>
+<p>If you assign a type to a field, you can subsequently change the type using the cast operators. If you don't assign a type to a field, the field defaults to bytearray; you can change the default type using the cast operators.</p>
+<p></p>
+<a name="unknown-schema"></a>
+<p id="unknown-schema">
+<strong>Unknown Schema Handling</strong>
+</p>
+<p>Note the following:</p>
+<ul>
+
+<li>When you JOIN/COGROUP/CROSS multiple relations, if any relation has an unknown schema (or no defined schema, also referred to as a null schema), the schema for the resulting relation is null. </li>
+
+<li>If you FLATTEN a bag with empty inner schema, the schema for the resulting relation is null.</li>
+
+<li>If you UNION two relations with incompatible schema, the schema for resulting relation is null.</li>
+
+<li>If the schema is null, Pig treats all fields as bytearray (in the backend, Pig will determine the real type for the fields dynamically) </li>
+
+</ul>
+<p>See the examples below. If a field's data type is not specified, Pig will use bytearray to denote an unknown type. If the number of fields is not known, Pig will derive an unknown schema.</p>
+<pre class="code">
+/* The field data types are not specified ... */
+a = load '1.txt' as (a0, b0);
+a: {a0: bytearray,b0: bytearray}
+
+/* The number of fields is not known ... */
+a = load '1.txt';
+a: Schema for a unknown
+</pre>
+<p></p>
+<p>
+<strong>How Pig Handles Schema</strong>
+</p>
+<p>As shown above, with a few exceptions Pig can infer the schema of a relationship up front. You can examine the schema of particular relation using <a href="test.html#DESCRIBE">DESCRIBE</a>. Pig enforces this computed schema during the actual execution by casting the input data to the expected data type. If the process is successful the results are returned to the user; otherwise, a warning is generated for each record that failed to convert. Note that Pig does not know the actual types of the fields in the input data prior to the execution; rather, Pig determines the data types and performs the right conversions on the fly.</p>
+<p>Having a deterministic schema is very powerful; however, sometimes it comes at the cost of performance. Consider the following example:</p>
+<pre class="code">
+A = load ‘input’ as (x, y, z);
+B = foreach A generate x+y;
+</pre>
+<p>If you do <a href="test.html#DESCRIBE">DESCRIBE</a> on B, you will see a single column of type double. This is because Pig makes the safest choice and uses the largest numeric type when the schema is not know. In practice, the input data could contain integer values; however, Pig will cast the data to double and make sure that a double result is returned.</p>
+<p>If the schema of a relation can’t be inferred, Pig will just use the runtime data as is and propagate it through the pipeline.</p>
+<a name="schema-load"></a>
+<h4>Schemas with LOAD and STREAM </h4>
+<p>With LOAD and STREAM operators, the schema following the AS keyword must be enclosed in parentheses.</p>
+<p>In this example the LOAD statement includes a schema definition for simple data types.</p>
+<pre class="code">
+A = LOAD 'data' AS (f1:int, f2:int);
+</pre>
+<a name="schemaforeach"></a>
+<h4>Schemas with FOREACH </h4>
+<p>With FOREACH operators, the schema following the AS keyword must be enclosed in parentheses when the FLATTEN operator is used. Otherwise, the schema should not be enclosed in parentheses.</p>
+<p>In this example the FOREACH statement includes FLATTEN and a schema for simple data types.</p>
+<pre class="code">
+X = FOREACH C GENERATE FLATTEN(B) AS (f1:int, f2:int, f3:int), group;
+</pre>
+<p>In this example the FOREACH statement includes a schema for simple expression.</p>
+<pre class="code">
+X = FOREACH A GENERATE f1+f2 AS x1:int;
+</pre>
+<p>In this example the FOREACH statement includes a schemas for multiple fields.</p>
+<pre class="code">
+X = FOREACH A GENERATE f1 as user, f2 as age, f3 as gpa;
+</pre>
+<a name="schema-simple"></a>
+<h4>Schemas for Simple Data Types</h4>
+<p>Simple data types include int, long, float, double, chararray, bytearray, boolean and datetime.</p>
+<a name="Syntax-N10D4A"></a>
+<h5>Syntax</h5>
+<table class="ForrestTable" cellspacing="1" cellpadding="4">
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>(alias[:type]) [, (alias[:type]) …] )</p>
+
+</td>
+
+</tr>
+
+</table>
+<a name="Terms-N10D5E"></a>
+<h5>Terms</h5>
+<table class="ForrestTable" cellspacing="1" cellpadding="4">
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>alias</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>The name assigned to the field.</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>type</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>(Optional) The simple data type assigned to the field.</p>
+
+<p>The alias and type are separated by a colon ( : ).</p>
+
+<p>If the type is omitted, the field defaults to type bytearray.</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>( , )</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>Multiple fields are enclosed in parentheses and separated by commas.</p>
+
+</td>
+
+</tr>
+
+</table>
+<a name="Examples"></a>
+<h5>Examples</h5>
+<p>In this example the schema defines multiple types.</p>
+<pre class="code">
+cat student;
+John 18 4.0
+Mary 19 3.8
+Bill 20 3.9
+Joe 18 3.8
+
+A = LOAD 'student' AS (name:chararray, age:int, gpa:float);
+
+DESCRIBE A;
+A: {name: chararray,age: int,gpa: float}
+
+DUMP A;
+(John,18,4.0F)
+(Mary,19,3.8F)
+(Bill,20,3.9F)
+(Joe,18,3.8F)
+</pre>
+<p>In this example field "gpa" will default to bytearray because no type is declared. </p>
+<pre class="code">
+cat student;
+John 18 4.0
+Mary 19 3.8
+Bill 20 3.9
+Joe 18 3.8
+
+A = LOAD 'data' AS (name:chararray, age:int, gpa);
+
+DESCRIBE A;
+A: {name: chararray,age: int,gpa: bytearray}
+
+DUMP A;
+(John,18,4.0)
+(Mary,19,3.8)
+(Bill,20,3.9)
+(Joe,18,3.8)
+</pre>
+<a name="schema-complex"></a>
+<h4>Schemas for Complex Data Types</h4>
+<p>Complex data types include tuples, bags, and maps.</p>
+<a name="tuple-schema"></a>
+<h4>Tuple Schemas</h4>
+<p>A tuple is an ordered set of fields.</p>
+<a name="Syntax-N10DD1"></a>
+<h5>Syntax</h5>
+<table class="ForrestTable" cellspacing="1" cellpadding="4">
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>alias[:tuple] (alias[:type]) [, (alias[:type]) …] )</p>
+
+</td>
+
+</tr>
+
+</table>
+<a name="Terms-N10DE6"></a>
+<h5>Terms</h5>
+<table class="ForrestTable" cellspacing="1" cellpadding="4">
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>alias</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>The name assigned to the tuple.</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>:tuple</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>(Optional) The data type, tuple (case insensitive).</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>( )</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>The designation for a tuple, a set of parentheses.</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>alias[:type]</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>The constituents of the tuple, where the schema definition rules for the corresponding type applies to the constituents of the tuple:</p>
+
+<ul>
+
+<li>
+
+<p>alias – the name assigned to the field</p>
+
+</li>
+
+<li>
+
+<p>type (optional) – the simple or complex data type assigned to the field</p>
+
+</li>
+
+</ul>
+
+</td>
+
+</tr>
+
+</table>
+<a name="Examples-N10E4B"></a>
+<h5>Examples</h5>
+<p>In this example the schema defines one tuple. The load statements are equivalent.</p>
+<pre class="code">
+cat data;
+(3,8,9)
+(1,4,7)
+(2,5,8)
+
+A = LOAD 'data' AS (T: tuple (f1:int, f2:int, f3:int));
+A = LOAD 'data' AS (T: (f1:int, f2:int, f3:int));
+
+DESCRIBE A;
+A: {T: (f1: int,f2: int,f3: int)}
+
+DUMP A;
+((3,8,9))
+((1,4,7))
+((2,5,8))
+</pre>
+<p>In this example the schema defines two tuples.</p>
+<pre class="code">
+cat data;
+(3,8,9) (mary,19)
+(1,4,7) (john,18)
+(2,5,8) (joe,18)
+
+A = LOAD data AS (F:tuple(f1:int,f2:int,f3:int),T:tuple(t1:chararray,t2:int));
+
+DESCRIBE A;
+A: {F: (f1: int,f2: int,f3: int),T: (t1: chararray,t2: int)}
+
+DUMP A;
+((3,8,9),(mary,19))
+((1,4,7),(john,18))
+((2,5,8),(joe,18))
+</pre>
+<a name="bag-schema"></a>
+<h4>Bag Schemas</h4>
+<p>A bag is a collection of tuples.</p>
+<a name="Syntax-N10E6B"></a>
+<h5>Syntax</h5>
+<table class="ForrestTable" cellspacing="1" cellpadding="4">
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>alias[:bag] {tuple} </p>
+
+</td>
+
+</tr>
+
+</table>
+<a name="Terms-N10E80"></a>
+<h5>Terms</h5>
+<table class="ForrestTable" cellspacing="1" cellpadding="4">
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>alias</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>The name assigned to the bag.</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>:bag</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>(Optional) The data type, bag (case insensitive).</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>{ }</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>The designation for a bag, a set of curly brackets.</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>tuple</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>A tuple (see Tuple Schema).</p>
+
+</td>
+
+</tr>
+
+</table>
+<a name="Examples-N10ED6"></a>
+<h5>Examples</h5>
+<p>In this example the schema defines a bag. The two load statements are equivalent.</p>
+<pre class="code">
+cat data;
+{(3,8,9)}
+{(1,4,7)}
+{(2,5,8)}
+
+A = LOAD 'data' AS (B: bag {T: tuple(t1:int, t2:int, t3:int)});
+A = LOAD 'data' AS (B: {T: (t1:int, t2:int, t3:int)});
+
+DESCRIBE A:
+A: {B: {T: (t1: int,t2: int,t3: int)}}
+
+DUMP A;
+({(3,8,9)})
+({(1,4,7)})
+({(2,5,8)})
+</pre>
+<a name="map-schema"></a>
+<h4>Map Schemas</h4>
+<p>A map is a set of key value pairs.</p>
+<a name="Syntax+%28%3C%3E+demotes+optional%29"></a>
+<h5>Syntax (<> demotes optional)</h5>
+<table class="ForrestTable" cellspacing="1" cellpadding="4">
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>alias<:map> [ <type> ] </p>
+
+</td>
+
+</tr>
+
+</table>
+<a name="Terms-N10F03"></a>
+<h5>Terms</h5>
+<table class="ForrestTable" cellspacing="1" cellpadding="4">
+
+<tr>
+
+<td colspan="1" rowspan="1">
+<p>alias</p>
+</td>
+ <td colspan="1" rowspan="1">
+<p>The name assigned to the map.</p>
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+<p>:map</p>
+</td>
+ <td colspan="1" rowspan="1">
+<p>(Optional) The data type, map (case insensitive).</p>
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+<p>[ ]</p>
+</td>
+ <td colspan="1" rowspan="1">
+<p>The designation for a map, a set of straight brackets [ ].</p>
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+<p>type</p>
+</td>
+ <td colspan="1" rowspan="1">
+<p>(Optional) The datatype (all types allowed, bytearray is the default).</p>
+
+<p>The type applies to the map value only; the map key is always type chararray (see <a href="#Map">Map</a>).</p>
+
+<p>If a type is declared then ALL values in the map must be of this type.</p>
+
+</td>
+
+</tr>
+
+</table>
+<a name="Examples-N10F54"></a>
+<h5>Examples</h5>
+<p>In this example the schema defines an untyped map (the map values default to bytearray). The load statements are equivalent.</p>
+<pre class="code">
+cat data;
+[open#apache]
+[apache#hadoop]
+
+A = LOAD 'data' AS (M:map []);
+A = LOAD 'data' AS (M:[]);
+
+DESCRIBE A;
+a: {M: map[ ]}
+
+DUMP A;
+([open#apache])
+([apache#hadoop])
+</pre>
+<p>This example shows the use of a typed maps.</p>
+<pre class="code">
+/* Map types are declared*/
+a = load '1.txt' as(map[int]); --Map value is int
+b = foreach a generate (map[(i:int)])a0; -- Map value is tuple
+b = stream a through `cat` as (m:map[{(i:int,j:chararray)}]); -- Map value is bag
+
+/* The MapLookup of a typed map will result in a datatype of the map value */
+a = load '1.txt' as(map[int]);
+b = foreach a generate $0#'key';
+
+/* Schema for b */
+b: {int}
+
+</pre>
+<a name="schema-multi"></a>
+<h4>Schemas for Multiple Types</h4>
+<p>You can define schemas for data that includes multiple types.</p>
+<a name="Example-N10F74"></a>
+<h5>Example</h5>
+<p>In this example the schema defines a tuple, bag, and map.</p>
+<pre class="code">
+A = LOAD 'mydata' AS (T1:tuple(f1:int, f2:int), B:bag{T2:tuple(t1:float,t2:float)}, M:map[] );
+
+A = LOAD 'mydata' AS (T1:(f1:int, f2:int), B:{T2:(t1:float,t2:float)}, M:[] );
+</pre>
+</div>
+
+<!-- =================================================================== -->
+<!-- ARITHMETIC OPERATORS, ETC -->
+
+<a name="artichmetic-ops"></a>
+<h2 class="h3">Arithmetic Operators and More</h2>
+<div class="section">
+<a name="arithmetic"></a>
+<h3 class="h4">Arithmetic Operators</h3>
+<a name="Description"></a>
+<h4>Description</h4>
+<table class="ForrestTable" cellspacing="1" cellpadding="4">
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>Operator</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>Symbol</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p> Notes</p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>addition </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>+</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>subtraction </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>-</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>multiplication </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>*</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>division </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>/</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>modulo </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>%</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>Returns the remainder of a divided by b (a%b).</p>
+
+<p>Works with integral numbers (int, long). </p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>bincond </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>? :</p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>(condition ? value_if_true : value_if_false) </p>
+
+<p>The bincond should be enclosed in parenthesis. </p>
+
+<p>The schemas for the two conditional outputs of the bincond should match.</p>
+
+<p>Use expressions only (relational operators are not allowed).</p>
+
+</td>
+
+</tr>
+
+</table>
+<a name="Examples-N11061"></a>
+<h5>Examples </h5>
+<p>Suppose we have relation A.</p>
+<pre class="code">
+A = LOAD 'data' AS (f1:int, f2:int, B:bag{T:tuple(t1:int,t2:int)});
+
+DUMP A;
+(10,1,{(2,3),(4,6)})
+(10,3,{(2,3),(4,6)})
+(10,6,{(2,3),(4,6),(5,7)})
+</pre>
+<p>In this example the modulo operator is used with fields f1 and f2.</p>
+<pre class="code">
+X = FOREACH A GENERATE f1, f2, f1%f2;
+
+DUMP X;
+(10,1,0)
+(10,3,1)
+(10,6,4)
+</pre>
+<p>In this example the bincond operator is used with fields f2 and B. The condition is "f2 equals 1"; if the condition is true, return 1; if the condition is false, return the count of the number of tuples in B.</p>
+<pre class="code">
+X = FOREACH A GENERATE f2, (f2==1?1:COUNT(B));
+
+DUMP X;
+(1,1L)
+(3,2L)
+(6,3L)
+</pre>
+<a name="types-table-add"></a>
+<h5> Types Table: addition (+) and subtraction (-) operators</h5>
+<p>* bytearray cast as this data type</p>
+<table class="ForrestTable" cellspacing="1" cellpadding="4">
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>bag </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>tuple </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>map </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>int </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>long </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>float </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>double </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>chararray </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>bytearray </p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>bag </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>error </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>error </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>error </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>error </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>error </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>error </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>error </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>error </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>error </p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>tuple </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>not yet </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>error </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>error </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>error </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>error </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>error </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>error </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>error </p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>map </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>error </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>error </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>error </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>error </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>error </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>error </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>error </p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>int </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>int </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>long </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>float </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>double </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>error </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>cast as int </p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>long </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>long </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>float </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>double </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>error </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>cast as long </p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>float </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>float </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>double </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>error </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>cast as float </p>
+
+</td>
+
+</tr>
+
+<tr>
+
+<td colspan="1" rowspan="1">
+
+<p>double </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p></p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>double </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
+<p>error </p>
+
+</td>
+ <td colspan="1" rowspan="1">
+
[... 8594 lines stripped ...]