You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by zs...@apache.org on 2009/01/13 03:14:09 UTC
svn commit: r733992 - in /hadoop/hive/trunk: ./ data/files/ lib/ ql/
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/udf/
ql/src/test/org/apache/hadoop/hive/ql/ ql/src/test/queries/clientpositive/
ql/src/test/results/c...
Author: zshao
Date: Mon Jan 12 18:14:07 2009
New Revision: 733992
URL: http://svn.apache.org/viewvc?rev=733992&view=rev
Log:
HIVE-163. JSON udf function added. (Hao Liu via zshao)
Added:
hadoop/hive/trunk/data/files/json.txt
hadoop/hive/trunk/lib/json-LICENSE.txt
hadoop/hive/trunk/lib/json-README.txt
hadoop/hive/trunk/lib/json.jar (with props)
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFJson.java
hadoop/hive/trunk/ql/src/test/queries/clientpositive/udf_json.q
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_json.q.out
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/lib/README
hadoop/hive/trunk/ql/build.xml
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
hadoop/hive/trunk/ql/src/test/results/clientpositive/input2.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input3.q.out
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=733992&r1=733991&r2=733992&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Mon Jan 12 18:14:07 2009
@@ -9,6 +9,8 @@
NEW FEATURES
+ HIVE-163. JSON udf function added. (Hao Liu via zshao)
+
HIVE-142. Add a metastore check command. (Johan Oskarsson via zshao)
HIVE-148. Extend bin/hive to include the lineage tool.
Added: hadoop/hive/trunk/data/files/json.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/data/files/json.txt?rev=733992&view=auto
==============================================================================
--- hadoop/hive/trunk/data/files/json.txt (added)
+++ hadoop/hive/trunk/data/files/json.txt Mon Jan 12 18:14:07 2009
@@ -0,0 +1 @@
+{"store":{"fruit":[{"weight":8,"type":"apple"},{"weight":9,"type":"pear"}],"basket":[[1,2,{"b":"y","a":"x"}],[3,4],[5,6]],"book":[{"author":"Nigel Rees","title":"Sayings of the Century","category":"reference","price":8.95},{"author":"Herman Melville","title":"Moby Dick","category":"fiction","price":8.99,"isbn":"0-553-21311-3"},{"author":"J. R. R. Tolkien","title":"The Lord of the Rings","category":"fiction","reader":[{"age":25,"name":"bob"},{"age":26,"name":"jack"}],"price":22.99,"isbn":"0-395-19395-8"}],"bicycle":{"price":19.95,"color":"red"}},"email":"amy@only_for_json_udf_test.net","owner":"amy"}
Modified: hadoop/hive/trunk/lib/README
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/lib/README?rev=733992&r1=733991&r2=733992&view=diff
==============================================================================
--- hadoop/hive/trunk/lib/README (original)
+++ hadoop/hive/trunk/lib/README Mon Jan 12 18:14:07 2009
@@ -12,3 +12,4 @@
* velocity-1.5.jar - http://velocity.apache.org/engine/releases/velocity-1.5/
* commons-collections-3.2.1.jar - http://commons.apache.org/downloads/download_collections.cgi
* commons-lang-2.4.jar - http://commons.apache.org/downloads/download_lang.cgi
+* json.jar - http://www.json.org/java/index.html
Added: hadoop/hive/trunk/lib/json-LICENSE.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/lib/json-LICENSE.txt?rev=733992&view=auto
==============================================================================
--- hadoop/hive/trunk/lib/json-LICENSE.txt (added)
+++ hadoop/hive/trunk/lib/json-LICENSE.txt Mon Jan 12 18:14:07 2009
@@ -0,0 +1,9 @@
+Copyright (c) 2002 JSON.org
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+The Software shall be used for Good, not Evil.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
Added: hadoop/hive/trunk/lib/json-README.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/lib/json-README.txt?rev=733992&view=auto
==============================================================================
--- hadoop/hive/trunk/lib/json-README.txt (added)
+++ hadoop/hive/trunk/lib/json-README.txt Mon Jan 12 18:14:07 2009
@@ -0,0 +1,563 @@
+
+
+
+
+
+
+Network Working Group D. Crockford
+Request for Comments: 4627 JSON.org
+Category: Informational July 2006
+
+
+ The application/json Media Type for JavaScript Object Notation (JSON)
+
+Status of This Memo
+
+ This memo provides information for the Internet community. It does
+ not specify an Internet standard of any kind. Distribution of this
+ memo is unlimited.
+
+Copyright Notice
+
+ Copyright (C) The Internet Society (2006).
+
+Abstract
+
+ JavaScript Object Notation (JSON) is a lightweight, text-based,
+ language-independent data interchange format. It was derived from
+ the ECMAScript Programming Language Standard. JSON defines a small
+ set of formatting rules for the portable representation of structured
+ data.
+
+1. Introduction
+
+ JavaScript Object Notation (JSON) is a text format for the
+ serialization of structured data. It is derived from the object
+ literals of JavaScript, as defined in the ECMAScript Programming
+ Language Standard, Third Edition [ECMA].
+
+ JSON can represent four primitive types (strings, numbers, booleans,
+ and null) and two structured types (objects and arrays).
+
+ A string is a sequence of zero or more Unicode characters [UNICODE].
+
+ An object is an unordered collection of zero or more name/value
+ pairs, where a name is a string and a value is a string, number,
+ boolean, null, object, or array.
+
+ An array is an ordered sequence of zero or more values.
+
+ The terms "object" and "array" come from the conventions of
+ JavaScript.
+
+ JSON's design goals were for it to be minimal, portable, textual, and
+ a subset of JavaScript.
+
+
+
+Crockford Informational [Page 1]
+
+RFC 4627 JSON July 2006
+
+
+1.1. Conventions Used in This Document
+
+ The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT",
+ "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this
+ document are to be interpreted as described in [RFC2119].
+
+ The grammatical rules in this document are to be interpreted as
+ described in [RFC4234].
+
+2. JSON Grammar
+
+ A JSON text is a sequence of tokens. The set of tokens includes six
+ structural characters, strings, numbers, and three literal names.
+
+ A JSON text is a serialized object or array.
+
+ JSON-text = object / array
+
+ These are the six structural characters:
+
+ begin-array = ws %x5B ws ; [ left square bracket
+
+ begin-object = ws %x7B ws ; { left curly bracket
+
+ end-array = ws %x5D ws ; ] right square bracket
+
+ end-object = ws %x7D ws ; } right curly bracket
+
+ name-separator = ws %x3A ws ; : colon
+
+ value-separator = ws %x2C ws ; , comma
+
+ Insignificant whitespace is allowed before or after any of the six
+ structural characters.
+
+ ws = *(
+ %x20 / ; Space
+ %x09 / ; Horizontal tab
+ %x0A / ; Line feed or New line
+ %x0D ; Carriage return
+ )
+
+2.1. Values
+
+ A JSON value MUST be an object, array, number, or string, or one of
+ the following three literal names:
+
+ false null true
+
+
+
+Crockford Informational [Page 2]
+
+RFC 4627 JSON July 2006
+
+
+ The literal names MUST be lowercase. No other literal names are
+ allowed.
+
+ value = false / null / true / object / array / number / string
+
+ false = %x66.61.6c.73.65 ; false
+
+ null = %x6e.75.6c.6c ; null
+
+ true = %x74.72.75.65 ; true
+
+2.2. Objects
+
+ An object structure is represented as a pair of curly brackets
+ surrounding zero or more name/value pairs (or members). A name is a
+ string. A single colon comes after each name, separating the name
+ from the value. A single comma separates a value from a following
+ name. The names within an object SHOULD be unique.
+
+ object = begin-object [ member *( value-separator member ) ]
+ end-object
+
+ member = string name-separator value
+
+2.3. Arrays
+
+ An array structure is represented as square brackets surrounding zero
+ or more values (or elements). Elements are separated by commas.
+
+ array = begin-array [ value *( value-separator value ) ] end-array
+
+2.4. Numbers
+
+ The representation of numbers is similar to that used in most
+ programming languages. A number contains an integer component that
+ may be prefixed with an optional minus sign, which may be followed by
+ a fraction part and/or an exponent part.
+
+ Octal and hex forms are not allowed. Leading zeros are not allowed.
+
+ A fraction part is a decimal point followed by one or more digits.
+
+ An exponent part begins with the letter E in upper or lowercase,
+ which may be followed by a plus or minus sign. The E and optional
+ sign are followed by one or more digits.
+
+ Numeric values that cannot be represented as sequences of digits
+ (such as Infinity and NaN) are not permitted.
+
+
+
+Crockford Informational [Page 3]
+
+RFC 4627 JSON July 2006
+
+
+ number = [ minus ] int [ frac ] [ exp ]
+
+ decimal-point = %x2E ; .
+
+ digit1-9 = %x31-39 ; 1-9
+
+ e = %x65 / %x45 ; e E
+
+ exp = e [ minus / plus ] 1*DIGIT
+
+ frac = decimal-point 1*DIGIT
+
+ int = zero / ( digit1-9 *DIGIT )
+
+ minus = %x2D ; -
+
+ plus = %x2B ; +
+
+ zero = %x30 ; 0
+
+2.5. Strings
+
+ The representation of strings is similar to conventions used in the C
+ family of programming languages. A string begins and ends with
+ quotation marks. All Unicode characters may be placed within the
+ quotation marks except for the characters that must be escaped:
+ quotation mark, reverse solidus, and the control characters (U+0000
+ through U+001F).
+
+ Any character may be escaped. If the character is in the Basic
+ Multilingual Plane (U+0000 through U+FFFF), then it may be
+ represented as a six-character sequence: a reverse solidus, followed
+ by the lowercase letter u, followed by four hexadecimal digits that
+ encode the character's code point. The hexadecimal letters A though
+ F can be upper or lowercase. So, for example, a string containing
+ only a single reverse solidus character may be represented as
+ "\u005C".
+
+ Alternatively, there are two-character sequence escape
+ representations of some popular characters. So, for example, a
+ string containing only a single reverse solidus character may be
+ represented more compactly as "\\".
+
+ To escape an extended character that is not in the Basic Multilingual
+ Plane, the character is represented as a twelve-character sequence,
+ encoding the UTF-16 surrogate pair. So, for example, a string
+ containing only the G clef character (U+1D11E) may be represented as
+ "\uD834\uDD1E".
+
+
+
+Crockford Informational [Page 4]
+
+RFC 4627 JSON July 2006
+
+
+ string = quotation-mark *char quotation-mark
+
+ char = unescaped /
+ escape (
+ %x22 / ; " quotation mark U+0022
+ %x5C / ; \ reverse solidus U+005C
+ %x2F / ; / solidus U+002F
+ %x62 / ; b backspace U+0008
+ %x66 / ; f form feed U+000C
+ %x6E / ; n line feed U+000A
+ %x72 / ; r carriage return U+000D
+ %x74 / ; t tab U+0009
+ %x75 4HEXDIG ) ; uXXXX U+XXXX
+
+ escape = %x5C ; \
+
+ quotation-mark = %x22 ; "
+
+ unescaped = %x20-21 / %x23-5B / %x5D-10FFFF
+
+3. Encoding
+
+ JSON text SHALL be encoded in Unicode. The default encoding is
+ UTF-8.
+
+ Since the first two characters of a JSON text will always be ASCII
+ characters [RFC0020], it is possible to determine whether an octet
+ stream is UTF-8, UTF-16 (BE or LE), or UTF-32 (BE or LE) by looking
+ at the pattern of nulls in the first four octets.
+
+ 00 00 00 xx UTF-32BE
+ 00 xx 00 xx UTF-16BE
+ xx 00 00 00 UTF-32LE
+ xx 00 xx 00 UTF-16LE
+ xx xx xx xx UTF-8
+
+4. Parsers
+
+ A JSON parser transforms a JSON text into another representation. A
+ JSON parser MUST accept all texts that conform to the JSON grammar.
+ A JSON parser MAY accept non-JSON forms or extensions.
+
+ An implementation may set limits on the size of texts that it
+ accepts. An implementation may set limits on the maximum depth of
+ nesting. An implementation may set limits on the range of numbers.
+ An implementation may set limits on the length and character contents
+ of strings.
+
+
+
+
+Crockford Informational [Page 5]
+
+RFC 4627 JSON July 2006
+
+
+5. Generators
+
+ A JSON generator produces JSON text. The resulting text MUST
+ strictly conform to the JSON grammar.
+
+6. IANA Considerations
+
+ The MIME media type for JSON text is application/json.
+
+ Type name: application
+
+ Subtype name: json
+
+ Required parameters: n/a
+
+ Optional parameters: n/a
+
+ Encoding considerations: 8bit if UTF-8; binary if UTF-16 or UTF-32
+
+ JSON may be represented using UTF-8, UTF-16, or UTF-32. When JSON
+ is written in UTF-8, JSON is 8bit compatible. When JSON is
+ written in UTF-16 or UTF-32, the binary content-transfer-encoding
+ must be used.
+
+ Security considerations:
+
+ Generally there are security issues with scripting languages. JSON
+ is a subset of JavaScript, but it is a safe subset that excludes
+ assignment and invocation.
+
+ A JSON text can be safely passed into JavaScript's eval() function
+ (which compiles and executes a string) if all the characters not
+ enclosed in strings are in the set of characters that form JSON
+ tokens. This can be quickly determined in JavaScript with two
+ regular expressions and calls to the test and replace methods.
+
+ var my_JSON_object = !(/[^,:{}\[\]0-9.\-+Eaeflnr-u \n\r\t]/.test(
+ text.replace(/"(\\.|[^"\\])*"/g, ''))) &&
+ eval('(' + text + ')');
+
+ Interoperability considerations: n/a
+
+ Published specification: RFC 4627
+
+
+
+
+
+
+
+
+Crockford Informational [Page 6]
+
+RFC 4627 JSON July 2006
+
+
+ Applications that use this media type:
+
+ JSON has been used to exchange data between applications written
+ in all of these programming languages: ActionScript, C, C#,
+ ColdFusion, Common Lisp, E, Erlang, Java, JavaScript, Lua,
+ Objective CAML, Perl, PHP, Python, Rebol, Ruby, and Scheme.
+
+ Additional information:
+
+ Magic number(s): n/a
+ File extension(s): .json
+ Macintosh file type code(s): TEXT
+
+ Person & email address to contact for further information:
+ Douglas Crockford
+ douglas@crockford.com
+
+ Intended usage: COMMON
+
+ Restrictions on usage: none
+
+ Author:
+ Douglas Crockford
+ douglas@crockford.com
+
+ Change controller:
+ Douglas Crockford
+ douglas@crockford.com
+
+7. Security Considerations
+
+ See Security Considerations in Section 6.
+
+8. Examples
+
+ This is a JSON object:
+
+ {
+ "Image": {
+ "Width": 800,
+ "Height": 600,
+ "Title": "View from 15th Floor",
+ "Thumbnail": {
+ "Url": "http://www.example.com/image/481989943",
+ "Height": 125,
+ "Width": "100"
+ },
+ "IDs": [116, 943, 234, 38793]
+
+
+
+Crockford Informational [Page 7]
+
+RFC 4627 JSON July 2006
+
+
+ }
+ }
+
+ Its Image member is an object whose Thumbnail member is an object
+ and whose IDs member is an array of numbers.
+
+ This is a JSON array containing two objects:
+
+ [
+ {
+ "precision": "zip",
+ "Latitude": 37.7668,
+ "Longitude": -122.3959,
+ "Address": "",
+ "City": "SAN FRANCISCO",
+ "State": "CA",
+ "Zip": "94107",
+ "Country": "US"
+ },
+ {
+ "precision": "zip",
+ "Latitude": 37.371991,
+ "Longitude": -122.026020,
+ "Address": "",
+ "City": "SUNNYVALE",
+ "State": "CA",
+ "Zip": "94085",
+ "Country": "US"
+ }
+ ]
+
+9. References
+
+9.1. Normative References
+
+ [ECMA] European Computer Manufacturers Association, "ECMAScript
+ Language Specification 3rd Edition", December 1999,
+ <http://www.ecma-international.org/publications/files/
+ ecma-st/ECMA-262.pdf>.
+
+ [RFC0020] Cerf, V., "ASCII format for network interchange", RFC 20,
+ October 1969.
+
+ [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate
+ Requirement Levels", BCP 14, RFC 2119, March 1997.
+
+ [RFC4234] Crocker, D. and P. Overell, "Augmented BNF for Syntax
+ Specifications: ABNF", RFC 4234, October 2005.
+
+
+
+Crockford Informational [Page 8]
+
+RFC 4627 JSON July 2006
+
+
+ [UNICODE] The Unicode Consortium, "The Unicode Standard Version 4.0",
+ 2003, <http://www.unicode.org/versions/Unicode4.1.0/>.
+
+Author's Address
+
+ Douglas Crockford
+ JSON.org
+ EMail: douglas@crockford.com
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Crockford Informational [Page 9]
+
+RFC 4627 JSON July 2006
+
+
+Full Copyright Statement
+
+ Copyright (C) The Internet Society (2006).
+
+ This document is subject to the rights, licenses and restrictions
+ contained in BCP 78, and except as set forth therein, the authors
+ retain all their rights.
+
+ This document and the information contained herein are provided on an
+ "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS
+ OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY AND THE INTERNET
+ ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED,
+ INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE
+ INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED
+ WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
+
+Intellectual Property
+
+ The IETF takes no position regarding the validity or scope of any
+ Intellectual Property Rights or other rights that might be claimed to
+ pertain to the implementation or use of the technology described in
+ this document or the extent to which any license under such rights
+ might or might not be available; nor does it represent that it has
+ made any independent effort to identify any such rights. Information
+ on the procedures with respect to rights in RFC documents can be
+ found in BCP 78 and BCP 79.
+
+ Copies of IPR disclosures made to the IETF Secretariat and any
+ assurances of licenses to be made available, or the result of an
+ attempt made to obtain a general license or permission for the use of
+ such proprietary rights by implementers or users of this
+ specification can be obtained from the IETF on-line IPR repository at
+ http://www.ietf.org/ipr.
+
+ The IETF invites any interested party to bring to its attention any
+ copyrights, patents or patent applications, or other proprietary
+ rights that may cover technology that may be required to implement
+ this standard. Please address the information to the IETF at
+ ietf-ipr@ietf.org.
+
+Acknowledgement
+
+ Funding for the RFC Editor function is provided by the IETF
+ Administrative Support Activity (IASA).
+
+
+
+
+
+
+
+Crockford Informational [Page 10]
+
Added: hadoop/hive/trunk/lib/json.jar
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/lib/json.jar?rev=733992&view=auto
==============================================================================
Binary file - no diff available.
Propchange: hadoop/hive/trunk/lib/json.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Modified: hadoop/hive/trunk/ql/build.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/build.xml?rev=733992&r1=733991&r2=733992&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/build.xml (original)
+++ hadoop/hive/trunk/ql/build.xml Mon Jan 12 18:14:07 2009
@@ -167,6 +167,12 @@
<exclude name="META-INF/MANIFEST.MF"/>
</patternset>
</unzip>
+ <unzip src="${hive.root}/lib/json.jar" dest="${build.dir.hive}/json/classes">
+ <patternset>
+ <exclude name="META-INF"/>
+ <exclude name="META-INF/MANIFEST.MF"/>
+ </patternset>
+ </unzip>
<!-- jar jarfile="${build.dir}/hive_${name}.jar" basedir="${build.classes}" / -->
<jar jarfile="${build.dir}/hive_exec.jar">
<fileset dir="${build.dir.hive}/common/classes" includes="**/*.class"/>
@@ -175,6 +181,7 @@
<fileset dir="${build.dir.hive}/jexl/classes" includes="**/*.class"/>
<fileset dir="${build.dir.hive}/thrift/classes" includes="**/*.class"/>
<fileset dir="${build.dir.hive}/commons-lang/classes" includes="**/StringUtils.class,**/WordUtils.class"/>
+ <fileset dir="${build.dir.hive}/json/classes" includes="**/*.class"/>
</jar>
</target>
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java?rev=733992&r1=733991&r2=733992&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java Mon Jan 12 18:14:07 2009
@@ -86,7 +86,9 @@
registerUDF("year", UDFYear.class, OperatorType.PREFIX, false);
registerUDF("from_unixtime", UDFFromUnixTime.class, OperatorType.PREFIX, false);
registerUDF("to_date", UDFDate.class, OperatorType.PREFIX, false);
-
+
+ registerUDF("get_json_object", UDFJson.class, OperatorType.PREFIX, false);
+
registerUDF("+", UDFOPPlus.class, OperatorType.INFIX, true);
registerUDF("-", UDFOPMinus.class, OperatorType.INFIX, true);
registerUDF("*", UDFOPMultiply.class, OperatorType.INFIX, true);
Added: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFJson.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFJson.java?rev=733992&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFJson.java (added)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFJson.java Mon Jan 12 18:14:07 2009
@@ -0,0 +1,171 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.udf;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.json.JSONArray;
+import org.json.JSONObject;
+import org.json.JSONException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.exec.UDF;
+
+
+public class UDFJson implements UDF {
+ private static Log LOG = LogFactory.getLog(UDFJson.class.getName());
+ private Pattern pattern_key = Pattern.compile("^([a-zA-Z0-9_\\-]+).*");
+ private Pattern pattern_index = Pattern.compile("\\[([0-9]+|\\*)\\]");
+
+ public UDFJson() {
+ }
+
+ /**
+ * Extract json object from a json string based on json path specified,
+ * and return json string of the extracted json object. It will return null
+ * if the input json string is invalid.
+ *
+ * A limited version of JSONPath supported:
+ * $ : Root object
+ * . : Child operator
+ * [] : Subscript operator for array
+ * * : Wildcard for []
+ *
+ * Syntax not supported that's worth noticing:
+ * '' : Zero length string as key
+ * .. : Recursive descent
+ * @ : Current object/element
+ * () : Script expression
+ * ?() : Filter (script) expression.
+ * [,] : Union operator
+ * [start:end:step] : array slice operator
+ *
+ * @param jsonString the json string.
+ * @param pathString the json path expression.
+ * @return json string or null when error happens.
+ */
+ public String evaluate(String jsonString, String pathString) {
+ if (jsonString == null || pathString == null) {
+ return null;
+ }
+
+ try {
+ String[] pathExpr = pathString.split("\\.", -1);
+ if (!pathExpr[0].equalsIgnoreCase("$")) {
+ return null;
+ }
+ Object extractObject = new JSONObject(jsonString);
+ for (int i = 1; i < pathExpr.length; i++) {
+ extractObject = extract(extractObject, pathExpr[i]);
+ }
+ return extractObject.toString();
+ } catch (Exception e) {
+ return null;
+ }
+ }
+
+ private Object extract(Object json, String path) throws JSONException {
+ Matcher m_key = pattern_key.matcher(path);
+ if (!m_key.matches()) {
+ return null;
+ }
+ json = extract_json_withkey(json, m_key.group(1));
+
+ Matcher m_index = pattern_index.matcher(path);
+ ArrayList<String> index_list = new ArrayList<String>();
+ while (m_index.find()) {
+ index_list.add(m_index.group(1));
+ }
+ if (index_list.size() > 0) {
+ json = extract_json_withindex(json, index_list);
+ }
+
+ return json;
+ }
+
+ private Object extract_json_withindex(Object json, ArrayList<String> indexList)
+ throws JSONException {
+ ArrayList<Object> jsonList = new ArrayList<Object>();
+ jsonList.add(json);
+ Iterator<String> itr = indexList.iterator();
+ while (itr.hasNext()) {
+ String index = itr.next();
+ if (index.equalsIgnoreCase("*")) {
+ ArrayList<Object> tmp_jsonList = new ArrayList<Object>();
+ for (int i = 0; i < ((ArrayList<Object>) jsonList).size(); i++) {
+ try {
+ JSONArray array = (JSONArray) ((ArrayList<Object>) jsonList).get(i);
+ for (int j = 0; j < array.length(); j++) {
+ tmp_jsonList.add(array.get(j));
+ }
+ } catch (Exception e) {
+ continue;
+ }
+ }
+ jsonList = tmp_jsonList;
+ } else {
+ ArrayList<Object> tmp_jsonList = new ArrayList<Object>();
+ for (int i = 0; i < ((ArrayList<Object>) jsonList).size(); i++) {
+ try {
+ tmp_jsonList
+ .add(((JSONArray) ((ArrayList<Object>) jsonList).get(i))
+ .get(Integer.parseInt(index)));
+ } catch (ClassCastException e) {
+ continue;
+ } catch (JSONException e) {
+ return null;
+ }
+ jsonList = tmp_jsonList;
+ }
+ }
+ }
+ return (jsonList.size() > 1) ? new JSONArray((Collection) jsonList)
+ : jsonList.get(0);
+ }
+
+ private Object extract_json_withkey(Object json, String path)
+ throws JSONException {
+ if (json.getClass() == org.json.JSONArray.class) {
+ JSONArray jsonArray = new JSONArray();
+ for (int i = 0; i < ((JSONArray) json).length(); i++) {
+ Object josn_elem = ((JSONArray) json).get(i);
+ try {
+ Object json_obj = ((JSONObject) josn_elem).get(path);
+ if (json_obj.getClass() == org.json.JSONArray.class) {
+ for (int j = 0; j < ((JSONArray) json_obj).length(); j++) {
+ jsonArray.put(((JSONArray) json_obj).get(j));
+ }
+ } else {
+ jsonArray.put(json_obj);
+ }
+ } catch (Exception e) {
+ continue;
+ }
+ }
+ return (jsonArray.length() == 0) ? null : jsonArray;
+ } else {
+ return ((JSONObject) json).get(path);
+ }
+ }
+}
Modified: hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java?rev=733992&r1=733991&r2=733992&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (original)
+++ hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java Mon Jan 12 18:14:07 2009
@@ -198,7 +198,7 @@
public void cleanUp() throws Exception {
String warehousePath = ((new URI(testWarehouse)).getPath());
// Drop any tables that remain due to unsuccessful runs
- for(String s: new String [] {"src", "src1", "src_thrift", "src_sequencefile",
+ for(String s: new String [] {"src", "src1", "src_json", "src_thrift", "src_sequencefile",
"srcpart", "srcbucket", "dest1", "dest2",
"dest3", "dest4", "dest4_sequencefile",
"dest_j1", "dest_j2", "dest_g1", "dest_g2"}) {
@@ -279,6 +279,11 @@
db.createTable(srcThrift);
srcTables.add("src_thrift");
+ LinkedList<String> json_cols = new LinkedList<String>();
+ json_cols.add("json");
+ db.createTable("src_json", json_cols, null, TextInputFormat.class, IgnoreKeyTextOutputFormat.class);
+ srcTables.add("src_json");
+
// load the input data into the src table
fpath = new Path(testFiles, "kv1.txt");
newfpath = new Path(tmppath, "kv1.txt");
@@ -307,6 +312,13 @@
newfpath = new Path(tmppath, "complex.seq");
fs.copyFromLocalFile(false, true, fpath, newfpath);
db.loadTable(newfpath, "src_thrift", true);
+
+ // load the json data into the src_json table
+ fpath = new Path(testFiles, "json.txt");
+ newfpath = new Path(tmppath, "json.txt");
+ fs.copyFromLocalFile(false, true, fpath, newfpath);
+ db.loadTable(newfpath, "src_json", false);
+
}
public void init() throws Exception {
Added: hadoop/hive/trunk/ql/src/test/queries/clientpositive/udf_json.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/udf_json.q?rev=733992&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/udf_json.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/udf_json.q Mon Jan 12 18:14:07 2009
@@ -0,0 +1,22 @@
+CREATE TABLE dest1(c1 STRING) STORED AS TEXTFILE;
+
+FROM src INSERT OVERWRITE TABLE dest1 SELECT ' abc ' WHERE src.key = 86;
+
+EXPLAIN
+SELECT get_json_object(src_json.json, '$.owner') FROM src_json;
+
+SELECT get_json_object(src_json.json, '$') FROM src_json;
+
+SELECT get_json_object(src_json.json, '$.owner'), get_json_object(src_json.json, '$.store') FROM src_json;
+
+SELECT get_json_object(src_json.json, '$.store.bicycle'), get_json_object(src_json.json, '$.store.book') FROM src_json;
+
+SELECT get_json_object(src_json.json, '$.store.book[0]'), get_json_object(src_json.json, '$.store.book[*]') FROM src_json;
+
+SELECT get_json_object(src_json.json, '$.store.book[0].category'), get_json_object(src_json.json, '$.store.book[*].category'), get_json_object(src_json.json, '$.store.book[*].isbn'), get_json_object(src_json.json, '$.store.book[*].reader') FROM src_json;
+
+SELECT get_json_object(src_json.json, '$.store.book[*].reader[0].age'), get_json_object(src_json.json, '$.store.book[*].reader[*].age') FROM src_json;
+
+SELECT get_json_object(src_json.json, '$.store.basket[0][1]'), get_json_object(src_json.json, '$.store.basket[*]'), get_json_object(src_json.json, '$.store.basket[*][0]'), get_json_object(src_json.json, '$.store.basket[0][*]'), get_json_object(src_json.json, '$.store.basket[*][*]'), get_json_object(src_json.json, '$.store.basket[0][2].b'), get_json_object(src_json.json, '$.store.basket[0][*].b') FROM src_json;
+
+SELECT get_json_object(src_json.json, '$.non_exist_key'), get_json_object(src_json.json, '$..no_recursive'), get_json_object(src_json.json, '$.store.book[10]'), get_json_object(src_json.json, '$.store.book[0].non_exist_key'), get_json_object(src_json.json, '$.store.basket[*].non_exist_key'), get_json_object(src_json.json, '$.store.basket[0][*].non_exist_key') FROM src_json;
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/input2.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/input2.q.out?rev=733992&r1=733991&r2=733992&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/input2.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/input2.q.out Mon Jan 12 18:14:07 2009
@@ -3,8 +3,8 @@
a array<int>
b double
c map<double,int>
-src src1 src_sequencefile src_thrift srcbucket srcpart test2a test2b
-src src1 src_sequencefile src_thrift srcbucket srcpart test2b
+src src1 src_json src_sequencefile src_thrift srcbucket srcpart test2a test2b
+src src1 src_json src_sequencefile src_thrift srcbucket srcpart test2b
ABSTRACT SYNTAX TREE:
TOK_SHOWTABLES
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/input3.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/input3.q.out?rev=733992&r1=733991&r2=733992&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/input3.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/input3.q.out Mon Jan 12 18:14:07 2009
@@ -3,7 +3,7 @@
a array<int>
b double
c map<double,int>
-src src1 src_sequencefile src_thrift srcbucket srcpart test3a test3b
+src src1 src_json src_sequencefile src_thrift srcbucket srcpart test3a test3b
ABSTRACT SYNTAX TREE:
(TOK_ALTERTABLE_ADDCOLS TEST3b (TOK_TABCOLLIST (TOK_TABCOL X TOK_DOUBLE)))
@@ -42,7 +42,7 @@
b double
c map<double,int>
x double
-src src1 src_sequencefile src_thrift srcbucket srcpart test3a test3c
+src src1 src_json src_sequencefile src_thrift srcbucket srcpart test3a test3c
ABSTRACT SYNTAX TREE:
(TOK_ALTERTABLE_REPLACECOLS TEST3c (TOK_TABCOLLIST (TOK_TABCOL R1 TOK_INT) (TOK_TABCOL R2 TOK_DOUBLE)))
@@ -61,4 +61,4 @@
r1 int
r2 double
Detailed Table Information:
-Table(tableName:test3c,dbName:default,owner:zshao,createTime:1229498330,lastAccessTime:0,retention:0,sd:StorageDescriptor(cols:[FieldSchema(name:r1,type:int,comment:null), FieldSchema(name:r2,type:double,comment:null)],location:file:/data/users/zshao/sync/apache-trunk-HIVE-104/build/ql/test/data/warehouse/test3b,inputFormat:org.apache.hadoop.mapred.TextInputFormat,outputFormat:org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat,compressed:false,numBuckets:-1,serdeInfo:SerDeInfo(name:null,serializationLib:org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe,parameters:{serialization.format=org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol}),bucketCols:[],sortCols:[],parameters:{}),partitionKeys:[],parameters:{last_modified_by=zshao,last_modified_time=1229498331})
+Table(tableName:test3c,dbName:default,owner:zshao,createTime:1231804109,lastAccessTime:0,retention:0,sd:StorageDescriptor(cols:[FieldSchema(name:r1,type:int,comment:null), FieldSchema(name:r2,type:double,comment:null)],location:file:/data/users/zshao/sync/pp/build/ql/test/data/warehouse/test3b,inputFormat:org.apache.hadoop.mapred.TextInputFormat,outputFormat:org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat,compressed:false,numBuckets:-1,serdeInfo:SerDeInfo(name:null,serializationLib:org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe,parameters:{serialization.format=org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol}),bucketCols:[],sortCols:[],parameters:{}),partitionKeys:[],parameters:{last_modified_by=zshao,last_modified_time=1231804109})
Added: hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_json.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_json.q.out?rev=733992&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_json.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_json.q.out Mon Jan 12 18:14:07 2009
@@ -0,0 +1,35 @@
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_TABREF src_json)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION get_json_object (TOK_COLREF src_json json) '$.owner')))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src_json
+ Select Operator
+ expressions:
+ expr: get_json_object(json, '$.owner')
+ type: string
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+{"store":{"fruit":[{"weight":8,"type":"apple"},{"weight":9,"type":"pear"}],"book":[{"author":"Nigel Rees","category":"reference","title":"Sayings of the Century","price":8.95},{"author":"Herman Melville","category":"fiction","title":"Moby Dick","price":8.99,"isbn":"0-553-21311-3"},{"author":"J. R. R. Tolkien","category":"fiction","title":"The Lord of the Rings","price":22.99,"reader":[{"name":"bob","age":25},{"name":"jack","age":26}],"isbn":"0-395-19395-8"}],"basket":[[1,2,{"b":"y","a":"x"}],[3,4],[5,6]],"bicycle":{"price":19.95,"color":"red"}},"email":"amy@only_for_json_udf_test.net","owner":"amy"}
+amy {"fruit":[{"weight":8,"type":"apple"},{"weight":9,"type":"pear"}],"book":[{"author":"Nigel Rees","category":"reference","title":"Sayings of the Century","price":8.95},{"author":"Herman Melville","category":"fiction","title":"Moby Dick","price":8.99,"isbn":"0-553-21311-3"},{"author":"J. R. R. Tolkien","category":"fiction","title":"The Lord of the Rings","price":22.99,"reader":[{"name":"bob","age":25},{"name":"jack","age":26}],"isbn":"0-395-19395-8"}],"basket":[[1,2,{"b":"y","a":"x"}],[3,4],[5,6]],"bicycle":{"price":19.95,"color":"red"}}
+{"price":19.95,"color":"red"} [{"author":"Nigel Rees","category":"reference","title":"Sayings of the Century","price":8.95},{"author":"Herman Melville","category":"fiction","title":"Moby Dick","price":8.99,"isbn":"0-553-21311-3"},{"author":"J. R. R. Tolkien","category":"fiction","title":"The Lord of the Rings","price":22.99,"reader":[{"name":"bob","age":25},{"name":"jack","age":26}],"isbn":"0-395-19395-8"}]
+{"author":"Nigel Rees","category":"reference","title":"Sayings of the Century","price":8.95} [{"author":"Nigel Rees","category":"reference","title":"Sayings of the Century","price":8.95},{"author":"Herman Melville","category":"fiction","title":"Moby Dick","price":8.99,"isbn":"0-553-21311-3"},{"author":"J. R. R. Tolkien","category":"fiction","title":"The Lord of the Rings","price":22.99,"reader":[{"name":"bob","age":25},{"name":"jack","age":26}],"isbn":"0-395-19395-8"}]
+reference ["reference","fiction","fiction"] ["0-553-21311-3","0-395-19395-8"] [{"name":"bob","age":25},{"name":"jack","age":26}]
+25 [25,26]
+2 [[1,2,{"b":"y","a":"x"}],[3,4],[5,6]] 1 [1,2,{"b":"y","a":"x"}] [1,2,{"b":"y","a":"x"},3,4,5,6] y ["y"]
+NULL NULL NULL NULL NULL NULL