You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2018/05/10 21:48:53 UTC
[1/8] impala git commit: Update version to 3.1.0-SNAPSHOT
Repository: impala
Updated Branches:
refs/heads/master 3e7364503 -> 49413d9c5
Update version to 3.1.0-SNAPSHOT
Change-Id: Ia2e2df73d27fa332d17fec4e9aa469ea91b14167
Reviewed-on: http://gerrit.cloudera.org:8080/10360
Reviewed-by: Michael Brown <mi...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/8668d18e
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/8668d18e
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/8668d18e
Branch: refs/heads/master
Commit: 8668d18ebb0e137993a12d5303411b6c06e91c59
Parents: 3e73645
Author: Sailesh Mukil <sa...@cloudera.com>
Authored: Wed May 9 13:40:21 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Thu May 10 18:55:08 2018 +0000
----------------------------------------------------------------------
bin/save-version.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/8668d18e/bin/save-version.sh
----------------------------------------------------------------------
diff --git a/bin/save-version.sh b/bin/save-version.sh
index 46de902..0a433dd 100755
--- a/bin/save-version.sh
+++ b/bin/save-version.sh
@@ -21,7 +21,7 @@
# Note: for internal (aka pre-release) versions, the version should have
# "-INTERNAL" appended. Parts of the code will look for this to distinguish
# between released and internal versions.
-VERSION=3.0.0-SNAPSHOT
+VERSION=3.1.0-SNAPSHOT
GIT_HASH=$(git rev-parse HEAD 2> /dev/null)
if [ -z $GIT_HASH ]
then
[8/8] impala git commit: IMPALA-6999: Upgrade to sqlparse-0.1.19 for
Impala shell
Posted by jo...@apache.org.
IMPALA-6999: Upgrade to sqlparse-0.1.19 for Impala shell
sqlparse-0.1.19 is the last version of sqlparse that supports Python
2.6.
Testing:
- Ran all end-to-end tests
Change-Id: Ide51ef3ac52d25a96b0fa832e29b6535197d23cb
Reviewed-on: http://gerrit.cloudera.org:8080/10354
Reviewed-by: David Knupp <dk...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/49413d9c
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/49413d9c
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/49413d9c
Branch: refs/heads/master
Commit: 49413d9c5b96153b79ce5aa756a2411e25610fd2
Parents: 8668d18
Author: Fredy Wijaya <fw...@cloudera.com>
Authored: Tue May 8 20:22:11 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Thu May 10 19:27:52 2018 +0000
----------------------------------------------------------------------
LICENSE.txt | 4 +-
README.md | 2 +-
bin/rat_exclude_files.txt | 2 +-
shell/.gitignore | 6 +-
shell/ext-py/sqlparse-0.1.14/AUTHORS | 26 -
shell/ext-py/sqlparse-0.1.14/CHANGES | 238 ------
shell/ext-py/sqlparse-0.1.14/COPYING | 25 -
shell/ext-py/sqlparse-0.1.14/MANIFEST.in | 11 -
shell/ext-py/sqlparse-0.1.14/PKG-INFO | 74 --
shell/ext-py/sqlparse-0.1.14/README.rst | 56 --
shell/ext-py/sqlparse-0.1.14/TODO | 7 -
shell/ext-py/sqlparse-0.1.14/bin/sqlformat | 109 ---
.../sqlparse-0.1.14/docs/source/analyzing.rst | 64 --
.../ext-py/sqlparse-0.1.14/docs/source/api.rst | 57 --
.../sqlparse-0.1.14/docs/source/changes.rst | 13 -
.../ext-py/sqlparse-0.1.14/docs/source/conf.py | 200 -----
.../sqlparse-0.1.14/docs/source/index.rst | 61 --
.../sqlparse-0.1.14/docs/source/indices.rst | 7 -
.../sqlparse-0.1.14/docs/source/intro.rst | 143 ----
shell/ext-py/sqlparse-0.1.14/docs/source/ui.rst | 15 -
shell/ext-py/sqlparse-0.1.14/docs/sqlformat.1 | 65 --
shell/ext-py/sqlparse-0.1.14/pytest.ini | 6 -
shell/ext-py/sqlparse-0.1.14/setup.cfg | 5 -
shell/ext-py/sqlparse-0.1.14/setup.py | 123 ----
.../ext-py/sqlparse-0.1.14/sqlparse/__init__.py | 78 --
.../sqlparse-0.1.14/sqlparse/engine/__init__.py | 80 --
.../sqlparse-0.1.14/sqlparse/engine/filter.py | 109 ---
.../sqlparse-0.1.14/sqlparse/engine/grouping.py | 390 ----------
.../sqlparse-0.1.14/sqlparse/exceptions.py | 10 -
.../ext-py/sqlparse-0.1.14/sqlparse/filters.py | 716 ------------------
.../sqlparse-0.1.14/sqlparse/formatter.py | 137 ----
.../sqlparse-0.1.14/sqlparse/functions.py | 44 --
.../ext-py/sqlparse-0.1.14/sqlparse/keywords.py | 571 ---------------
shell/ext-py/sqlparse-0.1.14/sqlparse/lexer.py | 350 ---------
.../ext-py/sqlparse-0.1.14/sqlparse/pipeline.py | 31 -
shell/ext-py/sqlparse-0.1.14/sqlparse/sql.py | 639 ----------------
shell/ext-py/sqlparse-0.1.14/sqlparse/tokens.py | 83 ---
shell/ext-py/sqlparse-0.1.14/sqlparse/utils.py | 137 ----
shell/ext-py/sqlparse-0.1.14/tests/__init__.py | 0
.../tests/files/_Make_DirEntry.sql | 6 -
.../sqlparse-0.1.14/tests/files/begintag.sql | 4 -
.../sqlparse-0.1.14/tests/files/begintag_2.sql | 13 -
.../sqlparse-0.1.14/tests/files/dashcomment.sql | 5 -
.../sqlparse-0.1.14/tests/files/function.sql | 13 -
.../tests/files/function_psql.sql | 72 --
.../tests/files/function_psql2.sql | 7 -
.../tests/files/function_psql3.sql | 8 -
.../sqlparse-0.1.14/tests/files/huge_select.sql | 1 -
.../sqlparse-0.1.14/tests/files/test_cp1251.sql | 1 -
.../sqlparse-0.1.14/tests/test_filters.py | 78 --
.../ext-py/sqlparse-0.1.14/tests/test_format.py | 328 ---------
.../sqlparse-0.1.14/tests/test_functions.py | 164 -----
.../sqlparse-0.1.14/tests/test_grouping.py | 341 ---------
.../ext-py/sqlparse-0.1.14/tests/test_parse.py | 191 -----
.../sqlparse-0.1.14/tests/test_pipeline.py | 70 --
.../sqlparse-0.1.14/tests/test_regressions.py | 246 -------
.../ext-py/sqlparse-0.1.14/tests/test_split.py | 144 ----
.../sqlparse-0.1.14/tests/test_tokenize.py | 190 -----
shell/ext-py/sqlparse-0.1.14/tests/utils.py | 46 --
shell/ext-py/sqlparse-0.1.14/tox.ini | 37 -
shell/ext-py/sqlparse-0.1.19/.travis.yml | 14 +
shell/ext-py/sqlparse-0.1.19/AUTHORS | 31 +
shell/ext-py/sqlparse-0.1.19/CHANGES | 302 ++++++++
shell/ext-py/sqlparse-0.1.19/COPYING | 25 +
shell/ext-py/sqlparse-0.1.19/MANIFEST.in | 11 +
shell/ext-py/sqlparse-0.1.19/README.rst | 56 ++
shell/ext-py/sqlparse-0.1.19/TODO | 7 +
shell/ext-py/sqlparse-0.1.19/bin/sqlformat | 109 +++
.../sqlparse-0.1.19/docs/source/analyzing.rst | 64 ++
.../ext-py/sqlparse-0.1.19/docs/source/api.rst | 57 ++
.../sqlparse-0.1.19/docs/source/changes.rst | 13 +
.../ext-py/sqlparse-0.1.19/docs/source/conf.py | 200 +++++
.../sqlparse-0.1.19/docs/source/index.rst | 61 ++
.../sqlparse-0.1.19/docs/source/indices.rst | 7 +
.../sqlparse-0.1.19/docs/source/intro.rst | 143 ++++
shell/ext-py/sqlparse-0.1.19/docs/source/ui.rst | 15 +
shell/ext-py/sqlparse-0.1.19/docs/sqlformat.1 | 65 ++
shell/ext-py/sqlparse-0.1.19/pytest.ini | 6 +
shell/ext-py/sqlparse-0.1.19/setup.py | 123 ++++
.../ext-py/sqlparse-0.1.19/sqlparse/__init__.py | 78 ++
.../sqlparse-0.1.19/sqlparse/engine/__init__.py | 80 ++
.../sqlparse-0.1.19/sqlparse/engine/filter.py | 112 +++
.../sqlparse-0.1.19/sqlparse/engine/grouping.py | 461 ++++++++++++
.../sqlparse-0.1.19/sqlparse/exceptions.py | 10 +
.../ext-py/sqlparse-0.1.19/sqlparse/filters.py | 728 +++++++++++++++++++
.../sqlparse-0.1.19/sqlparse/formatter.py | 137 ++++
.../sqlparse-0.1.19/sqlparse/functions.py | 44 ++
.../ext-py/sqlparse-0.1.19/sqlparse/keywords.py | 574 +++++++++++++++
shell/ext-py/sqlparse-0.1.19/sqlparse/lexer.py | 362 +++++++++
.../ext-py/sqlparse-0.1.19/sqlparse/pipeline.py | 31 +
shell/ext-py/sqlparse-0.1.19/sqlparse/sql.py | 684 +++++++++++++++++
shell/ext-py/sqlparse-0.1.19/sqlparse/tokens.py | 83 +++
shell/ext-py/sqlparse-0.1.19/sqlparse/utils.py | 137 ++++
shell/ext-py/sqlparse-0.1.19/tests/__init__.py | 0
.../tests/files/_Make_DirEntry.sql | 6 +
.../sqlparse-0.1.19/tests/files/begintag.sql | 4 +
.../sqlparse-0.1.19/tests/files/begintag_2.sql | 13 +
.../sqlparse-0.1.19/tests/files/dashcomment.sql | 5 +
.../sqlparse-0.1.19/tests/files/function.sql | 13 +
.../tests/files/function_psql.sql | 72 ++
.../tests/files/function_psql2.sql | 7 +
.../tests/files/function_psql3.sql | 8 +
.../sqlparse-0.1.19/tests/files/huge_select.sql | 1 +
.../sqlparse-0.1.19/tests/files/test_cp1251.sql | 1 +
.../sqlparse-0.1.19/tests/test_filters.py | 78 ++
.../ext-py/sqlparse-0.1.19/tests/test_format.py | 346 +++++++++
.../sqlparse-0.1.19/tests/test_functions.py | 164 +++++
.../sqlparse-0.1.19/tests/test_grouping.py | 399 ++++++++++
.../ext-py/sqlparse-0.1.19/tests/test_parse.py | 305 ++++++++
.../sqlparse-0.1.19/tests/test_pipeline.py | 70 ++
.../sqlparse-0.1.19/tests/test_regressions.py | 276 +++++++
.../ext-py/sqlparse-0.1.19/tests/test_split.py | 144 ++++
.../sqlparse-0.1.19/tests/test_tokenize.py | 190 +++++
shell/ext-py/sqlparse-0.1.19/tests/utils.py | 46 ++
shell/ext-py/sqlparse-0.1.19/tox.ini | 37 +
115 files changed, 7022 insertions(+), 6652 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/LICENSE.txt
----------------------------------------------------------------------
diff --git a/LICENSE.txt b/LICENSE.txt
index 1f6e222..789aa18 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -589,7 +589,7 @@ shell/ext-py/prettytable-0.7.1: 3-clause BSD
--------------------------------------------------------------------------------
-shell/ext-py/sqlparse-0.1.14 (most parts): 3-clause BSD
+shell/ext-py/sqlparse-0.1.19 (most parts): 3-clause BSD
Copyright (c) 2009, Andi Albrecht <al...@gmail.com>
@@ -641,7 +641,7 @@ THE SOFTWARE.
--------------------------------------------------------------------------------
-shell/ext-py/sqlparse-0.1.14/sqlparse/pipeline.py:
+shell/ext-py/sqlparse-0.1.19/sqlparse/pipeline.py:
Copyright (C) 2011 Jesus Leganes "piranna", piranna@gmail.com
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 1227fa2..c0565ab 100644
--- a/README.md
+++ b/README.md
@@ -72,7 +72,7 @@ can do so through the environment variables and scripts listed below.
| JAVA_VERSION | "java-7-oracle-amd64" | Can override to set a local Java version. |
| JAVA | "${JAVA_HOME}/bin/java" | Java binary location. |
| CLASSPATH | | See bin/set-classpath.sh for details. |
-| PYTHONPATH | Will be changed to include: "${IMPALA_HOME}/shell/gen-py" "${IMPALA_HOME}/testdata" "${THRIFT_HOME}/python/lib/python2.7/site-packages" "${HIVE_HOME}/lib/py" "${IMPALA_HOME}/shell/ext-py/prettytable-0.7.1/dist/prettytable-0.7.1" "${IMPALA_HOME}/shell/ext-py/sasl-0.1.1/dist/sasl-0.1.1-py2.7-linux-x "${IMPALA_HOME}/shell/ext-py/sqlparse-0.1.14/dist/sqlparse-0.1.14-py2 |
+| PYTHONPATH | Will be changed to include: "${IMPALA_HOME}/shell/gen-py" "${IMPALA_HOME}/testdata" "${THRIFT_HOME}/python/lib/python2.7/site-packages" "${HIVE_HOME}/lib/py" "${IMPALA_HOME}/shell/ext-py/prettytable-0.7.1/dist/prettytable-0.7.1" "${IMPALA_HOME}/shell/ext-py/sasl-0.1.1/dist/sasl-0.1.1-py2.7-linux-x "${IMPALA_HOME}/shell/ext-py/sqlparse-0.1.19/dist/sqlparse-0.1.19-py2 |
##### Source Directories for Impala
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/bin/rat_exclude_files.txt
----------------------------------------------------------------------
diff --git a/bin/rat_exclude_files.txt b/bin/rat_exclude_files.txt
index aba4a94..a41cf3f 100644
--- a/bin/rat_exclude_files.txt
+++ b/bin/rat_exclude_files.txt
@@ -34,7 +34,7 @@ tests/comparison/leopard/static/css/bootstrap*
tests/comparison/leopard/static/fonts/glyphicons-halflings*
tests/comparison/leopard/static/js/bootstrap*
shell/ext-py/prettytable-0.7.1/*
-shell/ext-py/sqlparse-0.1.14/*
+shell/ext-py/sqlparse-0.1.19/*
shell/ext-py/sasl-0.1.1/*
www/d3.v3.min.js
www/jquery/jquery-1.12.4.min.js
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/.gitignore
----------------------------------------------------------------------
diff --git a/shell/.gitignore b/shell/.gitignore
index 436fee4..f9874f8 100644
--- a/shell/.gitignore
+++ b/shell/.gitignore
@@ -5,9 +5,9 @@ ext-py/sasl-0.1.1/sasl.egg-info/
ext-py/prettytable-0.7.1/dist/
ext-py/prettytable-0.7.1/build
ext-py/prettytable-0.7.1/prettytable.egg-info
-ext-py/sqlparse-0.1.14/dist/
-ext-py/sqlparse-0.1.14/build/
-ext-py/sqlparse-0.1.14/sqlparse.egg-info/
+ext-py/sqlparse-0.1.19/dist/
+ext-py/sqlparse-0.1.19/build/
+ext-py/sqlparse-0.1.19/sqlparse.egg-info/
# This file is used by buildall.sh to find files that need to be removed during the
# clean phase. Previous version of deps should be kept here for cleaning otherwise they
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/AUTHORS
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/AUTHORS b/shell/ext-py/sqlparse-0.1.14/AUTHORS
deleted file mode 100644
index f768180..0000000
--- a/shell/ext-py/sqlparse-0.1.14/AUTHORS
+++ /dev/null
@@ -1,26 +0,0 @@
-python-sqlparse is written and maintained by Andi Albrecht <al...@gmail.com>.
-
-This module contains code (namely the lexer and filter mechanism) from
-the pygments project that was written by Georg Brandl.
-
-Alphabetical list of contributors:
-* Alexander Beedie <ay...@gmail.com>
-* Alexey Malyshev <no...@gmail.com>
-* Cristian Orellana <cr...@groupon.com>
-* Florian Bauer <fl...@zmdi.com>
-* Gavin Wahl <gw...@fusionbox.com>
-* Jesús Leganés Combarro "Piranna" <pi...@gmail.com>
-* Kevin Jing Qiu <ke...@gmail.com>
-* Michael Schuller <mi...@artlogic.net>
-* Mike Amy <co...@googlemail.com>
-* mulos <da...@gmail.com>
-* Piet Delport <pj...@gmail.com>
-* Prudhvi Vatala <pv...@gmail.com>
-* quest <qu...@wonky.windwards.net>
-* Robert Nix <co...@rnix.org>
-* Rocky Meza <rm...@fusionbox.com>
-* spigwitmer <it...@gmail.com>
-* Victor Hahn <in...@victor-hahn.de>
-* vthriller <fa...@yandex.ru>
-* wayne.wuw <wa...@alibaba-inc.com>
-* Yago Riveiro <ya...@gmail.com>
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/CHANGES
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/CHANGES b/shell/ext-py/sqlparse-0.1.14/CHANGES
deleted file mode 100644
index 8f0a108..0000000
--- a/shell/ext-py/sqlparse-0.1.14/CHANGES
+++ /dev/null
@@ -1,238 +0,0 @@
-Release 0.1.14 (Nov 30, 2014)
------------------------------
-
-Bug Fixes
-* Floats in UPDATE statements are now handled correctly (issue145).
-* Properly handle string literals in comparisons (issue148, change proposed
- by aadis).
-* Fix indentation when using tabs (issue146).
-
-Enhancements
-* Improved formatting in list when newlines precede commas (issue140).
-
-
-Release 0.1.13 (Oct 09, 2014)
------------------------------
-
-Bug Fixes
-* Fix a regression in handling of NULL keywords introduced in 0.1.12.
-
-
-Release 0.1.12 (Sep 20, 2014)
------------------------------
-
-Bug Fixes
-* Fix handling of NULL keywords in aliased identifiers.
-* Fix SerializerUnicode to split unquoted newlines (issue131, by Michael Schuller).
-* Fix handling of modulo operators without spaces (by gavinwahl).
-
-Enhancements
-* Improve parsing of identifier lists containing placeholders.
-* Speed up query parsing of unquoted lines (by Michael Schuller).
-
-
-Release 0.1.11 (Feb 07, 2014)
------------------------------
-
-Bug Fixes
-* Fix incorrect parsing of string literals containing line breaks (issue118).
-* Fix typo in keywords, add MERGE, COLLECT keywords (issue122/124,
- by Cristian Orellana).
-* Improve parsing of string literals in columns.
-* Fix parsing and formatting of statements containing EXCEPT keyword.
-* Fix Function.get_parameters() (issue126/127, by spigwitmer).
-
-Enhancements
-* Classify DML keywords (issue116, by Victor Hahn).
-* Add missing FOREACH keyword.
-* Grouping of BEGIN/END blocks.
-
-Other
-* Python 2.5 isn't automatically tested anymore, neither Travis nor Tox
- still support it out of the box.
-
-
-Release 0.1.10 (Nov 02, 2013)
------------------------------
-
-Bug Fixes
-* Removed buffered reading again, it obviously causes wrong parsing in some rare
- cases (issue114).
-* Fix regression in setup.py introduced 10 months ago (issue115).
-
-Enhancements
-* Improved support for JOINs, by Alexander Beedie.
-
-
-Release 0.1.9 (Sep 28, 2013)
-----------------------------
-
-Bug Fixes
-* Fix an regression introduced in 0.1.5 where sqlparse didn't properly
- distinguished between single and double quoted strings when tagging
- identifier (issue111).
-
-Enhancements
-* New option to truncate long string literals when formatting.
-* Scientific numbers are pares correctly (issue107).
-* Support for arithmetic expressions (issue109, issue106; by prudhvi).
-
-
-Release 0.1.8 (Jun 29, 2013)
-----------------------------
-
-Bug Fixes
-* Whitespaces within certain keywords are now allowed (issue97, patch proposed
- by xcombelle).
-
-Enhancements
-* Improve parsing of assignments in UPDATE statements (issue90).
-* Add STRAIGHT_JOIN statement (by Yago Riveiro).
-* Function.get_parameters() now returns the parameter if only one parameter is
- given (issue94, by wayne.wuw).
-* sqlparse.split() now removes leading and trailing whitespaces from splitted
- statements.
-* Add USE as keyword token (by mulos).
-* Improve parsing of PEP249-style placeholders (issue103).
-
-
-Release 0.1.7 (Apr 06, 2013)
-----------------------------
-
-Bug Fixes
- * Fix Python 3 compatibility of sqlformat script (by Piet Delport).
- * Fix parsing of SQL statements that contain binary data (by Alexey
- Malyshev).
- * Fix a bug where keywords were identified as aliased identifiers in
- invalid SQL statements.
- * Fix parsing of identifier lists where identifiers are keywords too
- (issue10).
-
-Enhancements
- * Top-level API functions now accept encoding keyword to parse
- statements in certain encodings more reliable (issue20).
- * Improve parsing speed when SQL contains CLOBs or BLOBs (issue86).
- * Improve formatting of ORDER BY clauses (issue89).
- * Formatter now tries to detect runaway indentations caused by
- parsing errors or invalid SQL statements. When re-indenting such
- statements the formatter flips back to column 0 before going crazy.
-
-Other
- * Documentation updates.
-
-
-Release 0.1.6 (Jan 01, 2013)
-----------------------------
-
-sqlparse is now compatible with Python 3 without any patches. The
-Python 3 version is generated during install by 2to3. You'll need
-distribute to install sqlparse for Python 3.
-
-Bug Fixes
- * Fix parsing error with dollar-quoted procedure bodies (issue83).
-
-Other
- * Documentation updates.
- * Test suite now uses tox and py.test.
- * py3k fixes (by vthriller).
- * py3k fixes in setup.py (by Florian Bauer).
- * setup.py now requires distribute (by Florian Bauer).
-
-
-Release 0.1.5 (Nov 13, 2012)
-----------------------------
-
-Bug Fixes
- * Improve handling of quoted identifiers (issue78).
- * Improve grouping and formatting of identifiers with operators (issue53).
- * Improve grouping and formatting of concatenated strings (issue53).
- * Improve handling of varchar() (by Mike Amy).
- * Clean up handling of various SQL elements.
- * Switch to py.test and clean up tests.
- * Several minor fixes.
-
-Other
- * Deprecate sqlparse.SQLParseError. Please use
- sqlparse.exceptions.SQLParseError instead.
- * Add caching to speed up processing.
- * Add experimental filters for token processing.
- * Add sqlformat.parsestream (by quest).
-
-
-Release 0.1.4 (Apr 20, 2012)
-----------------------------
-
-Bug Fixes
- * Avoid "stair case" effects when identifiers, functions,
- placeholders or keywords are mixed in identifier lists (issue45,
- issue49, issue52) and when asterisks are used as operators
- (issue58).
- * Make keyword detection more restrict (issue47).
- * Improve handling of CASE statements (issue46).
- * Fix statement splitting when parsing recursive statements (issue57,
- thanks to piranna).
- * Fix for negative numbers (issue56, thanks to kevinjqiu).
- * Pretty format comments in identifier lists (issue59).
- * Several minor bug fixes and improvements.
-
-
-Release 0.1.3 (Jul 29, 2011)
-----------------------------
-
-Bug Fixes
- * Improve parsing of floats (thanks to Kris).
- * When formatting a statement a space before LIMIT was removed (issue35).
- * Fix strip_comments flag (issue38, reported by ooberm...@gmail.com).
- * Avoid parsing names as keywords (issue39, reported by djo...@taket.org).
- * Make sure identifier lists in subselects are grouped (issue40,
- reported by djo...@taket.org).
- * Split statements with IF as functions correctly (issue33 and
- issue29, reported by charles....@unige.ch).
- * Relax detection of keywords, esp. when used as function names
- (issue36, nyuhu...@gmail.com).
- * Don't treat single characters as keywords (issue32).
- * Improve parsing of stand-alone comments (issue26).
- * Detection of placeholders in paramterized queries (issue22,
- reported by Glyph Lefkowitz).
- * Add parsing of MS Access column names with braces (issue27,
- reported by frankz...@gmail.com).
-
-Other
- * Replace Django by Flask in App Engine frontend (issue11).
-
-
-Release 0.1.2 (Nov 23, 2010)
-----------------------------
-
-Bug Fixes
- * Fixed incorrect detection of keyword fragments embed in names (issue7,
- reported and initial patch by andyboyko).
- * Stricter detection of identfier aliases (issue8, reported by estama).
- * WHERE grouping consumed closing parenthesis (issue9, reported by estama).
- * Fixed an issue with trailing whitespaces (reported by Kris).
- * Better detection of escaped single quotes (issue13, reported by
- Martin Brochhaus, patch by bluemaro with test case by Dan Carley).
- * Ignore identifier in double-quotes when changing cases (issue 21).
- * Lots of minor fixes targeting encoding, indentation, statement
- parsing and more (issues 12, 14, 15, 16, 18, 19).
- * Code cleanup with a pinch of refactoring.
-
-
-Release 0.1.1 (May 6, 2009)
----------------------------
-
-Bug Fixes
- * Lexers preserves original line breaks (issue1).
- * Improved identifier parsing: backtick quotes, wildcards, T-SQL variables
- prefixed with @.
- * Improved parsing of identifier lists (issue2).
- * Recursive recognition of AS (issue4) and CASE.
- * Improved support for UPDATE statements.
-
-Other
- * Code cleanup and better test coverage.
-
-
-Release 0.1.0 (Apr 8, 2009)
----------------------------
- * Initial release.
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/COPYING
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/COPYING b/shell/ext-py/sqlparse-0.1.14/COPYING
deleted file mode 100644
index 7b158da..0000000
--- a/shell/ext-py/sqlparse-0.1.14/COPYING
+++ /dev/null
@@ -1,25 +0,0 @@
-Copyright (c) 2009, Andi Albrecht <al...@gmail.com>
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without modification,
-are permitted provided that the following conditions are met:
-
- * Redistributions of source code must retain the above copyright notice,
- this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright notice,
- this list of conditions and the following disclaimer in the documentation
- and/or other materials provided with the distribution.
- * Neither the name of the authors nor the names of its contributors may be
- used to endorse or promote products derived from this software without
- specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/MANIFEST.in
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/MANIFEST.in b/shell/ext-py/sqlparse-0.1.14/MANIFEST.in
deleted file mode 100644
index e546b1e..0000000
--- a/shell/ext-py/sqlparse-0.1.14/MANIFEST.in
+++ /dev/null
@@ -1,11 +0,0 @@
-recursive-include docs source/*
-include docs/sqlformat.1
-include docs/Makefile
-recursive-include tests *.py *.sql
-include COPYING
-include TODO
-include AUTHORS
-include CHANGES
-include Makefile
-include pytest.ini
-include tox.ini
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/PKG-INFO
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/PKG-INFO b/shell/ext-py/sqlparse-0.1.14/PKG-INFO
deleted file mode 100644
index 5aae628..0000000
--- a/shell/ext-py/sqlparse-0.1.14/PKG-INFO
+++ /dev/null
@@ -1,74 +0,0 @@
-Metadata-Version: 1.1
-Name: sqlparse
-Version: 0.1.14
-Summary: Non-validating SQL parser
-Home-page: https://github.com/andialbrecht/sqlparse
-Author: Andi Albrecht
-Author-email: albrecht.andi@gmail.com
-License: BSD
-Description:
- ``sqlparse`` is a non-validating SQL parser module.
- It provides support for parsing, splitting and formatting SQL statements.
-
- Visit the `project page <https://github.com/andialbrecht/sqlparse>`_ for
- additional information and documentation.
-
- **Example Usage**
-
-
- Splitting SQL statements::
-
- >>> import sqlparse
- >>> sqlparse.split('select * from foo; select * from bar;')
- [u'select * from foo; ', u'select * from bar;']
-
-
- Formatting statemtents::
-
- >>> sql = 'select * from foo where id in (select id from bar);'
- >>> print sqlparse.format(sql, reindent=True, keyword_case='upper')
- SELECT *
- FROM foo
- WHERE id IN
- (SELECT id
- FROM bar);
-
-
- Parsing::
-
- >>> sql = 'select * from someschema.mytable where id = 1'
- >>> res = sqlparse.parse(sql)
- >>> res
- (<Statement 'select...' at 0x9ad08ec>,)
- >>> stmt = res[0]
- >>> unicode(stmt) # converting it back to unicode
- u'select * from someschema.mytable where id = 1'
- >>> # This is how the internal representation looks like:
- >>> stmt.tokens
- (<DML 'select' at 0x9b63c34>,
- <Whitespace ' ' at 0x9b63e8c>,
- <Operator '*' at 0x9b63e64>,
- <Whitespace ' ' at 0x9b63c5c>,
- <Keyword 'from' at 0x9b63c84>,
- <Whitespace ' ' at 0x9b63cd4>,
- <Identifier 'somes...' at 0x9b5c62c>,
- <Whitespace ' ' at 0x9b63f04>,
- <Where 'where ...' at 0x9b5caac>)
-
-
-Platform: UNKNOWN
-Classifier: Development Status :: 4 - Beta
-Classifier: Intended Audience :: Developers
-Classifier: License :: OSI Approved :: BSD License
-Classifier: Operating System :: OS Independent
-Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 2
-Classifier: Programming Language :: Python :: 2.4
-Classifier: Programming Language :: Python :: 2.5
-Classifier: Programming Language :: Python :: 2.6
-Classifier: Programming Language :: Python :: 2.7
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.2
-Classifier: Programming Language :: Python :: 3.3
-Classifier: Topic :: Database
-Classifier: Topic :: Software Development
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/README.rst
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/README.rst b/shell/ext-py/sqlparse-0.1.14/README.rst
deleted file mode 100644
index dbe2fe4..0000000
--- a/shell/ext-py/sqlparse-0.1.14/README.rst
+++ /dev/null
@@ -1,56 +0,0 @@
-python-sqlparse - Parse SQL statements
-======================================
-
-sqlparse is a non-validating SQL parser module for Python.
-
-|buildstatus|_
-
-
-Install
--------
-
-Run::
-
- python setup.py install
-
-to install python-sqlparse on your system.
-
-python-sqlparse is compatible with Python 2 (>= 2.5) and Python 3 (>= 3.2).
-
-
-Run Tests
----------
-
-To run the test suite run::
-
- tox
-
-Note, you'll need tox installed, of course.
-
-
-Links
------
-
-Project Page
- https://github.com/andialbrecht/sqlparse
-
-Documentation
- http://readthedocs.org/docs/sqlparse/en/latest/
-
-Discussions
- http://groups.google.com/group/sqlparse
-
-Issues/Bugs
- https://github.com/andialbrecht/sqlparse/issues
-
-Online Demo
- http://sqlformat.org
-
-
-python-sqlparse is licensed under the BSD license.
-
-Parts of the code are based on pygments written by Georg Brandl and others.
-pygments-Homepage: http://pygments.org/
-
-.. |buildstatus| image:: https://secure.travis-ci.org/andialbrecht/sqlparse.png?branch=master
-.. _buildstatus: http://travis-ci.org/#!/andialbrecht/sqlparse
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/TODO
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/TODO b/shell/ext-py/sqlparse-0.1.14/TODO
deleted file mode 100644
index 166df20..0000000
--- a/shell/ext-py/sqlparse-0.1.14/TODO
+++ /dev/null
@@ -1,7 +0,0 @@
-* See
- https://groups.google.com/d/msg/sqlparse/huz9lKXt0Lc/11ybIKPJWbUJ
- for some interesting hints and suggestions.
-* Provide a function to replace tokens. See this thread: https://groups.google.com/d/msg/sqlparse/5xmBL2UKqX4/ZX9z_peve-AJ
-* Fix bugs on issue tracker.
-* Document filter stack and processing phases.
-* See KnownIssues http://code.google.com/p/python-sqlparse/wiki/KnownIssues
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/bin/sqlformat
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/bin/sqlformat b/shell/ext-py/sqlparse-0.1.14/bin/sqlformat
deleted file mode 100755
index fcee452..0000000
--- a/shell/ext-py/sqlparse-0.1.14/bin/sqlformat
+++ /dev/null
@@ -1,109 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright (C) 2008 Andi Albrecht, albrecht.andi@gmail.com
-#
-# This module is part of python-sqlparse and is released under
-# the BSD License: http://www.opensource.org/licenses/bsd-license.php.
-
-import optparse
-import os
-import sys
-
-import sqlparse
-from sqlparse.exceptions import SQLParseError
-
-
-_CASE_CHOICES = ['upper', 'lower', 'capitalize']
-
-
-parser = optparse.OptionParser(usage='%prog [OPTIONS] FILE, ...',
- version='%%prog %s' % sqlparse.__version__)
-parser.set_description(('Format FILE according to OPTIONS. Use "-" as FILE '
- 'to read from stdin.'))
-parser.add_option('-v', '--verbose', dest='verbose', action='store_true')
-parser.add_option('-o', '--outfile', dest='outfile', metavar='FILE',
- help='write output to FILE (defaults to stdout)')
-group = parser.add_option_group('Formatting Options')
-group.add_option('-k', '--keywords', metavar='CHOICE',
- dest='keyword_case', choices=_CASE_CHOICES,
- help=('change case of keywords, CHOICE is one of %s'
- % ', '.join('"%s"' % x for x in _CASE_CHOICES)))
-group.add_option('-i', '--identifiers', metavar='CHOICE',
- dest='identifier_case', choices=_CASE_CHOICES,
- help=('change case of identifiers, CHOICE is one of %s'
- % ', '.join('"%s"' % x for x in _CASE_CHOICES)))
-group.add_option('-l', '--language', metavar='LANG',
- dest='output_format', choices=['python', 'php'],
- help=('output a snippet in programming language LANG, '
- 'choices are "python", "php"'))
-group.add_option('--strip-comments', dest='strip_comments',
- action='store_true', default=False,
- help='remove comments')
-group.add_option('-r', '--reindent', dest='reindent',
- action='store_true', default=False,
- help='reindent statements')
-group.add_option('--indent_width', dest='indent_width', default=2,
- help='indentation width (defaults to 2 spaces)')
-
-_FORMATTING_GROUP = group
-
-
-def _error(msg, exit_=None):
- """Print msg and optionally exit with return code exit_."""
- sys.stderr.write('[ERROR] %s\n' % msg)
- if exit_ is not None:
- sys.exit(exit_)
-
-
-def _build_formatter_opts(options):
- """Convert command line options to dictionary."""
- d = {}
- for option in _FORMATTING_GROUP.option_list:
- d[option.dest] = getattr(options, option.dest)
- return d
-
-
-def main():
- options, args = parser.parse_args()
- if options.verbose:
- sys.stderr.write('Verbose mode\n')
-
- if len(args) != 1:
- _error('No input data.')
- parser.print_usage()
- sys.exit(1)
-
- if '-' in args: # read from stdin
- data = sys.stdin.read()
- else:
- try:
- data = '\n'.join(open(args[0]).readlines())
- except OSError:
- err = sys.exc_info()[1] # Python 2.5 compatibility
- _error('Failed to read %s: %s' % (args[0], err), exit_=1)
-
- if options.outfile:
- try:
- stream = open(options.outfile, 'w')
- except OSError:
- err = sys.exc_info()[1] # Python 2.5 compatibility
- _error('Failed to open %s: %s' % (options.outfile, err), exit_=1)
- else:
- stream = sys.stdout
-
- formatter_opts = _build_formatter_opts(options)
- try:
- formatter_opts = sqlparse.formatter.validate_options(formatter_opts)
- except SQLParseError:
- err = sys.exc_info()[1] # Python 2.5 compatibility
- _error('Invalid options: %s' % err, exit_=1)
-
- s = sqlparse.format(data, **formatter_opts)
- if sys.version_info < (3,):
- s = s.encode('utf-8', 'replace')
- stream.write(s)
- stream.flush()
-
-
-if __name__ == '__main__':
- main()
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/docs/source/analyzing.rst
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/docs/source/analyzing.rst b/shell/ext-py/sqlparse-0.1.14/docs/source/analyzing.rst
deleted file mode 100644
index 5af5350..0000000
--- a/shell/ext-py/sqlparse-0.1.14/docs/source/analyzing.rst
+++ /dev/null
@@ -1,64 +0,0 @@
-.. _analyze:
-
-Analyzing the Parsed Statement
-==============================
-
-When the :meth:`~sqlparse.parse` function is called the returned value
-is a tree-ish representation of the analyzed statements. The returned
-objects can be used by applications to retrieve further information about
-the parsed SQL.
-
-
-Base Classes
-------------
-
-All returned objects inherit from these base classes.
-The :class:`~sqlparse.sql.Token` class represents a single token and
-:class:`~sqlparse.sql.TokenList` class is a group of tokens.
-The latter provides methods for inspecting it's child tokens.
-
-.. autoclass:: sqlparse.sql.Token
- :members:
-
-.. autoclass:: sqlparse.sql.TokenList
- :members:
-
-
-SQL Representing Classes
-------------------------
-
-The following classes represent distinct parts of a SQL statement.
-
-.. autoclass:: sqlparse.sql.Statement
- :members:
-
-.. autoclass:: sqlparse.sql.Comment
- :members:
-
-.. autoclass:: sqlparse.sql.Identifier
- :members:
-
-.. autoclass:: sqlparse.sql.IdentifierList
- :members:
-
-.. autoclass:: sqlparse.sql.Where
- :members:
-
-.. autoclass:: sqlparse.sql.Case
- :members:
-
-.. autoclass:: sqlparse.sql.Parenthesis
- :members:
-
-.. autoclass:: sqlparse.sql.If
- :members:
-
-.. autoclass:: sqlparse.sql.For
- :members:
-
-.. autoclass:: sqlparse.sql.Assignment
- :members:
-
-.. autoclass:: sqlparse.sql.Comparison
- :members:
-
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/docs/source/api.rst
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/docs/source/api.rst b/shell/ext-py/sqlparse-0.1.14/docs/source/api.rst
deleted file mode 100644
index 518a428..0000000
--- a/shell/ext-py/sqlparse-0.1.14/docs/source/api.rst
+++ /dev/null
@@ -1,57 +0,0 @@
-:mod:`sqlparse` -- Parse SQL statements
-=======================================
-
-.. module:: sqlparse
- :synopsis: Parse SQL statements.
-
-The :mod:`sqlparse` module provides the following functions on module-level.
-
-.. autofunction:: sqlparse.split
-
-.. autofunction:: sqlparse.format
-
-.. autofunction:: sqlparse.parse
-
-In most cases there's no need to set the `encoding` parameter. If
-`encoding` is not set, sqlparse assumes that the given SQL statement
-is encoded either in utf-8 or latin-1.
-
-
-.. _formatting:
-
-Formatting of SQL Statements
-----------------------------
-
-The :meth:`~sqlparse.format` function accepts the following keyword arguments.
-
-``keyword_case``
- Changes how keywords are formatted. Allowed values are "upper", "lower"
- and "capitalize".
-
-``identifier_case``
- Changes how identifiers are formatted. Allowed values are "upper", "lower",
- and "capitalize".
-
-``strip_comments``
- If ``True`` comments are removed from the statements.
-
-``truncate_strings``
- If ``truncate_strings`` is a positive integer, string literals longer than
- the given value will be truncated.
-
-``truncate_char`` (default: "[...]")
- If long string literals are truncated (see above) this value will be append
- to the truncated string.
-
-``reindent``
- If ``True`` the indentations of the statements are changed.
-
-``indent_tabs``
- If ``True`` tabs instead of spaces are used for indentation.
-
-``indent_width``
- The width of the indentation, defaults to 2.
-
-``output_format``
- If given the output is additionally formatted to be used as a variable
- in a programming language. Allowed values are "python" and "php".
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/docs/source/changes.rst
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/docs/source/changes.rst b/shell/ext-py/sqlparse-0.1.14/docs/source/changes.rst
deleted file mode 100644
index 7c1b861..0000000
--- a/shell/ext-py/sqlparse-0.1.14/docs/source/changes.rst
+++ /dev/null
@@ -1,13 +0,0 @@
-.. _changes:
-
-Changes in python-sqlparse
-==========================
-
-Upcoming Deprecations
----------------------
-
-* ``sqlparse.SQLParseError`` is deprecated (version 0.1.5), use
- ``sqlparse.exceptions.SQLParseError`` instead.
-
-.. include:: ../../CHANGES
-
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/docs/source/conf.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/docs/source/conf.py b/shell/ext-py/sqlparse-0.1.14/docs/source/conf.py
deleted file mode 100644
index 5f7d34f..0000000
--- a/shell/ext-py/sqlparse-0.1.14/docs/source/conf.py
+++ /dev/null
@@ -1,200 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# python-sqlparse documentation build configuration file, created by
-# sphinx-quickstart on Thu Feb 26 08:19:28 2009.
-#
-# This file is execfile()d with the current directory set to its containing dir.
-#
-# Note that not all possible configuration values are present in this
-# autogenerated file.
-#
-# All configuration values have a default; values that are commented out
-# serve to show the default.
-
-import datetime
-import sys, os
-
-# If extensions (or modules to document with autodoc) are in another directory,
-# add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
-#sys.path.append(os.path.abspath('.'))
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../../'))
-
-import sqlparse
-
-# -- General configuration -----------------------------------------------------
-
-# Add any Sphinx extension module names here, as strings. They can be extensions
-# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo', 'sphinx.ext.coverage',
- 'sphinx.ext.autosummary']
-
-# Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
-
-# The suffix of source filenames.
-source_suffix = '.rst'
-
-# The encoding of source files.
-#source_encoding = 'utf-8'
-
-# The master toctree document.
-master_doc = 'index'
-
-# General information about the project.
-project = u'python-sqlparse'
-copyright = u'%s, Andi Albrecht' % datetime.date.today().strftime('%Y')
-
-# The version info for the project you're documenting, acts as replacement for
-# |version| and |release|, also used in various other places throughout the
-# built documents.
-#
-# The short X.Y version.
-version = sqlparse.__version__
-# The full version, including alpha/beta/rc tags.
-release = sqlparse.__version__
-
-# The language for content autogenerated by Sphinx. Refer to documentation
-# for a list of supported languages.
-#language = None
-
-# There are two options for replacing |today|: either, you set today to some
-# non-false value, then it is used:
-#today = ''
-# Else, today_fmt is used as the format for a strftime call.
-#today_fmt = '%B %d, %Y'
-
-# List of documents that shouldn't be included in the build.
-#unused_docs = []
-
-# List of directories, relative to source directory, that shouldn't be searched
-# for source files.
-exclude_trees = []
-
-# The reST default role (used for this markup: `text`) to use for all documents.
-#default_role = None
-
-# If true, '()' will be appended to :func: etc. cross-reference text.
-#add_function_parentheses = True
-
-# If true, the current module name will be prepended to all description
-# unit titles (such as .. function::).
-#add_module_names = True
-
-# If true, sectionauthor and moduleauthor directives will be shown in the
-# output. They are ignored by default.
-#show_authors = False
-
-# The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'tango'
-
-# A list of ignored prefixes for module index sorting.
-#modindex_common_prefix = []
-
-
-# -- Options for HTML output ---------------------------------------------------
-
-# The theme to use for HTML and HTML Help pages. Major themes that come with
-# Sphinx are currently 'default' and 'sphinxdoc'.
-#html_theme = 'agogo'
-
-# Theme options are theme-specific and customize the look and feel of a theme
-# further. For a list of options available for each theme, see the
-# documentation.
-#html_theme_options = {}
-
-# Add any paths that contain custom themes here, relative to this directory.
-#html_theme_path = [os.path.abspath('../')]
-
-# The name for this set of Sphinx documents. If None, it defaults to
-# "<project> v<release> documentation".
-#html_title = None
-
-# A shorter title for the navigation bar. Default is the same as html_title.
-#html_short_title = None
-
-# The name of an image file (relative to this directory) to place at the top
-# of the sidebar.
-#html_logo = None
-
-# The name of an image file (within the static path) to use as favicon of the
-# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
-# pixels large.
-#html_favicon = None
-
-# Add any paths that contain custom static files (such as style sheets) here,
-# relative to this directory. They are copied after the builtin static files,
-# so a file named "default.css" will overwrite the builtin "default.css".
-#html_static_path = ['_static']
-
-# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
-# using the given strftime format.
-#html_last_updated_fmt = '%b %d, %Y'
-
-# If true, SmartyPants will be used to convert quotes and dashes to
-# typographically correct entities.
-#html_use_smartypants = True
-
-# Custom sidebar templates, maps document names to template names.
-#html_sidebars = {}
-
-# Additional templates that should be rendered to pages, maps page names to
-# template names.
-#html_additional_pages = {}
-
-# If false, no module index is generated.
-#html_use_modindex = True
-
-# If false, no index is generated.
-#html_use_index = True
-
-# If true, the index is split into individual pages for each letter.
-#html_split_index = False
-
-# If true, links to the reST sources are added to the pages.
-#html_show_sourcelink = True
-
-# If true, an OpenSearch description file will be output, and all pages will
-# contain a <link> tag referring to it. The value of this option must be the
-# base URL from which the finished HTML is served.
-#html_use_opensearch = ''
-
-# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
-#html_file_suffix = ''
-
-# Output file base name for HTML help builder.
-htmlhelp_basename = 'python-sqlparsedoc'
-
-
-# -- Options for LaTeX output --------------------------------------------------
-
-# The paper size ('letter' or 'a4').
-#latex_paper_size = 'letter'
-
-# The font size ('10pt', '11pt' or '12pt').
-#latex_font_size = '10pt'
-
-# Grouping the document tree into LaTeX files. List of tuples
-# (source start file, target name, title, author, documentclass [howto/manual]).
-latex_documents = [
- ('index', 'python-sqlparse.tex', ur'python-sqlparse Documentation',
- ur'Andi Albrecht', 'manual'),
-]
-
-# The name of an image file (relative to this directory) to place at the top of
-# the title page.
-#latex_logo = None
-
-# For "manual" documents, if this is true, then toplevel headings are parts,
-# not chapters.
-#latex_use_parts = False
-
-# Additional stuff for the LaTeX preamble.
-#latex_preamble = ''
-
-# Documents to append as an appendix to all manuals.
-#latex_appendices = []
-
-# If false, no module index is generated.
-#latex_use_modindex = True
-todo_include_todos = True
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/docs/source/index.rst
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/docs/source/index.rst b/shell/ext-py/sqlparse-0.1.14/docs/source/index.rst
deleted file mode 100644
index 5eb76f7..0000000
--- a/shell/ext-py/sqlparse-0.1.14/docs/source/index.rst
+++ /dev/null
@@ -1,61 +0,0 @@
-.. python-sqlparse documentation master file, created by
- sphinx-quickstart on Thu Feb 26 08:19:28 2009.
- You can adapt this file completely to your liking, but it should at least
- contain the root `toctree` directive.
-
-python-sqlparse
-===============
-
-:mod:`sqlparse` is a non-validating SQL parser for Python.
-It provides support for parsing, splitting and formatting SQL statements.
-
-The module is compatible with Python 2 (>= 2.5) and Python 3 (>= 3.2)
-and released under the terms of the `New BSD license
-<http://www.opensource.org/licenses/bsd-license.php>`_.
-
-Visit the project page at https://github.com/andialbrecht/sqlparse for
-further information about this project.
-
-
-tl;dr
------
-
-.. code-block:: bash
-
- $ pip install sqlparse
- $ python
- >>> import sqlparse
- >>> print(sqlparse.format('select * from foo', reindent=True))
- select *
- from foo
- >>> parsed = sqlparse.parse('select * from foo')[0]
- >>> parsed.tokens
- [<DML 'select' at 0x7f22c5e15368>, <Whitespace ' ' at 0x7f22c5e153b0>, <Wildcard '*' … ]
- >>>
-
-
-Contents
---------
-
-.. toctree::
- :maxdepth: 2
-
- intro
- api
- analyzing
- ui
- changes
- indices
-
-
-Resources
----------
-
-Project page
- https://github.com/andialbrecht/sqlparse
-
-Bug tracker
- https://github.com/andialbrecht/sqlparse/issues
-
-Documentation
- http://sqlparse.readthedocs.org/
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/docs/source/indices.rst
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/docs/source/indices.rst b/shell/ext-py/sqlparse-0.1.14/docs/source/indices.rst
deleted file mode 100644
index f74c5d8..0000000
--- a/shell/ext-py/sqlparse-0.1.14/docs/source/indices.rst
+++ /dev/null
@@ -1,7 +0,0 @@
-Indices and tables
-==================
-
-* :ref:`genindex`
-* :ref:`modindex`
-* :ref:`search`
-
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/docs/source/intro.rst
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/docs/source/intro.rst b/shell/ext-py/sqlparse-0.1.14/docs/source/intro.rst
deleted file mode 100644
index 76d8fba..0000000
--- a/shell/ext-py/sqlparse-0.1.14/docs/source/intro.rst
+++ /dev/null
@@ -1,143 +0,0 @@
-Introduction
-============
-
-
-Download & Installation
------------------------
-
-The latest released version can be obtained from the `Python Package
-Index (PyPI) <http://pypi.python.org/pypi/sqlparse/>`_. To extract the
-install the module system-wide run
-
-.. code-block:: bash
-
- $ tar cvfz python-sqlparse-VERSION.tar.gz
- $ cd python-sqlparse/
- $ sudo python setup.py install
-
-Alternatively you can install :mod:`sqlparse` using :command:`pip`:
-
-.. code-block:: bash
-
- $ pip install sqlparse
-
-
-Getting Started
----------------
-
-The :mod:`sqlparse` module provides three simple functions on module level
-to achieve some common tasks when working with SQL statements.
-This section shows some simple usage examples of these functions.
-
-Let's get started with splitting a string containing one or more SQL
-statements into a list of single statements using :meth:`~sqlparse.split`:
-
-.. code-block:: python
-
- >>> import sqlparse
- >>> sql = 'select * from foo; select * from bar;'
- >>> sqlparse.split(sql)
- [u'select * from foo; ', u'select * from bar;']
-
-The end of a statement is identified by the occurrence of a semicolon.
-Semicolons within certain SQL constructs like ``BEGIN ... END`` blocks
-are handled correctly by the splitting mechanism.
-
-SQL statements can be beautified by using the :meth:`~sqlarse.format` function.
-
-.. code-block:: python
-
- >>> sql = 'select * from foo where id in (select id from bar);'
- >>> print sqlparse.format(sql, reindent=True, keyword_case='upper')
- SELECT *
- FROM foo
- WHERE id IN
- (SELECT id
- FROM bar);
-
-In this case all keywords in the given SQL are uppercased and the
-indentation is changed to make it more readable. Read :ref:`formatting` for
-a full reference of supported options given as keyword arguments
-to that function.
-
-Before proceeding with a closer look at the internal representation of
-SQL statements, you should be aware that this SQL parser is intentionally
-non-validating. It assumes that the given input is at least some kind
-of SQL and then it tries to analyze as much as possible without making
-too much assumptions about the concrete dialect or the actual statement.
-At least it's up to the user of this API to interpret the results right.
-
-When using the :meth:`~sqlparse.parse` function a tuple of
-:class:`~sqlparse.sql.Statement` instances is returned:
-
-.. code-block:: python
-
- >>> sql = 'select * from "someschema"."mytable" where id = 1'
- >>> parsed = sqlparse.parse(sql)
- >>> parsed
- (<Statement 'select...' at 0x9ad08ec>,)
-
-Each item of the tuple is a single statement as identified by the above
-mentioned :meth:`~sqlparse.split` function. So let's grab the only element
-from that list and have a look at the ``tokens`` attribute.
-Sub-tokens are stored in this attribute.
-
-.. code-block:: python
-
- >>> stmt = parsed[0] # grab the Statement object
- >>> stmt.tokens
- (<DML 'select' at 0x9b63c34>,
- <Whitespace ' ' at 0x9b63e8c>,
- <Operator '*' at 0x9b63e64>,
- <Whitespace ' ' at 0x9b63c5c>,
- <Keyword 'from' at 0x9b63c84>,
- <Whitespace ' ' at 0x9b63cd4>,
- <Identifier '"somes...' at 0x9b5c62c>,
- <Whitespace ' ' at 0x9b63f04>,
- <Where 'where ...' at 0x9b5caac>)
-
-Each object can be converted back to a string at any time:
-
-.. code-block:: python
-
- >>> unicode(stmt) # str(stmt) for Python 3
- u'select * from "someschema"."mytable" where id = 1'
- >>> unicode(stmt.tokens[-1]) # or just the WHERE part
- u'where id = 1'
-
-Details of the returned objects are described in :ref:`analyze`.
-
-
-Development & Contributing
---------------------------
-
-To check out the latest sources of this module run
-
-.. code-block:: bash
-
- $ git clone git://github.com/andialbrecht/sqlparse.git
-
-
-to check out the latest sources from the repository.
-
-:mod:`sqlparse` is currently tested under Python 2.5, 2.6, 2.7, 3.2 and
-pypy. Tests are automatically run on each commit and for each pull
-request on Travis: https://travis-ci.org/andialbrecht/sqlparse
-
-Make sure to run the test suite before sending a pull request by running
-
-.. code-block:: bash
-
- $ tox
-
-It's ok, if :command:`tox` doesn't find all interpreters listed
-above. Ideally a Python 2 and a Python 3 version should be tested
-locally.
-
-Please file bug reports and feature requests on the project site at
-https://github.com/andialbrecht/sqlparse/issues/new or if you have
-code to contribute upload it to http://codereview.appspot.com and
-add albrecht.andi@googlemail.com as reviewer.
-
-For more information about the review tool and how to use it visit
-it's project page: http://code.google.com/p/rietveld.
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/docs/source/ui.rst
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/docs/source/ui.rst b/shell/ext-py/sqlparse-0.1.14/docs/source/ui.rst
deleted file mode 100644
index 264916e..0000000
--- a/shell/ext-py/sqlparse-0.1.14/docs/source/ui.rst
+++ /dev/null
@@ -1,15 +0,0 @@
-User Interfaces
-===============
-
-``sqlformat``
- The ``sqlformat`` command line script ist distributed with the module.
- Run :command:`sqlformat --help` to list available options and for usage
- hints.
-
-``sqlformat.appspot.com``
- An example `Google App Engine <http://code.google.com/appengine/>`_
- application that exposes the formatting features using a web front-end.
- See http://sqlformat.appspot.com for details.
- The source for this application is available from a source code check out
- of the :mod:`sqlparse` module (see :file:`extras/appengine`).
-
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/docs/sqlformat.1
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/docs/sqlformat.1 b/shell/ext-py/sqlparse-0.1.14/docs/sqlformat.1
deleted file mode 100644
index a42ea60..0000000
--- a/shell/ext-py/sqlparse-0.1.14/docs/sqlformat.1
+++ /dev/null
@@ -1,65 +0,0 @@
-.\" Based on template /usr/share/man-db/examples/manpage.example provided by
-.\" Tom Christiansen <tc...@jhereg.perl.com>.
-.TH SQLFORMAT "1" "December 2010" "python-sqlparse version: 0.1.2" "User Commands"
-.SH NAME
-sqlformat \- reformat SQL
-.SH SYNOPSIS
-.PP
-.B sqlformat
-[
-.I "OPTION"
-] ... [
-.I "FILE"
-] ...
-.SH DESCRIPTION
-.\" Putting a newline after each sentence can generate better output.
-The `sqlformat' command-line tool can be used to reformat SQL file according to
-specified options or prepare a snippet in in some programming language (only
-Python and PHP currently supported).
-Use "-" for
-.I FILE
-to read from stdin.
-.SH OPTIONS
-.TP
-\fB\-i\fR \fICHOICE\fR|\fB\-\-identifiers\fR=\fIFORMAT\fR
-Change case of identifiers.
-.I FORMAT
-is one of "upper", "lower", "capitalize".
-.TP
-\fB\-k\fR \fICHOICE\fR|\fB\-\-keywords\fR=\fIFORMAT\fR
-Change case of keywords.
-.I FORMAT
-is one of "upper", "lower", "capitalize".
-.TP
-\fB\-l\fR \fICHOICE\fR|\fB\-\-language\fR=\fILANG\fR
-Output a snippet in programming language LANG.
-.I LANG
-can be "python", "php".
-.TP
-\fB\-o\fR \fIFILE\fR|\fB\-\-outfile\fR=\fIFILE\fR
-Write output to
-.I FILE
-(defaults to stdout).
-.TP
-.BR \-r | \-\-reindent
-Reindent statements.
-.TP
-\fB\-\-indent_width\fR=\fIINDENT_WIDTH\fR
-Set indent width to
-.IR INDENT_WIDTH .
-Default is 2 spaces.
-.TP
-\fB\-\-strip\-comments
-Remove comments.
-.TP
-.BR \-h | \-\-help
-Print a short help message and exit.
-All subsequent options are ignored.
-.TP
-.BR --verbose
-Verbose output.
-.TP
-.BR \-\-version
-Print program's version number and exit.
-.SH AUTHORS
-This man page was written by Andriy Senkovych <jo...@itblog.org.ua>
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/pytest.ini
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/pytest.ini b/shell/ext-py/sqlparse-0.1.14/pytest.ini
deleted file mode 100644
index a2cbd90..0000000
--- a/shell/ext-py/sqlparse-0.1.14/pytest.ini
+++ /dev/null
@@ -1,6 +0,0 @@
-[pytest]
-pep8ignore =
- extras/* ALL
- examples/* ALL
- docs/* ALL
- * E125 E127
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/setup.cfg
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/setup.cfg b/shell/ext-py/sqlparse-0.1.14/setup.cfg
deleted file mode 100644
index 861a9f5..0000000
--- a/shell/ext-py/sqlparse-0.1.14/setup.cfg
+++ /dev/null
@@ -1,5 +0,0 @@
-[egg_info]
-tag_build =
-tag_date = 0
-tag_svn_revision = 0
-
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/setup.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/setup.py b/shell/ext-py/sqlparse-0.1.14/setup.py
deleted file mode 100644
index 2c6dce8..0000000
--- a/shell/ext-py/sqlparse-0.1.14/setup.py
+++ /dev/null
@@ -1,123 +0,0 @@
-# Copyright (C) 2008 Andi Albrecht, albrecht.andi@gmail.com
-#
-# This setup script is part of python-sqlparse and is released under
-# the BSD License: http://www.opensource.org/licenses/bsd-license.php.
-
-import re
-import sys
-
-try:
- from setuptools import setup, find_packages
- packages = find_packages(exclude=('tests',))
-except ImportError:
- if sys.version_info[0] == 3:
- raise RuntimeError('distribute is required to install this package.')
- from distutils.core import setup
- packages = ['sqlparse', 'sqlparse.engine']
-
-
-def get_version():
- """parse __init__.py for version number instead of importing the file
-
- see http://stackoverflow.com/questions/458550/standard-way-to-embed-version-into-python-package
- """
- VERSIONFILE='sqlparse/__init__.py'
- verstrline = open(VERSIONFILE, "rt").read()
- VSRE = r'^__version__ = [\'"]([^\'"]*)[\'"]'
- mo = re.search(VSRE, verstrline, re.M)
- if mo:
- return mo.group(1)
- else:
- raise RuntimeError('Unable to find version string in %s.'
- % (VERSIONFILE,))
-
-
-LONG_DESCRIPTION = """
-``sqlparse`` is a non-validating SQL parser module.
-It provides support for parsing, splitting and formatting SQL statements.
-
-Visit the `project page <https://github.com/andialbrecht/sqlparse>`_ for
-additional information and documentation.
-
-**Example Usage**
-
-
-Splitting SQL statements::
-
- >>> import sqlparse
- >>> sqlparse.split('select * from foo; select * from bar;')
- [u'select * from foo; ', u'select * from bar;']
-
-
-Formatting statemtents::
-
- >>> sql = 'select * from foo where id in (select id from bar);'
- >>> print sqlparse.format(sql, reindent=True, keyword_case='upper')
- SELECT *
- FROM foo
- WHERE id IN
- (SELECT id
- FROM bar);
-
-
-Parsing::
-
- >>> sql = 'select * from someschema.mytable where id = 1'
- >>> res = sqlparse.parse(sql)
- >>> res
- (<Statement 'select...' at 0x9ad08ec>,)
- >>> stmt = res[0]
- >>> unicode(stmt) # converting it back to unicode
- u'select * from someschema.mytable where id = 1'
- >>> # This is how the internal representation looks like:
- >>> stmt.tokens
- (<DML 'select' at 0x9b63c34>,
- <Whitespace ' ' at 0x9b63e8c>,
- <Operator '*' at 0x9b63e64>,
- <Whitespace ' ' at 0x9b63c5c>,
- <Keyword 'from' at 0x9b63c84>,
- <Whitespace ' ' at 0x9b63cd4>,
- <Identifier 'somes...' at 0x9b5c62c>,
- <Whitespace ' ' at 0x9b63f04>,
- <Where 'where ...' at 0x9b5caac>)
-
-"""
-
-VERSION = get_version()
-
-
-kwargs = {}
-if sys.version_info[0] == 3:
- kwargs['use_2to3'] = True
-
-
-setup(
- name='sqlparse',
- version=VERSION,
- packages=packages,
- description='Non-validating SQL parser',
- author='Andi Albrecht',
- author_email='albrecht.andi@gmail.com',
- long_description=LONG_DESCRIPTION,
- license='BSD',
- url='https://github.com/andialbrecht/sqlparse',
- classifiers=[
- 'Development Status :: 4 - Beta',
- 'Intended Audience :: Developers',
- 'License :: OSI Approved :: BSD License',
- 'Operating System :: OS Independent',
- 'Programming Language :: Python',
- 'Programming Language :: Python :: 2',
- 'Programming Language :: Python :: 2.4',
- 'Programming Language :: Python :: 2.5',
- 'Programming Language :: Python :: 2.6',
- 'Programming Language :: Python :: 2.7',
- 'Programming Language :: Python :: 3',
- 'Programming Language :: Python :: 3.2',
- 'Programming Language :: Python :: 3.3',
- 'Topic :: Database',
- 'Topic :: Software Development'
- ],
- scripts=['bin/sqlformat'],
- **kwargs
-)
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/sqlparse/__init__.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/sqlparse/__init__.py b/shell/ext-py/sqlparse-0.1.14/sqlparse/__init__.py
deleted file mode 100644
index c956f66..0000000
--- a/shell/ext-py/sqlparse-0.1.14/sqlparse/__init__.py
+++ /dev/null
@@ -1,78 +0,0 @@
-# Copyright (C) 2008 Andi Albrecht, albrecht.andi@gmail.com
-#
-# This module is part of python-sqlparse and is released under
-# the BSD License: http://www.opensource.org/licenses/bsd-license.php.
-
-"""Parse SQL statements."""
-
-
-__version__ = '0.1.14'
-
-
-# Setup namespace
-from sqlparse import engine
-from sqlparse import filters
-from sqlparse import formatter
-
-# Deprecated in 0.1.5. Will be removed in 0.2.0
-from sqlparse.exceptions import SQLParseError
-
-
-def parse(sql, encoding=None):
- """Parse sql and return a list of statements.
-
- :param sql: A string containting one or more SQL statements.
- :param encoding: The encoding of the statement (optional).
- :returns: A tuple of :class:`~sqlparse.sql.Statement` instances.
- """
- return tuple(parsestream(sql, encoding))
-
-
-def parsestream(stream, encoding=None):
- """Parses sql statements from file-like object.
-
- :param stream: A file-like object.
- :param encoding: The encoding of the stream contents (optional).
- :returns: A generator of :class:`~sqlparse.sql.Statement` instances.
- """
- stack = engine.FilterStack()
- stack.full_analyze()
- return stack.run(stream, encoding)
-
-
-def format(sql, **options):
- """Format *sql* according to *options*.
-
- Available options are documented in :ref:`formatting`.
-
- In addition to the formatting options this function accepts the
- keyword "encoding" which determines the encoding of the statement.
-
- :returns: The formatted SQL statement as string.
- """
- encoding = options.pop('encoding', None)
- stack = engine.FilterStack()
- options = formatter.validate_options(options)
- stack = formatter.build_filter_stack(stack, options)
- stack.postprocess.append(filters.SerializerUnicode())
- return ''.join(stack.run(sql, encoding))
-
-
-def split(sql, encoding=None):
- """Split *sql* into single statements.
-
- :param sql: A string containting one or more SQL statements.
- :param encoding: The encoding of the statement (optional).
- :returns: A list of strings.
- """
- stack = engine.FilterStack()
- stack.split_statements = True
- return [unicode(stmt).strip() for stmt in stack.run(sql, encoding)]
-
-
-from sqlparse.engine.filter import StatementFilter
-
-
-def split2(stream):
- splitter = StatementFilter()
- return list(splitter.process(None, stream))
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/sqlparse/engine/__init__.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/sqlparse/engine/__init__.py b/shell/ext-py/sqlparse-0.1.14/sqlparse/engine/__init__.py
deleted file mode 100644
index 62c82b8..0000000
--- a/shell/ext-py/sqlparse-0.1.14/sqlparse/engine/__init__.py
+++ /dev/null
@@ -1,80 +0,0 @@
-# Copyright (C) 2008 Andi Albrecht, albrecht.andi@gmail.com
-#
-# This module is part of python-sqlparse and is released under
-# the BSD License: http://www.opensource.org/licenses/bsd-license.php.
-
-"""filter"""
-
-from sqlparse import lexer
-from sqlparse.engine import grouping
-from sqlparse.engine.filter import StatementFilter
-
-# XXX remove this when cleanup is complete
-Filter = object
-
-
-class FilterStack(object):
-
- def __init__(self):
- self.preprocess = []
- self.stmtprocess = []
- self.postprocess = []
- self.split_statements = False
- self._grouping = False
-
- def _flatten(self, stream):
- for token in stream:
- if token.is_group():
- for t in self._flatten(token.tokens):
- yield t
- else:
- yield token
-
- def enable_grouping(self):
- self._grouping = True
-
- def full_analyze(self):
- self.enable_grouping()
-
- def run(self, sql, encoding=None):
- stream = lexer.tokenize(sql, encoding)
- # Process token stream
- if self.preprocess:
- for filter_ in self.preprocess:
- stream = filter_.process(self, stream)
-
- if (self.stmtprocess or self.postprocess or self.split_statements
- or self._grouping):
- splitter = StatementFilter()
- stream = splitter.process(self, stream)
-
- if self._grouping:
-
- def _group(stream):
- for stmt in stream:
- grouping.group(stmt)
- yield stmt
- stream = _group(stream)
-
- if self.stmtprocess:
-
- def _run1(stream):
- ret = []
- for stmt in stream:
- for filter_ in self.stmtprocess:
- filter_.process(self, stmt)
- ret.append(stmt)
- return ret
- stream = _run1(stream)
-
- if self.postprocess:
-
- def _run2(stream):
- for stmt in stream:
- stmt.tokens = list(self._flatten(stmt.tokens))
- for filter_ in self.postprocess:
- stmt = filter_.process(self, stmt)
- yield stmt
- stream = _run2(stream)
-
- return stream
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/sqlparse/engine/filter.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/sqlparse/engine/filter.py b/shell/ext-py/sqlparse-0.1.14/sqlparse/engine/filter.py
deleted file mode 100644
index 9af2f99..0000000
--- a/shell/ext-py/sqlparse-0.1.14/sqlparse/engine/filter.py
+++ /dev/null
@@ -1,109 +0,0 @@
-# -*- coding: utf-8 -*-
-
-from sqlparse.sql import Statement, Token
-from sqlparse import tokens as T
-
-
-class StatementFilter:
- "Filter that split stream at individual statements"
-
- def __init__(self):
- self._in_declare = False
- self._in_dbldollar = False
- self._is_create = False
- self._begin_depth = 0
-
- def _reset(self):
- "Set the filter attributes to its default values"
- self._in_declare = False
- self._in_dbldollar = False
- self._is_create = False
- self._begin_depth = 0
-
- def _change_splitlevel(self, ttype, value):
- "Get the new split level (increase, decrease or remain equal)"
- # PostgreSQL
- if (ttype == T.Name.Builtin
- and value.startswith('$') and value.endswith('$')):
- if self._in_dbldollar:
- self._in_dbldollar = False
- return -1
- else:
- self._in_dbldollar = True
- return 1
- elif self._in_dbldollar:
- return 0
-
- # ANSI
- if ttype not in T.Keyword:
- return 0
-
- unified = value.upper()
-
- if unified == 'DECLARE' and self._is_create:
- self._in_declare = True
- return 1
-
- if unified == 'BEGIN':
- self._begin_depth += 1
- if self._in_declare or self._is_create:
- # FIXME(andi): This makes no sense.
- return 1
- return 0
-
- if unified == 'END':
- # Should this respect a preceeding BEGIN?
- # In CASE ... WHEN ... END this results in a split level -1.
- self._begin_depth = max(0, self._begin_depth - 1)
- return -1
-
- if ttype is T.Keyword.DDL and unified.startswith('CREATE'):
- self._is_create = True
- return 0
-
- if (unified in ('IF', 'FOR')
- and self._is_create and self._begin_depth > 0):
- return 1
-
- # Default
- return 0
-
- def process(self, stack, stream):
- "Process the stream"
- consume_ws = False
- splitlevel = 0
- stmt = None
- stmt_tokens = []
-
- # Run over all stream tokens
- for ttype, value in stream:
- # Yield token if we finished a statement and there's no whitespaces
- if consume_ws and ttype not in (T.Whitespace, T.Comment.Single):
- stmt.tokens = stmt_tokens
- yield stmt
-
- # Reset filter and prepare to process next statement
- self._reset()
- consume_ws = False
- splitlevel = 0
- stmt = None
-
- # Create a new statement if we are not currently in one of them
- if stmt is None:
- stmt = Statement()
- stmt_tokens = []
-
- # Change current split level (increase, decrease or remain equal)
- splitlevel += self._change_splitlevel(ttype, value)
-
- # Append the token to the current statement
- stmt_tokens.append(Token(ttype, value))
-
- # Check if we get the end of a statement
- if splitlevel <= 0 and ttype is T.Punctuation and value == ';':
- consume_ws = True
-
- # Yield pending statement (if any)
- if stmt is not None:
- stmt.tokens = stmt_tokens
- yield stmt
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/sqlparse/engine/grouping.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/sqlparse/engine/grouping.py b/shell/ext-py/sqlparse-0.1.14/sqlparse/engine/grouping.py
deleted file mode 100644
index ee534e3..0000000
--- a/shell/ext-py/sqlparse-0.1.14/sqlparse/engine/grouping.py
+++ /dev/null
@@ -1,390 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import itertools
-
-from sqlparse import sql
-from sqlparse import tokens as T
-
-try:
- next
-except NameError: # Python < 2.6
- next = lambda i: i.next()
-
-
-def _group_left_right(tlist, ttype, value, cls,
- check_right=lambda t: True,
- check_left=lambda t: True,
- include_semicolon=False):
- [_group_left_right(sgroup, ttype, value, cls, check_right, check_left,
- include_semicolon) for sgroup in tlist.get_sublists()
- if not isinstance(sgroup, cls)]
- idx = 0
- token = tlist.token_next_match(idx, ttype, value)
- while token:
- right = tlist.token_next(tlist.token_index(token))
- left = tlist.token_prev(tlist.token_index(token))
- if right is None or not check_right(right):
- token = tlist.token_next_match(tlist.token_index(token) + 1,
- ttype, value)
- elif left is None or not check_left(left):
- token = tlist.token_next_match(tlist.token_index(token) + 1,
- ttype, value)
- else:
- if include_semicolon:
- sright = tlist.token_next_match(tlist.token_index(right),
- T.Punctuation, ';')
- if sright is not None:
- # only overwrite "right" if a semicolon is actually
- # present.
- right = sright
- tokens = tlist.tokens_between(left, right)[1:]
- if not isinstance(left, cls):
- new = cls([left])
- new_idx = tlist.token_index(left)
- tlist.tokens.remove(left)
- tlist.tokens.insert(new_idx, new)
- left = new
- left.tokens.extend(tokens)
- for t in tokens:
- tlist.tokens.remove(t)
- token = tlist.token_next_match(tlist.token_index(left) + 1,
- ttype, value)
-
-
-def _group_matching(tlist, start_ttype, start_value, end_ttype, end_value,
- cls, include_semicolon=False, recurse=False):
- def _find_matching(i, tl, stt, sva, ett, eva):
- depth = 1
- for n in xrange(i, len(tl.tokens)):
- t = tl.tokens[n]
- if t.match(stt, sva):
- depth += 1
- elif t.match(ett, eva):
- depth -= 1
- if depth == 1:
- return t
- return None
- [_group_matching(sgroup, start_ttype, start_value, end_ttype, end_value,
- cls, include_semicolon) for sgroup in tlist.get_sublists()
- if recurse]
- if isinstance(tlist, cls):
- idx = 1
- else:
- idx = 0
- token = tlist.token_next_match(idx, start_ttype, start_value)
- while token:
- tidx = tlist.token_index(token)
- end = _find_matching(tidx, tlist, start_ttype, start_value,
- end_ttype, end_value)
- if end is None:
- idx = tidx + 1
- else:
- if include_semicolon:
- next_ = tlist.token_next(tlist.token_index(end))
- if next_ and next_.match(T.Punctuation, ';'):
- end = next_
- group = tlist.group_tokens(cls, tlist.tokens_between(token, end))
- _group_matching(group, start_ttype, start_value,
- end_ttype, end_value, cls, include_semicolon)
- idx = tlist.token_index(group) + 1
- token = tlist.token_next_match(idx, start_ttype, start_value)
-
-
-def group_if(tlist):
- _group_matching(tlist, T.Keyword, 'IF', T.Keyword, 'END IF', sql.If, True)
-
-
-def group_for(tlist):
- _group_matching(tlist, T.Keyword, 'FOR', T.Keyword, 'END LOOP',
- sql.For, True)
-
-
-def group_foreach(tlist):
- _group_matching(tlist, T.Keyword, 'FOREACH', T.Keyword, 'END LOOP',
- sql.For, True)
-
-
-def group_begin(tlist):
- _group_matching(tlist, T.Keyword, 'BEGIN', T.Keyword, 'END',
- sql.Begin, True)
-
-
-def group_as(tlist):
-
- def _right_valid(token):
- # Currently limited to DML/DDL. Maybe additional more non SQL reserved
- # keywords should appear here (see issue8).
- return not token.ttype in (T.DML, T.DDL)
-
- def _left_valid(token):
- if token.ttype is T.Keyword and token.value in ('NULL',):
- return True
- return token.ttype is not T.Keyword
-
- _group_left_right(tlist, T.Keyword, 'AS', sql.Identifier,
- check_right=_right_valid,
- check_left=_left_valid)
-
-
-def group_assignment(tlist):
- _group_left_right(tlist, T.Assignment, ':=', sql.Assignment,
- include_semicolon=True)
-
-
-def group_comparison(tlist):
-
- def _parts_valid(token):
- return (token.ttype in (T.String.Symbol, T.String.Single,
- T.Name, T.Number, T.Number.Float,
- T.Number.Integer, T.Literal,
- T.Literal.Number.Integer, T.Name.Placeholder)
- or isinstance(token, (sql.Identifier, sql.Parenthesis))
- or (token.ttype is T.Keyword
- and token.value.upper() in ['NULL', ]))
- _group_left_right(tlist, T.Operator.Comparison, None, sql.Comparison,
- check_left=_parts_valid, check_right=_parts_valid)
-
-
-def group_case(tlist):
- _group_matching(tlist, T.Keyword, 'CASE', T.Keyword, 'END', sql.Case,
- include_semicolon=True, recurse=True)
-
-
-def group_identifier(tlist):
- def _consume_cycle(tl, i):
- # TODO: Usage of Wildcard token is ambivalent here.
- x = itertools.cycle((
- lambda y: (y.match(T.Punctuation, '.')
- or y.ttype is T.Operator
- or y.ttype is T.Wildcard),
- lambda y: (y.ttype in (T.String.Symbol,
- T.Name,
- T.Wildcard,
- T.Literal.String.Single,
- T.Literal.Number.Integer,
- T.Literal.Number.Float)
- or isinstance(y, (sql.Parenthesis, sql.Function)))))
- for t in tl.tokens[i:]:
- # Don't take whitespaces into account.
- if t.ttype is T.Whitespace:
- yield t
- continue
- if next(x)(t):
- yield t
- else:
- raise StopIteration
-
- def _next_token(tl, i):
- # chooses the next token. if two tokens are found then the
- # first is returned.
- t1 = tl.token_next_by_type(
- i, (T.String.Symbol, T.Name, T.Literal.Number.Integer,
- T.Literal.Number.Float))
- t2 = tl.token_next_by_instance(i, (sql.Function, sql.Parenthesis))
- if t1 and t2:
- i1 = tl.token_index(t1)
- i2 = tl.token_index(t2)
- if i1 > i2:
- return t2
- else:
- return t1
- elif t1:
- return t1
- else:
- return t2
-
- # bottom up approach: group subgroups first
- [group_identifier(sgroup) for sgroup in tlist.get_sublists()
- if not isinstance(sgroup, sql.Identifier)]
-
- # real processing
- idx = 0
- token = _next_token(tlist, idx)
- while token:
- identifier_tokens = [token] + list(
- _consume_cycle(tlist,
- tlist.token_index(token) + 1))
- # remove trailing whitespace
- if identifier_tokens and identifier_tokens[-1].ttype is T.Whitespace:
- identifier_tokens = identifier_tokens[:-1]
- if not (len(identifier_tokens) == 1
- and (isinstance(identifier_tokens[0], (sql.Function, sql.Parenthesis))
- or identifier_tokens[0].ttype in (T.Literal.Number.Integer,
- T.Literal.Number.Float))):
- group = tlist.group_tokens(sql.Identifier, identifier_tokens)
- idx = tlist.token_index(group) + 1
- else:
- idx += 1
- token = _next_token(tlist, idx)
-
-
-def group_identifier_list(tlist):
- [group_identifier_list(sgroup) for sgroup in tlist.get_sublists()
- if not isinstance(sgroup, sql.IdentifierList)]
- idx = 0
- # Allowed list items
- fend1_funcs = [lambda t: isinstance(t, (sql.Identifier, sql.Function,
- sql.Case)),
- lambda t: t.is_whitespace(),
- lambda t: t.ttype == T.Name,
- lambda t: t.ttype == T.Wildcard,
- lambda t: t.match(T.Keyword, 'null'),
- lambda t: t.match(T.Keyword, 'role'),
- lambda t: t.ttype == T.Number.Integer,
- lambda t: t.ttype == T.String.Single,
- lambda t: t.ttype == T.Name.Placeholder,
- lambda t: t.ttype == T.Keyword,
- lambda t: isinstance(t, sql.Comparison),
- lambda t: isinstance(t, sql.Comment),
- lambda t: t.ttype == T.Comment.Multiline,
- ]
- tcomma = tlist.token_next_match(idx, T.Punctuation, ',')
- start = None
- while tcomma is not None:
- before = tlist.token_prev(tcomma)
- after = tlist.token_next(tcomma)
- # Check if the tokens around tcomma belong to a list
- bpassed = apassed = False
- for func in fend1_funcs:
- if before is not None and func(before):
- bpassed = True
- if after is not None and func(after):
- apassed = True
- if not bpassed or not apassed:
- # Something's wrong here, skip ahead to next ","
- start = None
- tcomma = tlist.token_next_match(tlist.token_index(tcomma) + 1,
- T.Punctuation, ',')
- else:
- if start is None:
- start = before
- next_ = tlist.token_next(after)
- if next_ is None or not next_.match(T.Punctuation, ','):
- # Reached the end of the list
- tokens = tlist.tokens_between(start, after)
- group = tlist.group_tokens(sql.IdentifierList, tokens)
- start = None
- tcomma = tlist.token_next_match(tlist.token_index(group) + 1,
- T.Punctuation, ',')
- else:
- tcomma = next_
-
-
-def group_parenthesis(tlist):
- _group_matching(tlist, T.Punctuation, '(', T.Punctuation, ')',
- sql.Parenthesis)
-
-
-def group_comments(tlist):
- [group_comments(sgroup) for sgroup in tlist.get_sublists()
- if not isinstance(sgroup, sql.Comment)]
- idx = 0
- token = tlist.token_next_by_type(idx, T.Comment)
- while token:
- tidx = tlist.token_index(token)
- end = tlist.token_not_matching(tidx + 1,
- [lambda t: t.ttype in T.Comment,
- lambda t: t.is_whitespace()])
- if end is None:
- idx = tidx + 1
- else:
- eidx = tlist.token_index(end)
- grp_tokens = tlist.tokens_between(token,
- tlist.token_prev(eidx, False))
- group = tlist.group_tokens(sql.Comment, grp_tokens)
- idx = tlist.token_index(group)
- token = tlist.token_next_by_type(idx, T.Comment)
-
-
-def group_where(tlist):
- [group_where(sgroup) for sgroup in tlist.get_sublists()
- if not isinstance(sgroup, sql.Where)]
- idx = 0
- token = tlist.token_next_match(idx, T.Keyword, 'WHERE')
- stopwords = ('ORDER', 'GROUP', 'LIMIT', 'UNION', 'EXCEPT')
- while token:
- tidx = tlist.token_index(token)
- end = tlist.token_next_match(tidx + 1, T.Keyword, stopwords)
- if end is None:
- end = tlist._groupable_tokens[-1]
- else:
- end = tlist.tokens[tlist.token_index(end) - 1]
- group = tlist.group_tokens(sql.Where,
- tlist.tokens_between(token, end),
- ignore_ws=True)
- idx = tlist.token_index(group)
- token = tlist.token_next_match(idx, T.Keyword, 'WHERE')
-
-
-def group_aliased(tlist):
- clss = (sql.Identifier, sql.Function, sql.Case)
- [group_aliased(sgroup) for sgroup in tlist.get_sublists()
- if not isinstance(sgroup, clss)]
- idx = 0
- token = tlist.token_next_by_instance(idx, clss)
- while token:
- next_ = tlist.token_next(tlist.token_index(token))
- if next_ is not None and isinstance(next_, clss):
- if not next_.value.upper().startswith('VARCHAR'):
- grp = tlist.tokens_between(token, next_)[1:]
- token.tokens.extend(grp)
- for t in grp:
- tlist.tokens.remove(t)
- idx = tlist.token_index(token) + 1
- token = tlist.token_next_by_instance(idx, clss)
-
-
-def group_typecasts(tlist):
- _group_left_right(tlist, T.Punctuation, '::', sql.Identifier)
-
-
-def group_functions(tlist):
- [group_functions(sgroup) for sgroup in tlist.get_sublists()
- if not isinstance(sgroup, sql.Function)]
- idx = 0
- token = tlist.token_next_by_type(idx, T.Name)
- while token:
- next_ = tlist.token_next(token)
- if not isinstance(next_, sql.Parenthesis):
- idx = tlist.token_index(token) + 1
- else:
- func = tlist.group_tokens(sql.Function,
- tlist.tokens_between(token, next_))
- idx = tlist.token_index(func) + 1
- token = tlist.token_next_by_type(idx, T.Name)
-
-
-def group_order(tlist):
- idx = 0
- token = tlist.token_next_by_type(idx, T.Keyword.Order)
- while token:
- prev = tlist.token_prev(token)
- if isinstance(prev, sql.Identifier):
- ido = tlist.group_tokens(sql.Identifier,
- tlist.tokens_between(prev, token))
- idx = tlist.token_index(ido) + 1
- else:
- idx = tlist.token_index(token) + 1
- token = tlist.token_next_by_type(idx, T.Keyword.Order)
-
-
-def group(tlist):
- for func in [
- group_comments,
- group_parenthesis,
- group_functions,
- group_where,
- group_case,
- group_identifier,
- group_order,
- group_typecasts,
- group_as,
- group_aliased,
- group_assignment,
- group_comparison,
- group_identifier_list,
- group_if,
- group_for,
- group_foreach,
- group_begin]:
- func(tlist)
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/sqlparse/exceptions.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/sqlparse/exceptions.py b/shell/ext-py/sqlparse-0.1.14/sqlparse/exceptions.py
deleted file mode 100644
index ec25afa..0000000
--- a/shell/ext-py/sqlparse-0.1.14/sqlparse/exceptions.py
+++ /dev/null
@@ -1,10 +0,0 @@
-# Copyright (C) 2012 Andi Albrecht, albrecht.andi@gmail.com
-#
-# This module is part of python-sqlparse and is released under
-# the BSD License: http://www.opensource.org/licenses/bsd-license.php.
-
-"""Exceptions used in this package."""
-
-
-class SQLParseError(Exception):
- """Base class for exceptions in this module."""
[3/8] impala git commit: IMPALA-6999: Upgrade to sqlparse-0.1.19 for
Impala shell
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/sqlparse/tokens.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/sqlparse/tokens.py b/shell/ext-py/sqlparse-0.1.19/sqlparse/tokens.py
new file mode 100644
index 0000000..01a9b89
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/sqlparse/tokens.py
@@ -0,0 +1,83 @@
+# Copyright (C) 2008 Andi Albrecht, albrecht.andi@gmail.com
+#
+# This module is part of python-sqlparse and is released under
+# the BSD License: http://www.opensource.org/licenses/bsd-license.php.
+
+# The Token implementation is based on pygment's token system written
+# by Georg Brandl.
+# http://pygments.org/
+
+"""Tokens"""
+
+
+class _TokenType(tuple):
+ parent = None
+
+ def split(self):
+ buf = []
+ node = self
+ while node is not None:
+ buf.append(node)
+ node = node.parent
+ buf.reverse()
+ return buf
+
+ def __contains__(self, val):
+ return val is not None and (self is val or val[:len(self)] == self)
+
+ def __getattr__(self, val):
+ if not val or not val[0].isupper():
+ return tuple.__getattribute__(self, val)
+ new = _TokenType(self + (val,))
+ setattr(self, val, new)
+ new.parent = self
+ return new
+
+ def __hash__(self):
+ return hash(tuple(self))
+
+ def __repr__(self):
+ return 'Token' + (self and '.' or '') + '.'.join(self)
+
+
+Token = _TokenType()
+
+# Special token types
+Text = Token.Text
+Whitespace = Text.Whitespace
+Newline = Whitespace.Newline
+Error = Token.Error
+# Text that doesn't belong to this lexer (e.g. HTML in PHP)
+Other = Token.Other
+
+# Common token types for source code
+Keyword = Token.Keyword
+Name = Token.Name
+Literal = Token.Literal
+String = Literal.String
+Number = Literal.Number
+Punctuation = Token.Punctuation
+Operator = Token.Operator
+Comparison = Operator.Comparison
+Wildcard = Token.Wildcard
+Comment = Token.Comment
+Assignment = Token.Assignement
+
+# Generic types for non-source code
+Generic = Token.Generic
+
+# String and some others are not direct childs of Token.
+# alias them:
+Token.Token = Token
+Token.String = String
+Token.Number = Number
+
+# SQL specific tokens
+DML = Keyword.DML
+DDL = Keyword.DDL
+Command = Keyword.Command
+
+Group = Token.Group
+Group.Parenthesis = Token.Group.Parenthesis
+Group.Comment = Token.Group.Comment
+Group.Where = Token.Group.Where
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/sqlparse/utils.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/sqlparse/utils.py b/shell/ext-py/sqlparse-0.1.19/sqlparse/utils.py
new file mode 100644
index 0000000..3a49ac2
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/sqlparse/utils.py
@@ -0,0 +1,137 @@
+'''
+Created on 17/05/2012
+
+@author: piranna
+'''
+
+import re
+
+try:
+ from collections import OrderedDict
+except ImportError:
+ OrderedDict = None
+
+
+if OrderedDict:
+ class Cache(OrderedDict):
+ """Cache with LRU algorithm using an OrderedDict as basis
+ """
+ def __init__(self, maxsize=100):
+ OrderedDict.__init__(self)
+
+ self._maxsize = maxsize
+
+ def __getitem__(self, key, *args, **kwargs):
+ # Get the key and remove it from the cache, or raise KeyError
+ value = OrderedDict.__getitem__(self, key)
+ del self[key]
+
+ # Insert the (key, value) pair on the front of the cache
+ OrderedDict.__setitem__(self, key, value)
+
+ # Return the value from the cache
+ return value
+
+ def __setitem__(self, key, value, *args, **kwargs):
+ # Key was inserted before, remove it so we put it at front later
+ if key in self:
+ del self[key]
+
+ # Too much items on the cache, remove the least recent used
+ elif len(self) >= self._maxsize:
+ self.popitem(False)
+
+ # Insert the (key, value) pair on the front of the cache
+ OrderedDict.__setitem__(self, key, value, *args, **kwargs)
+
+else:
+ class Cache(dict):
+ """Cache that reset when gets full
+ """
+ def __init__(self, maxsize=100):
+ dict.__init__(self)
+
+ self._maxsize = maxsize
+
+ def __setitem__(self, key, value, *args, **kwargs):
+ # Reset the cache if we have too much cached entries and start over
+ if len(self) >= self._maxsize:
+ self.clear()
+
+ # Insert the (key, value) pair on the front of the cache
+ dict.__setitem__(self, key, value, *args, **kwargs)
+
+
+def memoize_generator(func):
+ """Memoize decorator for generators
+
+ Store `func` results in a cache according to their arguments as 'memoize'
+ does but instead this works on decorators instead of regular functions.
+ Obviusly, this is only useful if the generator will always return the same
+ values for each specific parameters...
+ """
+ cache = Cache()
+
+ def wrapped_func(*args, **kwargs):
+# params = (args, kwargs)
+ params = (args, tuple(sorted(kwargs.items())))
+
+ # Look if cached
+ try:
+ cached = cache[params]
+
+ # Not cached, exec and store it
+ except KeyError:
+ cached = []
+
+ for item in func(*args, **kwargs):
+ cached.append(item)
+ yield item
+
+ cache[params] = cached
+
+ # Cached, yield its items
+ else:
+ for item in cached:
+ yield item
+
+ return wrapped_func
+
+
+# This regular expression replaces the home-cooked parser that was here before.
+# It is much faster, but requires an extra post-processing step to get the
+# desired results (that are compatible with what you would expect from the
+# str.splitlines() method).
+#
+# It matches groups of characters: newlines, quoted strings, or unquoted text,
+# and splits on that basis. The post-processing step puts those back together
+# into the actual lines of SQL.
+SPLIT_REGEX = re.compile(r"""
+(
+ (?: # Start of non-capturing group
+ (?:\r\n|\r|\n) | # Match any single newline, or
+ [^\r\n'"]+ | # Match any character series without quotes or
+ # newlines, or
+ "(?:[^"\\]|\\.)*" | # Match double-quoted strings, or
+ '(?:[^'\\]|\\.)*' # Match single quoted strings
+ )
+)
+""", re.VERBOSE)
+
+LINE_MATCH = re.compile(r'(\r\n|\r|\n)')
+
+def split_unquoted_newlines(text):
+ """Split a string on all unquoted newlines.
+
+ Unlike str.splitlines(), this will ignore CR/LF/CR+LF if the requisite
+ character is inside of a string."""
+ lines = SPLIT_REGEX.split(text)
+ outputlines = ['']
+ for line in lines:
+ if not line:
+ continue
+ elif LINE_MATCH.match(line):
+ outputlines.append('')
+ else:
+ outputlines[-1] += line
+ return outputlines
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/tests/__init__.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/tests/__init__.py b/shell/ext-py/sqlparse-0.1.19/tests/__init__.py
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/tests/files/_Make_DirEntry.sql
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/tests/files/_Make_DirEntry.sql b/shell/ext-py/sqlparse-0.1.19/tests/files/_Make_DirEntry.sql
new file mode 100644
index 0000000..e877bf1
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/tests/files/_Make_DirEntry.sql
@@ -0,0 +1,6 @@
+-- Make a new dir entry
+-- and return its inode
+
+
+INSERT INTO dir_entries(type)
+ VALUES(:type)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/tests/files/begintag.sql
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/tests/files/begintag.sql b/shell/ext-py/sqlparse-0.1.19/tests/files/begintag.sql
new file mode 100644
index 0000000..699b365
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/tests/files/begintag.sql
@@ -0,0 +1,4 @@
+begin;
+update foo
+ set bar = 1;
+commit;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/tests/files/begintag_2.sql
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/tests/files/begintag_2.sql b/shell/ext-py/sqlparse-0.1.19/tests/files/begintag_2.sql
new file mode 100644
index 0000000..0de26d6
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/tests/files/begintag_2.sql
@@ -0,0 +1,13 @@
+CREATE TRIGGER IF NOT EXISTS remove_if_it_was_the_last_file_link
+-- Delete the direntry when is removed it's last static link
+ AFTER DELETE ON links
+ WHEN NOT EXISTS
+ (
+ SELECT * FROM links
+ WHERE child_entry = OLD.child_entry
+ LIMIT 1
+ )
+BEGIN
+ DELETE FROM dir_entries
+ WHERE dir_entries.inode = OLD.child_entry;
+END;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/tests/files/dashcomment.sql
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/tests/files/dashcomment.sql b/shell/ext-py/sqlparse-0.1.19/tests/files/dashcomment.sql
new file mode 100644
index 0000000..0d5ac62
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/tests/files/dashcomment.sql
@@ -0,0 +1,5 @@
+select * from user;
+--select * from host;
+select * from user;
+select * -- foo;
+from foo;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/tests/files/function.sql
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/tests/files/function.sql b/shell/ext-py/sqlparse-0.1.19/tests/files/function.sql
new file mode 100644
index 0000000..d19227f
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/tests/files/function.sql
@@ -0,0 +1,13 @@
+CREATE OR REPLACE FUNCTION foo(
+ p_in1 VARCHAR
+ , p_in2 INTEGER
+) RETURNS INTEGER AS
+
+ DECLARE
+ v_foo INTEGER;
+ BEGIN
+ SELECT *
+ FROM foo
+ INTO v_foo;
+ RETURN v_foo.id;
+ END;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/tests/files/function_psql.sql
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/tests/files/function_psql.sql b/shell/ext-py/sqlparse-0.1.19/tests/files/function_psql.sql
new file mode 100644
index 0000000..e485f7a
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/tests/files/function_psql.sql
@@ -0,0 +1,72 @@
+CREATE OR REPLACE FUNCTION public.delete_data (
+ p_tabelle VARCHAR
+ , p_key VARCHAR
+ , p_value INTEGER
+) RETURNS INTEGER AS
+$$
+DECLARE
+ p_retval INTEGER;
+ v_constraint RECORD;
+ v_count INTEGER;
+ v_data RECORD;
+ v_fieldname VARCHAR;
+ v_sql VARCHAR;
+ v_key VARCHAR;
+ v_value INTEGER;
+BEGIN
+ v_sql := 'SELECT COUNT(*) FROM ' || p_tabelle || ' WHERE ' || p_key || ' = ' || p_value;
+ --RAISE NOTICE '%', v_sql;
+ EXECUTE v_sql INTO v_count;
+ IF v_count::integer != 0 THEN
+ SELECT att.attname
+ INTO v_key
+ FROM pg_attribute att
+ LEFT JOIN pg_constraint con ON con.conrelid = att.attrelid
+ AND con.conkey[1] = att.attnum
+ AND con.contype = 'p', pg_type typ, pg_class rel, pg_namespace ns
+ WHERE att.attrelid = rel.oid
+ AND att.attnum > 0
+ AND typ.oid = att.atttypid
+ AND att.attisdropped = false
+ AND rel.relname = p_tabelle
+ AND con.conkey[1] = 1
+ AND ns.oid = rel.relnamespace
+ AND ns.nspname = 'public'
+ ORDER BY att.attnum;
+ v_sql := 'SELECT ' || v_key || ' AS id FROM ' || p_tabelle || ' WHERE ' || p_key || ' = ' || p_value;
+ FOR v_data IN EXECUTE v_sql
+ LOOP
+ --RAISE NOTICE ' -> % %', p_tabelle, v_data.id;
+ FOR v_constraint IN SELECT t.constraint_name
+ , t.constraint_type
+ , t.table_name
+ , c.column_name
+ FROM public.v_table_constraints t
+ , public.v_constraint_columns c
+ WHERE t.constraint_name = c.constraint_name
+ AND t.constraint_type = 'FOREIGN KEY'
+ AND c.table_name = p_tabelle
+ AND t.table_schema = 'public'
+ AND c.table_schema = 'public'
+ LOOP
+ v_fieldname := substring(v_constraint.constraint_name from 1 for length(v_constraint.constraint_name) - length(v_constraint.column_name) - 1);
+ IF (v_constraint.table_name = p_tabelle) AND (p_value = v_data.id) THEN
+ --RAISE NOTICE 'Skip (Selbstverweis)';
+ CONTINUE;
+ ELSE
+ PERFORM delete_data(v_constraint.table_name::varchar, v_fieldname::varchar, v_data.id::integer);
+ END IF;
+ END LOOP;
+ END LOOP;
+ v_sql := 'DELETE FROM ' || p_tabelle || ' WHERE ' || p_key || ' = ' || p_value;
+ --RAISE NOTICE '%', v_sql;
+ EXECUTE v_sql;
+ p_retval := 1;
+ ELSE
+ --RAISE NOTICE ' -> Keine Sätze gefunden';
+ p_retval := 0;
+ END IF;
+ RETURN p_retval;
+END;
+$$
+LANGUAGE plpgsql;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/tests/files/function_psql2.sql
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/tests/files/function_psql2.sql b/shell/ext-py/sqlparse-0.1.19/tests/files/function_psql2.sql
new file mode 100644
index 0000000..b5d494c
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/tests/files/function_psql2.sql
@@ -0,0 +1,7 @@
+CREATE OR REPLACE FUNCTION update_something() RETURNS void AS
+$body$
+BEGIN
+ raise notice 'foo';
+END;
+$body$
+LANGUAGE 'plpgsql' VOLATILE CALLED ON NULL INPUT SECURITY INVOKER;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/tests/files/function_psql3.sql
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/tests/files/function_psql3.sql b/shell/ext-py/sqlparse-0.1.19/tests/files/function_psql3.sql
new file mode 100644
index 0000000..b25d818
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/tests/files/function_psql3.sql
@@ -0,0 +1,8 @@
+CREATE OR REPLACE FUNCTION foo() RETURNS integer AS
+$body$
+DECLARE
+BEGIN
+ select * from foo;
+END;
+$body$
+LANGUAGE 'plpgsql' VOLATILE CALLED ON NULL INPUT SECURITY INVOKER;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/tests/files/huge_select.sql
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/tests/files/huge_select.sql b/shell/ext-py/sqlparse-0.1.19/tests/files/huge_select.sql
new file mode 100644
index 0000000..ab39823
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/tests/files/huge_select.sql
@@ -0,0 +1 @@
+select case when i = 0 then 1 else 0 end as col0, case when i = 1 then 1 else 1 end as col1, case when i = 2 then 1 else 2 end as col2, case when i = 3 then 1 else 3 end as col3, case when i = 4 then 1 else 4 end as col4, case when i = 5 then 1 else 5 end as col5, case when i = 6 then 1 else 6 end as col6, case when i = 7 then 1 else 7 end as col7, case when i = 8 then 1 else 8 end as col8, case when i = 9 then 1 else 9 end as col9, case when i = 10 then 1 else 10 end as col10, case when i = 11 then 1 else 11 end as col11, case when i = 12 then 1 else 12 end as col12, case when i = 13 then 1 else 13 end as col13, case when i = 14 then 1 else 14 end as col14, case when i = 15 then 1 else 15 end as col15, case when i = 16 then 1 else 16 end as col16, case when i = 17 then 1 else 17 end as col17, case when i = 18 then 1 else 18 end as col18, case when i = 19 then 1 else 19 end as col19, case when i = 20 then 1 else 20 end as col20, case when i = 21 then 1 else 21 end as col21, case whe
n i = 22 then 1 else 22 end as col22 from foo UNION select case when i = 0 then 1 else 0 end as col0, case when i = 1 then 1 else 1 end as col1, case when i = 2 then 1 else 2 end as col2, case when i = 3 then 1 else 3 end as col3, case when i = 4 then 1 else 4 end as col4, case when i = 5 then 1 else 5 end as col5, case when i = 6 then 1 else 6 end as col6, case when i = 7 then 1 else 7 end as col7, case when i = 8 then 1 else 8 end as col8, case when i = 9 then 1 else 9 end as col9, case when i = 10 then 1 else 10 end as col10, case when i = 11 then 1 else 11 end as col11, case when i = 12 then 1 else 12 end as col12, case when i = 13 then 1 else 13 end as col13, case when i = 14 then 1 else 14 end as col14, case when i = 15 then 1 else 15 end as col15, case when i = 16 then 1 else 16 end as col16, case when i = 17 then 1 else 17 end as col17, case when i = 18 then 1 else 18 end as col18, case when i = 19 then 1 else 19 end as col19, case when i = 20 then 1 else 20 end as col20, ca
se when i = 21 then 1 else 21 end as col21, case when i = 22 then 1 else 22 end as col22 from foo UNION select case when i = 0 then 1 else 0 end as col0, case when i = 1 then 1 else 1 end as col1, case when i = 2 then 1 else 2 end as col2, case when i = 3 then 1 else 3 end as col3, case when i = 4 then 1 else 4 end as col4, case when i = 5 then 1 else 5 end as col5, case when i = 6 then 1 else 6 end as col6, case when i = 7 then 1 else 7 end as col7, case when i = 8 then 1 else 8 end as col8, case when i = 9 then 1 else 9 end as col9, case when i = 10 then 1 else 10 end as col10, case when i = 11 then 1 else 11 end as col11, case when i = 12 then 1 else 12 end as col12, case when i = 13 then 1 else 13 end as col13, case when i = 14 then 1 else 14 end as col14, case when i = 15 then 1 else 15 end as col15, case when i = 16 then 1 else 16 end as col16, case when i = 17 then 1 else 17 end as col17, case when i = 18 then 1 else 18 end as col18, case when i = 19 then 1 else 19 end as col
19, case when i = 20 then 1 else 20 end as col20, case when i = 21 then 1 else 21 end as col21, case when i = 22 then 1 else 22 end as col22 from foo UNION select case when i = 0 then 1 else 0 end as col0, case when i = 1 then 1 else 1 end as col1, case when i = 2 then 1 else 2 end as col2, case when i = 3 then 1 else 3 end as col3, case when i = 4 then 1 else 4 end as col4, case when i = 5 then 1 else 5 end as col5, case when i = 6 then 1 else 6 end as col6, case when i = 7 then 1 else 7 end as col7, case when i = 8 then 1 else 8 end as col8, case when i = 9 then 1 else 9 end as col9, case when i = 10 then 1 else 10 end as col10, case when i = 11 then 1 else 11 end as col11, case when i = 12 then 1 else 12 end as col12, case when i = 13 then 1 else 13 end as col13, case when i = 14 then 1 else 14 end as col14, case when i = 15 then 1 else 15 end as col15, case when i = 16 then 1 else 16 end as col16, case when i = 17 then 1 else 17 end as col17, case when i = 18 then 1 else 18 end
as col18, case when i = 19 then 1 else 19 end as col19, case when i = 20 then 1 else 20 end as col20, case when i = 21 then 1 else 21 end as col21, case when i = 22 then 1 else 22 end as col22 from foo UNION select case when i = 0 then 1 else 0 end as col0, case when i = 1 then 1 else 1 end as col1, case when i = 2 then 1 else 2 end as col2, case when i = 3 then 1 else 3 end as col3, case when i = 4 then 1 else 4 end as col4, case when i = 5 then 1 else 5 end as col5, case when i = 6 then 1 else 6 end as col6, case when i = 7 then 1 else 7 end as col7, case when i = 8 then 1 else 8 end as col8, case when i = 9 then 1 else 9 end as col9, case when i = 10 then 1 else 10 end as col10, case when i = 11 then 1 else 11 end as col11, case when i = 12 then 1 else 12 end as col12, case when i = 13 then 1 else 13 end as col13, case when i = 14 then 1 else 14 end as col14, case when i = 15 then 1 else 15 end as col15, case when i = 16 then 1 else 16 end as col16, case when i = 17 then 1 else 1
7 end as col17, case when i = 18 then 1 else 18 end as col18, case when i = 19 then 1 else 19 end as col19, case when i = 20 then 1 else 20 end as col20, case when i = 21 then 1 else 21 end as col21, case when i = 22 then 1 else 22 end as col22 from foo UNION select case when i = 0 then 1 else 0 end as col0, case when i = 1 then 1 else 1 end as col1, case when i = 2 then 1 else 2 end as col2, case when i = 3 then 1 else 3 end as col3, case when i = 4 then 1 else 4 end as col4, case when i = 5 then 1 else 5 end as col5, case when i = 6 then 1 else 6 end as col6, case when i = 7 then 1 else 7 end as col7, case when i = 8 then 1 else 8 end as col8, case when i = 9 then 1 else 9 end as col9, case when i = 10 then 1 else 10 end as col10, case when i = 11 then 1 else 11 end as col11, case when i = 12 then 1 else 12 end as col12, case when i = 13 then 1 else 13 end as col13, case when i = 14 then 1 else 14 end as col14, case when i = 15 then 1 else 15 end as col15, case when i = 16 then 1
else 16 end as col16, case when i = 17 then 1 else 17 end as col17, case when i = 18 then 1 else 18 end as col18, case when i = 19 then 1 else 19 end as col19, case when i = 20 then 1 else 20 end as col20, case when i = 21 then 1 else 21 end as col21, case when i = 22 then 1 else 22 end as col22 from foo UNION select case when i = 0 then 1 else 0 end as col0, case when i = 1 then 1 else 1 end as col1, case when i = 2 then 1 else 2 end as col2, case when i = 3 then 1 else 3 end as col3, case when i = 4 then 1 else 4 end as col4, case when i = 5 then 1 else 5 end as col5, case when i = 6 then 1 else 6 end as col6, case when i = 7 then 1 else 7 end as col7, case when i = 8 then 1 else 8 end as col8, case when i = 9 then 1 else 9 end as col9, case when i = 10 then 1 else 10 end as col10, case when i = 11 then 1 else 11 end as col11, case when i = 12 then 1 else 12 end as col12, case when i = 13 then 1 else 13 end as col13, case when i = 14 then 1 else 14 end as col14, case when i = 15 t
hen 1 else 15 end as col15, case when i = 16 then 1 else 16 end as col16, case when i = 17 then 1 else 17 end as col17, case when i = 18 then 1 else 18 end as col18, case when i = 19 then 1 else 19 end as col19, case when i = 20 then 1 else 20 end as col20, case when i = 21 then 1 else 21 end as col21, case when i = 22 then 1 else 22 end as col22 from foo UNION select case when i = 0 then 1 else 0 end as col0, case when i = 1 then 1 else 1 end as col1, case when i = 2 then 1 else 2 end as col2, case when i = 3 then 1 else 3 end as col3, case when i = 4 then 1 else 4 end as col4, case when i = 5 then 1 else 5 end as col5, case when i = 6 then 1 else 6 end as col6, case when i = 7 then 1 else 7 end as col7, case when i = 8 then 1 else 8 end as col8, case when i = 9 then 1 else 9 end as col9, case when i = 10 then 1 else 10 end as col10, case when i = 11 then 1 else 11 end as col11, case when i = 12 then 1 else 12 end as col12, case when i = 13 then 1 else 13 end as col13, case when i
= 14 then 1 else 14 end as col14, case when i = 15 then 1 else 15 end as col15, case when i = 16 then 1 else 16 end as col16, case when i = 17 then 1 else 17 end as col17, case when i = 18 then 1 else 18 end as col18, case when i = 19 then 1 else 19 end as col19, case when i = 20 then 1 else 20 end as col20, case when i = 21 then 1 else 21 end as col21, case when i = 22 then 1 else 22 end as col22 from foo UNION select case when i = 0 then 1 else 0 end as col0, case when i = 1 then 1 else 1 end as col1, case when i = 2 then 1 else 2 end as col2, case when i = 3 then 1 else 3 end as col3, case when i = 4 then 1 else 4 end as col4, case when i = 5 then 1 else 5 end as col5, case when i = 6 then 1 else 6 end as col6, case when i = 7 then 1 else 7 end as col7, case when i = 8 then 1 else 8 end as col8, case when i = 9 then 1 else 9 end as col9, case when i = 10 then 1 else 10 end as col10, case when i = 11 then 1 else 11 end as col11, case when i = 12 then 1 else 12 end as col12, case w
hen i = 13 then 1 else 13 end as col13, case when i = 14 then 1 else 14 end as col14, case when i = 15 then 1 else 15 end as col15, case when i = 16 then 1 else 16 end as col16, case when i = 17 then 1 else 17 end as col17, case when i = 18 then 1 else 18 end as col18, case when i = 19 then 1 else 19 end as col19, case when i = 20 then 1 else 20 end as col20, case when i = 21 then 1 else 21 end as col21, case when i = 22 then 1 else 22 end as col22 from foo
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/tests/files/test_cp1251.sql
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/tests/files/test_cp1251.sql b/shell/ext-py/sqlparse-0.1.19/tests/files/test_cp1251.sql
new file mode 100644
index 0000000..6c0228b
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/tests/files/test_cp1251.sql
@@ -0,0 +1 @@
+insert into foo values (1); -- ����� ��� �������
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/tests/test_filters.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/tests/test_filters.py b/shell/ext-py/sqlparse-0.1.19/tests/test_filters.py
new file mode 100644
index 0000000..d827454
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/tests/test_filters.py
@@ -0,0 +1,78 @@
+'''
+Created on 24/03/2012
+
+@author: piranna
+'''
+import unittest
+
+from sqlparse.filters import StripWhitespace, Tokens2Unicode
+from sqlparse.lexer import tokenize
+
+
+class Test__StripWhitespace(unittest.TestCase):
+ sql = """INSERT INTO dir_entries(type)VALUES(:type);
+
+ INSERT INTO directories(inode)
+ VALUES(:inode)
+ LIMIT 1"""
+
+ sql2 = """SELECT child_entry,asdf AS inode, creation
+ FROM links
+ WHERE parent_dir == :parent_dir AND name == :name
+ LIMIT 1"""
+
+ sql3 = """SELECT
+ 0 AS st_dev,
+ 0 AS st_uid,
+ 0 AS st_gid,
+
+ dir_entries.type AS st_mode,
+ dir_entries.inode AS st_ino,
+ COUNT(links.child_entry) AS st_nlink,
+
+ :creation AS st_ctime,
+ dir_entries.access AS st_atime,
+ dir_entries.modification AS st_mtime,
+
+ COALESCE(files.size,0) AS st_size,
+ COALESCE(files.size,0) AS size
+
+FROM dir_entries
+ LEFT JOIN files
+ ON dir_entries.inode == files.inode
+ LEFT JOIN links
+ ON dir_entries.inode == links.child_entry
+
+WHERE dir_entries.inode == :inode
+
+GROUP BY dir_entries.inode
+LIMIT 1"""
+
+ def test_StripWhitespace1(self):
+ self.assertEqual(
+ Tokens2Unicode(StripWhitespace(tokenize(self.sql))),
+ 'INSERT INTO dir_entries(type)VALUES(:type);INSERT INTO '
+ 'directories(inode)VALUES(:inode)LIMIT 1')
+
+ def test_StripWhitespace2(self):
+ self.assertEqual(
+ Tokens2Unicode(StripWhitespace(tokenize(self.sql2))),
+ 'SELECT child_entry,asdf AS inode,creation FROM links WHERE '
+ 'parent_dir==:parent_dir AND name==:name LIMIT 1')
+
+ def test_StripWhitespace3(self):
+ self.assertEqual(
+ Tokens2Unicode(StripWhitespace(tokenize(self.sql3))),
+ 'SELECT 0 AS st_dev,0 AS st_uid,0 AS st_gid,dir_entries.type AS '
+ 'st_mode,dir_entries.inode AS st_ino,COUNT(links.child_entry)AS '
+ 'st_nlink,:creation AS st_ctime,dir_entries.access AS st_atime,'
+ 'dir_entries.modification AS st_mtime,COALESCE(files.size,0)AS '
+ 'st_size,COALESCE(files.size,0)AS size FROM dir_entries LEFT JOIN'
+ ' files ON dir_entries.inode==files.inode LEFT JOIN links ON '
+ 'dir_entries.inode==links.child_entry WHERE dir_entries.inode=='
+ ':inode GROUP BY dir_entries.inode LIMIT 1')
+
+
+if __name__ == "__main__":
+ #import sys;sys.argv = ['', 'Test.testName']
+ unittest.main()
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/tests/test_format.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/tests/test_format.py b/shell/ext-py/sqlparse-0.1.19/tests/test_format.py
new file mode 100644
index 0000000..a105b1c
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/tests/test_format.py
@@ -0,0 +1,346 @@
+# -*- coding: utf-8 -*-
+
+import pytest
+
+from tests.utils import TestCaseBase
+
+import sqlparse
+from sqlparse.exceptions import SQLParseError
+
+
+class TestFormat(TestCaseBase):
+
+ def test_keywordcase(self):
+ sql = 'select * from bar; -- select foo\n'
+ res = sqlparse.format(sql, keyword_case='upper')
+ self.ndiffAssertEqual(res, 'SELECT * FROM bar; -- select foo\n')
+ res = sqlparse.format(sql, keyword_case='capitalize')
+ self.ndiffAssertEqual(res, 'Select * From bar; -- select foo\n')
+ res = sqlparse.format(sql.upper(), keyword_case='lower')
+ self.ndiffAssertEqual(res, 'select * from BAR; -- SELECT FOO\n')
+ self.assertRaises(SQLParseError, sqlparse.format, sql,
+ keyword_case='foo')
+
+ def test_identifiercase(self):
+ sql = 'select * from bar; -- select foo\n'
+ res = sqlparse.format(sql, identifier_case='upper')
+ self.ndiffAssertEqual(res, 'select * from BAR; -- select foo\n')
+ res = sqlparse.format(sql, identifier_case='capitalize')
+ self.ndiffAssertEqual(res, 'select * from Bar; -- select foo\n')
+ res = sqlparse.format(sql.upper(), identifier_case='lower')
+ self.ndiffAssertEqual(res, 'SELECT * FROM bar; -- SELECT FOO\n')
+ self.assertRaises(SQLParseError, sqlparse.format, sql,
+ identifier_case='foo')
+ sql = 'select * from "foo"."bar"'
+ res = sqlparse.format(sql, identifier_case="upper")
+ self.ndiffAssertEqual(res, 'select * from "foo"."bar"')
+
+ def test_strip_comments_single(self):
+ sql = 'select *-- statement starts here\nfrom foo'
+ res = sqlparse.format(sql, strip_comments=True)
+ self.ndiffAssertEqual(res, 'select * from foo')
+ sql = 'select * -- statement starts here\nfrom foo'
+ res = sqlparse.format(sql, strip_comments=True)
+ self.ndiffAssertEqual(res, 'select * from foo')
+ sql = 'select-- foo\nfrom -- bar\nwhere'
+ res = sqlparse.format(sql, strip_comments=True)
+ self.ndiffAssertEqual(res, 'select from where')
+ self.assertRaises(SQLParseError, sqlparse.format, sql,
+ strip_comments=None)
+
+ def test_strip_comments_multi(self):
+ sql = '/* sql starts here */\nselect'
+ res = sqlparse.format(sql, strip_comments=True)
+ self.ndiffAssertEqual(res, 'select')
+ sql = '/* sql starts here */ select'
+ res = sqlparse.format(sql, strip_comments=True)
+ self.ndiffAssertEqual(res, 'select')
+ sql = '/*\n * sql starts here\n */\nselect'
+ res = sqlparse.format(sql, strip_comments=True)
+ self.ndiffAssertEqual(res, 'select')
+ sql = 'select (/* sql starts here */ select 2)'
+ res = sqlparse.format(sql, strip_comments=True)
+ self.ndiffAssertEqual(res, 'select (select 2)')
+ sql = 'select (/* sql /* starts here */ select 2)'
+ res = sqlparse.format(sql, strip_comments=True)
+ self.ndiffAssertEqual(res, 'select (select 2)')
+
+ def test_strip_ws(self):
+ f = lambda sql: sqlparse.format(sql, strip_whitespace=True)
+ s = 'select\n* from foo\n\twhere ( 1 = 2 )\n'
+ self.ndiffAssertEqual(f(s), 'select * from foo where (1 = 2)')
+ s = 'select -- foo\nfrom bar\n'
+ self.ndiffAssertEqual(f(s), 'select -- foo\nfrom bar')
+ self.assertRaises(SQLParseError, sqlparse.format, s,
+ strip_whitespace=None)
+
+ def test_preserve_ws(self):
+ # preserve at least one whitespace after subgroups
+ f = lambda sql: sqlparse.format(sql, strip_whitespace=True)
+ s = 'select\n* /* foo */ from bar '
+ self.ndiffAssertEqual(f(s), 'select * /* foo */ from bar')
+
+ def test_notransform_of_quoted_crlf(self):
+ # Make sure that CR/CR+LF characters inside string literals don't get
+ # affected by the formatter.
+
+ s1 = "SELECT some_column LIKE 'value\r'"
+ s2 = "SELECT some_column LIKE 'value\r'\r\nWHERE id = 1\n"
+ s3 = "SELECT some_column LIKE 'value\\'\r' WHERE id = 1\r"
+ s4 = "SELECT some_column LIKE 'value\\\\\\'\r' WHERE id = 1\r\n"
+
+ f = lambda x: sqlparse.format(x)
+
+ # Because of the use of
+ self.ndiffAssertEqual(f(s1), "SELECT some_column LIKE 'value\r'")
+ self.ndiffAssertEqual(f(s2), "SELECT some_column LIKE 'value\r'\nWHERE id = 1\n")
+ self.ndiffAssertEqual(f(s3), "SELECT some_column LIKE 'value\\'\r' WHERE id = 1\n")
+ self.ndiffAssertEqual(f(s4), "SELECT some_column LIKE 'value\\\\\\'\r' WHERE id = 1\n")
+
+ def test_outputformat(self):
+ sql = 'select * from foo;'
+ self.assertRaises(SQLParseError, sqlparse.format, sql,
+ output_format='foo')
+
+
+class TestFormatReindent(TestCaseBase):
+
+ def test_option(self):
+ self.assertRaises(SQLParseError, sqlparse.format, 'foo',
+ reindent=2)
+ self.assertRaises(SQLParseError, sqlparse.format, 'foo',
+ indent_tabs=2)
+ self.assertRaises(SQLParseError, sqlparse.format, 'foo',
+ reindent=True, indent_width='foo')
+ self.assertRaises(SQLParseError, sqlparse.format, 'foo',
+ reindent=True, indent_width=-12)
+
+ def test_stmts(self):
+ f = lambda sql: sqlparse.format(sql, reindent=True)
+ s = 'select foo; select bar'
+ self.ndiffAssertEqual(f(s), 'select foo;\n\nselect bar')
+ s = 'select foo'
+ self.ndiffAssertEqual(f(s), 'select foo')
+ s = 'select foo; -- test\n select bar'
+ self.ndiffAssertEqual(f(s), 'select foo; -- test\n\nselect bar')
+
+ def test_keywords(self):
+ f = lambda sql: sqlparse.format(sql, reindent=True)
+ s = 'select * from foo union select * from bar;'
+ self.ndiffAssertEqual(f(s), '\n'.join(['select *',
+ 'from foo',
+ 'union',
+ 'select *',
+ 'from bar;']))
+
+ def test_keywords_between(self): # issue 14
+ # don't break AND after BETWEEN
+ f = lambda sql: sqlparse.format(sql, reindent=True)
+ s = 'and foo between 1 and 2 and bar = 3'
+ self.ndiffAssertEqual(f(s), '\n'.join(['',
+ 'and foo between 1 and 2',
+ 'and bar = 3']))
+
+ def test_parenthesis(self):
+ f = lambda sql: sqlparse.format(sql, reindent=True)
+ s = 'select count(*) from (select * from foo);'
+ self.ndiffAssertEqual(f(s),
+ '\n'.join(['select count(*)',
+ 'from',
+ ' (select *',
+ ' from foo);',
+ ])
+ )
+
+ def test_where(self):
+ f = lambda sql: sqlparse.format(sql, reindent=True)
+ s = 'select * from foo where bar = 1 and baz = 2 or bzz = 3;'
+ self.ndiffAssertEqual(f(s), ('select *\nfrom foo\n'
+ 'where bar = 1\n'
+ ' and baz = 2\n'
+ ' or bzz = 3;'))
+ s = 'select * from foo where bar = 1 and (baz = 2 or bzz = 3);'
+ self.ndiffAssertEqual(f(s), ('select *\nfrom foo\n'
+ 'where bar = 1\n'
+ ' and (baz = 2\n'
+ ' or bzz = 3);'))
+
+ def test_join(self):
+ f = lambda sql: sqlparse.format(sql, reindent=True)
+ s = 'select * from foo join bar on 1 = 2'
+ self.ndiffAssertEqual(f(s), '\n'.join(['select *',
+ 'from foo',
+ 'join bar on 1 = 2']))
+ s = 'select * from foo inner join bar on 1 = 2'
+ self.ndiffAssertEqual(f(s), '\n'.join(['select *',
+ 'from foo',
+ 'inner join bar on 1 = 2']))
+ s = 'select * from foo left outer join bar on 1 = 2'
+ self.ndiffAssertEqual(f(s), '\n'.join(['select *',
+ 'from foo',
+ 'left outer join bar on 1 = 2']
+ ))
+ s = 'select * from foo straight_join bar on 1 = 2'
+ self.ndiffAssertEqual(f(s), '\n'.join(['select *',
+ 'from foo',
+ 'straight_join bar on 1 = 2']
+ ))
+
+ def test_identifier_list(self):
+ f = lambda sql: sqlparse.format(sql, reindent=True)
+ s = 'select foo, bar, baz from table1, table2 where 1 = 2'
+ self.ndiffAssertEqual(f(s), '\n'.join(['select foo,',
+ ' bar,',
+ ' baz',
+ 'from table1,',
+ ' table2',
+ 'where 1 = 2']))
+ s = 'select a.*, b.id from a, b'
+ self.ndiffAssertEqual(f(s), '\n'.join(['select a.*,',
+ ' b.id',
+ 'from a,',
+ ' b']))
+
+ def test_identifier_list_with_functions(self):
+ f = lambda sql: sqlparse.format(sql, reindent=True)
+ s = ("select 'abc' as foo, coalesce(col1, col2)||col3 as bar,"
+ "col3 from my_table")
+ self.ndiffAssertEqual(f(s), '\n'.join(
+ ["select 'abc' as foo,",
+ " coalesce(col1, col2)||col3 as bar,",
+ " col3",
+ "from my_table"]))
+
+ def test_case(self):
+ f = lambda sql: sqlparse.format(sql, reindent=True)
+ s = 'case when foo = 1 then 2 when foo = 3 then 4 else 5 end'
+ self.ndiffAssertEqual(f(s), '\n'.join(['case',
+ ' when foo = 1 then 2',
+ ' when foo = 3 then 4',
+ ' else 5',
+ 'end']))
+
+ def test_case2(self):
+ f = lambda sql: sqlparse.format(sql, reindent=True)
+ s = 'case(foo) when bar = 1 then 2 else 3 end'
+ self.ndiffAssertEqual(f(s), '\n'.join(['case(foo)',
+ ' when bar = 1 then 2',
+ ' else 3',
+ 'end']))
+
+ def test_nested_identifier_list(self): # issue4
+ f = lambda sql: sqlparse.format(sql, reindent=True)
+ s = '(foo as bar, bar1, bar2 as bar3, b4 as b5)'
+ self.ndiffAssertEqual(f(s), '\n'.join(['(foo as bar,',
+ ' bar1,',
+ ' bar2 as bar3,',
+ ' b4 as b5)']))
+
+ def test_duplicate_linebreaks(self): # issue3
+ f = lambda sql: sqlparse.format(sql, reindent=True)
+ s = 'select c1 -- column1\nfrom foo'
+ self.ndiffAssertEqual(f(s), '\n'.join(['select c1 -- column1',
+ 'from foo']))
+ s = 'select c1 -- column1\nfrom foo'
+ r = sqlparse.format(s, reindent=True, strip_comments=True)
+ self.ndiffAssertEqual(r, '\n'.join(['select c1',
+ 'from foo']))
+ s = 'select c1\nfrom foo\norder by c1'
+ self.ndiffAssertEqual(f(s), '\n'.join(['select c1',
+ 'from foo',
+ 'order by c1']))
+ s = 'select c1 from t1 where (c1 = 1) order by c1'
+ self.ndiffAssertEqual(f(s), '\n'.join(['select c1',
+ 'from t1',
+ 'where (c1 = 1)',
+ 'order by c1']))
+
+ def test_keywordfunctions(self): # issue36
+ f = lambda sql: sqlparse.format(sql, reindent=True)
+ s = 'select max(a) b, foo, bar'
+ self.ndiffAssertEqual(f(s), '\n'.join(['select max(a) b,',
+ ' foo,',
+ ' bar']))
+
+ def test_identifier_and_functions(self): # issue45
+ f = lambda sql: sqlparse.format(sql, reindent=True)
+ s = 'select foo.bar, nvl(1) from dual'
+ self.ndiffAssertEqual(f(s), '\n'.join(['select foo.bar,',
+ ' nvl(1)',
+ 'from dual']))
+
+
+class TestOutputFormat(TestCaseBase):
+
+ def test_python(self):
+ sql = 'select * from foo;'
+ f = lambda sql: sqlparse.format(sql, output_format='python')
+ self.ndiffAssertEqual(f(sql), "sql = 'select * from foo;'")
+ f = lambda sql: sqlparse.format(sql, output_format='python',
+ reindent=True)
+ self.ndiffAssertEqual(f(sql), ("sql = ('select * '\n"
+ " 'from foo;')"))
+
+ def test_php(self):
+ sql = 'select * from foo;'
+ f = lambda sql: sqlparse.format(sql, output_format='php')
+ self.ndiffAssertEqual(f(sql), '$sql = "select * from foo;";')
+ f = lambda sql: sqlparse.format(sql, output_format='php',
+ reindent=True)
+ self.ndiffAssertEqual(f(sql), ('$sql = "select * ";\n'
+ '$sql .= "from foo;";'))
+
+ def test_sql(self): # "sql" is an allowed option but has no effect
+ sql = 'select * from foo;'
+ f = lambda sql: sqlparse.format(sql, output_format='sql')
+ self.ndiffAssertEqual(f(sql), 'select * from foo;')
+
+
+def test_format_column_ordering(): # issue89
+ sql = 'select * from foo order by c1 desc, c2, c3;'
+ formatted = sqlparse.format(sql, reindent=True)
+ expected = '\n'.join(['select *',
+ 'from foo',
+ 'order by c1 desc,',
+ ' c2,',
+ ' c3;'])
+ assert formatted == expected
+
+
+def test_truncate_strings():
+ sql = 'update foo set value = \'' + 'x' * 1000 + '\';'
+ formatted = sqlparse.format(sql, truncate_strings=10)
+ assert formatted == 'update foo set value = \'xxxxxxxxxx[...]\';'
+ formatted = sqlparse.format(sql, truncate_strings=3, truncate_char='YYY')
+ assert formatted == 'update foo set value = \'xxxYYY\';'
+
+
+def test_truncate_strings_invalid_option():
+ pytest.raises(SQLParseError, sqlparse.format,
+ 'foo', truncate_strings='bar')
+ pytest.raises(SQLParseError, sqlparse.format,
+ 'foo', truncate_strings=-1)
+ pytest.raises(SQLParseError, sqlparse.format,
+ 'foo', truncate_strings=0)
+
+
+@pytest.mark.parametrize('sql', ['select verrrylongcolumn from foo',
+ 'select "verrrylongcolumn" from "foo"'])
+def test_truncate_strings_doesnt_truncate_identifiers(sql):
+ formatted = sqlparse.format(sql, truncate_strings=2)
+ assert formatted == sql
+
+
+def test_having_produces_newline():
+ sql = (
+ 'select * from foo, bar where bar.id = foo.bar_id'
+ ' having sum(bar.value) > 100')
+ formatted = sqlparse.format(sql, reindent=True)
+ expected = [
+ 'select *',
+ 'from foo,',
+ ' bar',
+ 'where bar.id = foo.bar_id',
+ 'having sum(bar.value) > 100'
+ ]
+ assert formatted == '\n'.join(expected)
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/tests/test_functions.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/tests/test_functions.py b/shell/ext-py/sqlparse-0.1.19/tests/test_functions.py
new file mode 100644
index 0000000..52e2ce7
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/tests/test_functions.py
@@ -0,0 +1,164 @@
+'''
+Created on 13/02/2012
+
+@author: piranna
+'''
+from unittest import main, TestCase
+
+from sqlparse.filters import IncludeStatement, Tokens2Unicode
+from sqlparse.lexer import tokenize
+
+import sys
+sys.path.insert(0, '..')
+
+from sqlparse.filters import compact
+from sqlparse.functions import getcolumns, getlimit, IsType
+
+
+class Test_IncludeStatement(TestCase):
+ sql = """-- type: script
+ -- return: integer
+
+ INCLUDE "_Make_DirEntry.sql";
+
+ INSERT INTO directories(inode)
+ VALUES(:inode)
+ LIMIT 1"""
+
+ def test_includeStatement(self):
+ stream = tokenize(self.sql)
+ includeStatement = IncludeStatement('tests/files',
+ raiseexceptions=True)
+ stream = includeStatement.process(None, stream)
+ stream = compact(stream)
+
+ result = Tokens2Unicode(stream)
+
+ self.assertEqual(
+ result, (
+ 'INSERT INTO dir_entries(type)VALUES(:type);INSERT INTO '
+ 'directories(inode)VALUES(:inode)LIMIT 1'))
+
+
+class Test_SQL(TestCase):
+ sql = """-- type: script
+ -- return: integer
+
+ INSERT INTO directories(inode)
+ VALUES(:inode)
+ LIMIT 1"""
+
+ sql2 = """SELECT child_entry,asdf AS inode, creation
+ FROM links
+ WHERE parent_dir == :parent_dir AND name == :name
+ LIMIT 1"""
+
+ sql3 = """SELECT
+ 0 AS st_dev,
+ 0 AS st_uid,
+ 0 AS st_gid,
+
+ dir_entries.type AS st_mode,
+ dir_entries.inode AS st_ino,
+ COUNT(links.child_entry) AS st_nlink,
+
+ :creation AS st_ctime,
+ dir_entries.access AS st_atime,
+ dir_entries.modification AS st_mtime,
+-- :creation AS st_ctime,
+-- CAST(STRFTIME('%s',dir_entries.access) AS INTEGER) AS st_atime,
+-- CAST(STRFTIME('%s',dir_entries.modification) AS INTEGER) AS st_mtime,
+
+ COALESCE(files.size,0) AS st_size, -- Python-FUSE
+ COALESCE(files.size,0) AS size -- PyFilesystem
+
+FROM dir_entries
+ LEFT JOIN files
+ ON dir_entries.inode == files.inode
+ LEFT JOIN links
+ ON dir_entries.inode == links.child_entry
+
+WHERE dir_entries.inode == :inode
+
+GROUP BY dir_entries.inode
+LIMIT 1"""
+
+
+class Test_Compact(Test_SQL):
+ def test_compact1(self):
+ stream = compact(tokenize(self.sql))
+
+ result = Tokens2Unicode(stream)
+
+ self.assertEqual(result,
+ 'INSERT INTO directories(inode)VALUES(:inode)LIMIT 1')
+
+ def test_compact2(self):
+ stream = tokenize(self.sql2)
+
+ result = compact(stream)
+
+ self.assertEqual(
+ Tokens2Unicode(result),
+ 'SELECT child_entry,asdf AS inode,creation FROM links WHERE '
+ 'parent_dir==:parent_dir AND name==:name LIMIT 1')
+
+ def test_compact3(self):
+ stream = tokenize(self.sql3)
+
+ result = compact(stream)
+
+ self.assertEqual(
+ Tokens2Unicode(result),
+ 'SELECT 0 AS st_dev,0 AS st_uid,0 AS st_gid,dir_entries.type AS '
+ 'st_mode,dir_entries.inode AS st_ino,COUNT(links.child_entry)AS '
+ 'st_nlink,:creation AS st_ctime,dir_entries.access AS st_atime,'
+ 'dir_entries.modification AS st_mtime,COALESCE(files.size,0)AS '
+ 'st_size,COALESCE(files.size,0)AS size FROM dir_entries LEFT JOIN'
+ ' files ON dir_entries.inode==files.inode LEFT JOIN links ON '
+ 'dir_entries.inode==links.child_entry WHERE dir_entries.inode=='
+ ':inode GROUP BY dir_entries.inode LIMIT 1')
+
+
+class Test_GetColumns(Test_SQL):
+ def test_getcolumns1(self):
+ columns = getcolumns(tokenize(self.sql))
+ self.assertEqual(columns, [])
+
+ def test_getcolumns2(self):
+ columns = getcolumns(tokenize(self.sql2))
+ self.assertEqual(columns, ['child_entry', 'inode', 'creation'])
+
+ def test_getcolumns3(self):
+ columns = getcolumns(tokenize(self.sql3))
+ self.assertEqual(columns, ['st_dev', 'st_uid', 'st_gid', 'st_mode',
+ 'st_ino', 'st_nlink', 'st_ctime',
+ 'st_atime', 'st_mtime', 'st_size', 'size'])
+
+
+class Test_GetLimit(Test_SQL):
+ def test_getlimit1(self):
+ limit = getlimit(tokenize(self.sql))
+ self.assertEqual(limit, 1)
+
+ def test_getlimit2(self):
+ limit = getlimit(tokenize(self.sql2))
+ self.assertEqual(limit, 1)
+
+ def test_getlimit3(self):
+ limit = getlimit(tokenize(self.sql3))
+ self.assertEqual(limit, 1)
+
+
+class Test_IsType(Test_SQL):
+ def test_istype2(self):
+ stream = tokenize(self.sql2)
+ self.assertTrue(IsType('SELECT')(stream))
+
+ stream = tokenize(self.sql2)
+ self.assertFalse(IsType('INSERT')(stream))
+
+
+if __name__ == "__main__":
+ #import sys;sys.argv = ['', 'Test.testName']
+ main()
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/tests/test_grouping.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/tests/test_grouping.py b/shell/ext-py/sqlparse-0.1.19/tests/test_grouping.py
new file mode 100644
index 0000000..5ade830
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/tests/test_grouping.py
@@ -0,0 +1,399 @@
+# -*- coding: utf-8 -*-
+
+import pytest
+
+import sqlparse
+from sqlparse import sql
+from sqlparse import tokens as T
+
+from tests.utils import TestCaseBase
+
+
+class TestGrouping(TestCaseBase):
+
+ def test_parenthesis(self):
+ s = 'select (select (x3) x2) and (y2) bar'
+ parsed = sqlparse.parse(s)[0]
+ self.ndiffAssertEqual(s, str(parsed))
+ self.assertEqual(len(parsed.tokens), 7)
+ self.assert_(isinstance(parsed.tokens[2], sql.Parenthesis))
+ self.assert_(isinstance(parsed.tokens[-1], sql.Identifier))
+ self.assertEqual(len(parsed.tokens[2].tokens), 5)
+ self.assert_(isinstance(parsed.tokens[2].tokens[3], sql.Identifier))
+ self.assert_(isinstance(parsed.tokens[2].tokens[3].tokens[0], sql.Parenthesis))
+ self.assertEqual(len(parsed.tokens[2].tokens[3].tokens), 3)
+
+ def test_comments(self):
+ s = '/*\n * foo\n */ \n bar'
+ parsed = sqlparse.parse(s)[0]
+ self.ndiffAssertEqual(s, unicode(parsed))
+ self.assertEqual(len(parsed.tokens), 2)
+
+ def test_assignment(self):
+ s = 'foo := 1;'
+ parsed = sqlparse.parse(s)[0]
+ self.assertEqual(len(parsed.tokens), 1)
+ self.assert_(isinstance(parsed.tokens[0], sql.Assignment))
+ s = 'foo := 1'
+ parsed = sqlparse.parse(s)[0]
+ self.assertEqual(len(parsed.tokens), 1)
+ self.assert_(isinstance(parsed.tokens[0], sql.Assignment))
+
+ def test_identifiers(self):
+ s = 'select foo.bar from "myscheme"."table" where fail. order'
+ parsed = sqlparse.parse(s)[0]
+ self.ndiffAssertEqual(s, unicode(parsed))
+ self.assert_(isinstance(parsed.tokens[2], sql.Identifier))
+ self.assert_(isinstance(parsed.tokens[6], sql.Identifier))
+ self.assert_(isinstance(parsed.tokens[8], sql.Where))
+ s = 'select * from foo where foo.id = 1'
+ parsed = sqlparse.parse(s)[0]
+ self.ndiffAssertEqual(s, unicode(parsed))
+ self.assert_(isinstance(parsed.tokens[-1].tokens[-1].tokens[0],
+ sql.Identifier))
+ s = 'select * from (select "foo"."id" from foo)'
+ parsed = sqlparse.parse(s)[0]
+ self.ndiffAssertEqual(s, unicode(parsed))
+ self.assert_(isinstance(parsed.tokens[-1].tokens[3], sql.Identifier))
+
+ s = "INSERT INTO `test` VALUES('foo', 'bar');"
+ parsed = sqlparse.parse(s)[0]
+ types = [l.ttype for l in parsed.tokens if not l.is_whitespace()]
+ self.assertEquals(types, [T.DML, T.Keyword, None,
+ T.Keyword, None, T.Punctuation])
+
+ s = "select 1.0*(a+b) as col, sum(c)/sum(d) from myschema.mytable"
+ parsed = sqlparse.parse(s)[0]
+ self.assertEqual(len(parsed.tokens), 7)
+ self.assert_(isinstance(parsed.tokens[2], sql.IdentifierList))
+ self.assertEqual(len(parsed.tokens[2].tokens), 4)
+ identifiers = list(parsed.tokens[2].get_identifiers())
+ self.assertEqual(len(identifiers), 2)
+ self.assertEquals(identifiers[0].get_alias(), u"col")
+
+ def test_identifier_wildcard(self):
+ p = sqlparse.parse('a.*, b.id')[0]
+ self.assert_(isinstance(p.tokens[0], sql.IdentifierList))
+ self.assert_(isinstance(p.tokens[0].tokens[0], sql.Identifier))
+ self.assert_(isinstance(p.tokens[0].tokens[-1], sql.Identifier))
+
+ def test_identifier_name_wildcard(self):
+ p = sqlparse.parse('a.*')[0]
+ t = p.tokens[0]
+ self.assertEqual(t.get_name(), '*')
+ self.assertEqual(t.is_wildcard(), True)
+
+ def test_identifier_invalid(self):
+ p = sqlparse.parse('a.')[0]
+ self.assert_(isinstance(p.tokens[0], sql.Identifier))
+ self.assertEqual(p.tokens[0].has_alias(), False)
+ self.assertEqual(p.tokens[0].get_name(), None)
+ self.assertEqual(p.tokens[0].get_real_name(), None)
+ self.assertEqual(p.tokens[0].get_parent_name(), 'a')
+
+ def test_identifier_as_invalid(self): # issue8
+ p = sqlparse.parse('foo as select *')[0]
+ self.assert_(len(p.tokens), 5)
+ self.assert_(isinstance(p.tokens[0], sql.Identifier))
+ self.assertEqual(len(p.tokens[0].tokens), 1)
+ self.assertEqual(p.tokens[2].ttype, T.Keyword)
+
+ def test_identifier_function(self):
+ p = sqlparse.parse('foo() as bar')[0]
+ self.assert_(isinstance(p.tokens[0], sql.Identifier))
+ self.assert_(isinstance(p.tokens[0].tokens[0], sql.Function))
+ p = sqlparse.parse('foo()||col2 bar')[0]
+ self.assert_(isinstance(p.tokens[0], sql.Identifier))
+ self.assert_(isinstance(p.tokens[0].tokens[0], sql.Function))
+
+ def test_identifier_extended(self): # issue 15
+ p = sqlparse.parse('foo+100')[0]
+ self.assert_(isinstance(p.tokens[0], sql.Identifier))
+ p = sqlparse.parse('foo + 100')[0]
+ self.assert_(isinstance(p.tokens[0], sql.Identifier))
+ p = sqlparse.parse('foo*100')[0]
+ self.assert_(isinstance(p.tokens[0], sql.Identifier))
+
+ def test_identifier_list(self):
+ p = sqlparse.parse('a, b, c')[0]
+ self.assert_(isinstance(p.tokens[0], sql.IdentifierList))
+ p = sqlparse.parse('(a, b, c)')[0]
+ self.assert_(isinstance(p.tokens[0].tokens[1], sql.IdentifierList))
+
+ def test_identifier_list_case(self):
+ p = sqlparse.parse('a, case when 1 then 2 else 3 end as b, c')[0]
+ self.assert_(isinstance(p.tokens[0], sql.IdentifierList))
+ p = sqlparse.parse('(a, case when 1 then 2 else 3 end as b, c)')[0]
+ self.assert_(isinstance(p.tokens[0].tokens[1], sql.IdentifierList))
+
+ def test_identifier_list_other(self): # issue2
+ p = sqlparse.parse("select *, null, 1, 'foo', bar from mytable, x")[0]
+ self.assert_(isinstance(p.tokens[2], sql.IdentifierList))
+ l = p.tokens[2]
+ self.assertEqual(len(l.tokens), 13)
+
+ def test_identifier_list_with_inline_comments(self): # issue163
+ p = sqlparse.parse('foo /* a comment */, bar')[0]
+ self.assert_(isinstance(p.tokens[0], sql.IdentifierList))
+ self.assert_(isinstance(p.tokens[0].tokens[0], sql.Identifier))
+ self.assert_(isinstance(p.tokens[0].tokens[3], sql.Identifier))
+
+ def test_where(self):
+ s = 'select * from foo where bar = 1 order by id desc'
+ p = sqlparse.parse(s)[0]
+ self.ndiffAssertEqual(s, unicode(p))
+ self.assertTrue(len(p.tokens), 16)
+ s = 'select x from (select y from foo where bar = 1) z'
+ p = sqlparse.parse(s)[0]
+ self.ndiffAssertEqual(s, unicode(p))
+ self.assertTrue(isinstance(p.tokens[-1].tokens[0].tokens[-2], sql.Where))
+
+ def test_typecast(self):
+ s = 'select foo::integer from bar'
+ p = sqlparse.parse(s)[0]
+ self.ndiffAssertEqual(s, unicode(p))
+ self.assertEqual(p.tokens[2].get_typecast(), 'integer')
+ self.assertEqual(p.tokens[2].get_name(), 'foo')
+ s = 'select (current_database())::information_schema.sql_identifier'
+ p = sqlparse.parse(s)[0]
+ self.ndiffAssertEqual(s, unicode(p))
+ self.assertEqual(p.tokens[2].get_typecast(),
+ 'information_schema.sql_identifier')
+
+ def test_alias(self):
+ s = 'select foo as bar from mytable'
+ p = sqlparse.parse(s)[0]
+ self.ndiffAssertEqual(s, unicode(p))
+ self.assertEqual(p.tokens[2].get_real_name(), 'foo')
+ self.assertEqual(p.tokens[2].get_alias(), 'bar')
+ s = 'select foo from mytable t1'
+ p = sqlparse.parse(s)[0]
+ self.ndiffAssertEqual(s, unicode(p))
+ self.assertEqual(p.tokens[6].get_real_name(), 'mytable')
+ self.assertEqual(p.tokens[6].get_alias(), 't1')
+ s = 'select foo::integer as bar from mytable'
+ p = sqlparse.parse(s)[0]
+ self.ndiffAssertEqual(s, unicode(p))
+ self.assertEqual(p.tokens[2].get_alias(), 'bar')
+ s = ('SELECT DISTINCT '
+ '(current_database())::information_schema.sql_identifier AS view')
+ p = sqlparse.parse(s)[0]
+ self.ndiffAssertEqual(s, unicode(p))
+ self.assertEqual(p.tokens[4].get_alias(), 'view')
+
+ def test_alias_case(self): # see issue46
+ p = sqlparse.parse('CASE WHEN 1 THEN 2 ELSE 3 END foo')[0]
+ self.assertEqual(len(p.tokens), 1)
+ self.assertEqual(p.tokens[0].get_alias(), 'foo')
+
+ def test_alias_returns_none(self): # see issue185
+ p = sqlparse.parse('foo.bar')[0]
+ self.assertEqual(len(p.tokens), 1)
+ self.assertEqual(p.tokens[0].get_alias(), None)
+
+ def test_idlist_function(self): # see issue10 too
+ p = sqlparse.parse('foo(1) x, bar')[0]
+ self.assert_(isinstance(p.tokens[0], sql.IdentifierList))
+
+ def test_comparison_exclude(self):
+ # make sure operators are not handled too lazy
+ p = sqlparse.parse('(=)')[0]
+ self.assert_(isinstance(p.tokens[0], sql.Parenthesis))
+ self.assert_(not isinstance(p.tokens[0].tokens[1], sql.Comparison))
+ p = sqlparse.parse('(a=1)')[0]
+ self.assert_(isinstance(p.tokens[0].tokens[1], sql.Comparison))
+ p = sqlparse.parse('(a>=1)')[0]
+ self.assert_(isinstance(p.tokens[0].tokens[1], sql.Comparison))
+
+ def test_function(self):
+ p = sqlparse.parse('foo()')[0]
+ self.assert_(isinstance(p.tokens[0], sql.Function))
+ p = sqlparse.parse('foo(null, bar)')[0]
+ self.assert_(isinstance(p.tokens[0], sql.Function))
+ self.assertEqual(len(list(p.tokens[0].get_parameters())), 2)
+
+ def test_function_not_in(self): # issue183
+ p = sqlparse.parse('in(1, 2)')[0]
+ self.assertEqual(len(p.tokens), 2)
+ self.assertEqual(p.tokens[0].ttype, T.Keyword)
+ self.assert_(isinstance(p.tokens[1], sql.Parenthesis))
+
+ def test_varchar(self):
+ p = sqlparse.parse('"text" Varchar(50) NOT NULL')[0]
+ self.assert_(isinstance(p.tokens[2], sql.Function))
+
+
+class TestStatement(TestCaseBase):
+
+ def test_get_type(self):
+ f = lambda sql: sqlparse.parse(sql)[0]
+ self.assertEqual(f('select * from foo').get_type(), 'SELECT')
+ self.assertEqual(f('update foo').get_type(), 'UPDATE')
+ self.assertEqual(f(' update foo').get_type(), 'UPDATE')
+ self.assertEqual(f('\nupdate foo').get_type(), 'UPDATE')
+ self.assertEqual(f('foo').get_type(), 'UNKNOWN')
+ # Statements that have a whitespace after the closing semicolon
+ # are parsed as two statements where later only consists of the
+ # trailing whitespace.
+ self.assertEqual(f('\n').get_type(), 'UNKNOWN')
+
+
+def test_identifier_with_operators(): # issue 53
+ p = sqlparse.parse('foo||bar')[0]
+ assert len(p.tokens) == 1
+ assert isinstance(p.tokens[0], sql.Identifier)
+ # again with whitespaces
+ p = sqlparse.parse('foo || bar')[0]
+ assert len(p.tokens) == 1
+ assert isinstance(p.tokens[0], sql.Identifier)
+
+
+def test_identifier_with_op_trailing_ws():
+ # make sure trailing whitespace isn't grouped with identifier
+ p = sqlparse.parse('foo || bar ')[0]
+ assert len(p.tokens) == 2
+ assert isinstance(p.tokens[0], sql.Identifier)
+ assert p.tokens[1].ttype is T.Whitespace
+
+
+def test_identifier_with_string_literals():
+ p = sqlparse.parse('foo + \'bar\'')[0]
+ assert len(p.tokens) == 1
+ assert isinstance(p.tokens[0], sql.Identifier)
+
+
+# This test seems to be wrong. It was introduced when fixing #53, but #111
+# showed that this shouldn't be an identifier at all. I'm leaving this
+# commented in the source for a while.
+# def test_identifier_string_concat():
+# p = sqlparse.parse('\'foo\' || bar')[0]
+# assert len(p.tokens) == 1
+# assert isinstance(p.tokens[0], sql.Identifier)
+
+
+def test_identifier_consumes_ordering(): # issue89
+ p = sqlparse.parse('select * from foo order by c1 desc, c2, c3')[0]
+ assert isinstance(p.tokens[-1], sql.IdentifierList)
+ ids = list(p.tokens[-1].get_identifiers())
+ assert len(ids) == 3
+ assert ids[0].get_name() == 'c1'
+ assert ids[0].get_ordering() == 'DESC'
+ assert ids[1].get_name() == 'c2'
+ assert ids[1].get_ordering() is None
+
+
+def test_comparison_with_keywords(): # issue90
+ # in fact these are assignments, but for now we don't distinguish them
+ p = sqlparse.parse('foo = NULL')[0]
+ assert len(p.tokens) == 1
+ assert isinstance(p.tokens[0], sql.Comparison)
+ assert len(p.tokens[0].tokens) == 5
+ assert p.tokens[0].left.value == 'foo'
+ assert p.tokens[0].right.value == 'NULL'
+ # make sure it's case-insensitive
+ p = sqlparse.parse('foo = null')[0]
+ assert len(p.tokens) == 1
+ assert isinstance(p.tokens[0], sql.Comparison)
+
+
+def test_comparison_with_floats(): # issue145
+ p = sqlparse.parse('foo = 25.5')[0]
+ assert len(p.tokens) == 1
+ assert isinstance(p.tokens[0], sql.Comparison)
+ assert len(p.tokens[0].tokens) == 5
+ assert p.tokens[0].left.value == 'foo'
+ assert p.tokens[0].right.value == '25.5'
+
+
+def test_comparison_with_parenthesis(): # issue23
+ p = sqlparse.parse('(3 + 4) = 7')[0]
+ assert len(p.tokens) == 1
+ assert isinstance(p.tokens[0], sql.Comparison)
+ comp = p.tokens[0]
+ assert isinstance(comp.left, sql.Parenthesis)
+ assert comp.right.ttype is T.Number.Integer
+
+
+def test_comparison_with_strings(): # issue148
+ p = sqlparse.parse('foo = \'bar\'')[0]
+ assert len(p.tokens) == 1
+ assert isinstance(p.tokens[0], sql.Comparison)
+ assert p.tokens[0].right.value == '\'bar\''
+ assert p.tokens[0].right.ttype == T.String.Single
+
+
+@pytest.mark.parametrize('start', ['FOR', 'FOREACH'])
+def test_forloops(start):
+ p = sqlparse.parse('%s foo in bar LOOP foobar END LOOP' % start)[0]
+ assert (len(p.tokens)) == 1
+ assert isinstance(p.tokens[0], sql.For)
+
+
+def test_nested_for():
+ p = sqlparse.parse('FOR foo LOOP FOR bar LOOP END LOOP END LOOP')[0]
+ assert len(p.tokens) == 1
+ for1 = p.tokens[0]
+ assert for1.tokens[0].value == 'FOR'
+ assert for1.tokens[-1].value == 'END LOOP'
+ for2 = for1.tokens[6]
+ assert isinstance(for2, sql.For)
+ assert for2.tokens[0].value == 'FOR'
+ assert for2.tokens[-1].value == 'END LOOP'
+
+
+def test_begin():
+ p = sqlparse.parse('BEGIN foo END')[0]
+ assert len(p.tokens) == 1
+ assert isinstance(p.tokens[0], sql.Begin)
+
+
+def test_nested_begin():
+ p = sqlparse.parse('BEGIN foo BEGIN bar END END')[0]
+ assert len(p.tokens) == 1
+ outer = p.tokens[0]
+ assert outer.tokens[0].value == 'BEGIN'
+ assert outer.tokens[-1].value == 'END'
+ inner = outer.tokens[4]
+ assert inner.tokens[0].value == 'BEGIN'
+ assert inner.tokens[-1].value == 'END'
+ assert isinstance(inner, sql.Begin)
+
+
+def test_aliased_column_without_as():
+ p = sqlparse.parse('foo bar')[0].tokens
+ assert len(p) == 1
+ assert p[0].get_real_name() == 'foo'
+ assert p[0].get_alias() == 'bar'
+
+ p = sqlparse.parse('foo.bar baz')[0].tokens[0]
+ assert p.get_parent_name() == 'foo'
+ assert p.get_real_name() == 'bar'
+ assert p.get_alias() == 'baz'
+
+
+def test_qualified_function():
+ p = sqlparse.parse('foo()')[0].tokens[0]
+ assert p.get_parent_name() is None
+ assert p.get_real_name() == 'foo'
+
+ p = sqlparse.parse('foo.bar()')[0].tokens[0]
+ assert p.get_parent_name() == 'foo'
+ assert p.get_real_name() == 'bar'
+
+
+def test_aliased_function_without_as():
+ p = sqlparse.parse('foo() bar')[0].tokens[0]
+ assert p.get_parent_name() is None
+ assert p.get_real_name() == 'foo'
+ assert p.get_alias() == 'bar'
+
+ p = sqlparse.parse('foo.bar() baz')[0].tokens[0]
+ assert p.get_parent_name() == 'foo'
+ assert p.get_real_name() == 'bar'
+ assert p.get_alias() == 'baz'
+
+
+def test_aliased_literal_without_as():
+ p = sqlparse.parse('1 foo')[0].tokens
+ assert len(p) == 1
+ assert p[0].get_alias() == 'foo'
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/tests/test_parse.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/tests/test_parse.py b/shell/ext-py/sqlparse-0.1.19/tests/test_parse.py
new file mode 100644
index 0000000..6c9d6a6
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/tests/test_parse.py
@@ -0,0 +1,305 @@
+# -*- coding: utf-8 -*-
+
+"""Tests sqlparse function."""
+
+import pytest
+
+from tests.utils import TestCaseBase
+
+import sqlparse
+import sqlparse.sql
+
+from sqlparse import tokens as T
+
+
+class SQLParseTest(TestCaseBase):
+ """Tests sqlparse.parse()."""
+
+ def test_tokenize(self):
+ sql = 'select * from foo;'
+ stmts = sqlparse.parse(sql)
+ self.assertEqual(len(stmts), 1)
+ self.assertEqual(str(stmts[0]), sql)
+
+ def test_multistatement(self):
+ sql1 = 'select * from foo;'
+ sql2 = 'select * from bar;'
+ stmts = sqlparse.parse(sql1 + sql2)
+ self.assertEqual(len(stmts), 2)
+ self.assertEqual(str(stmts[0]), sql1)
+ self.assertEqual(str(stmts[1]), sql2)
+
+ def test_newlines(self):
+ sql = u'select\n*from foo;'
+ p = sqlparse.parse(sql)[0]
+ self.assertEqual(unicode(p), sql)
+ sql = u'select\r\n*from foo'
+ p = sqlparse.parse(sql)[0]
+ self.assertEqual(unicode(p), sql)
+ sql = u'select\r*from foo'
+ p = sqlparse.parse(sql)[0]
+ self.assertEqual(unicode(p), sql)
+ sql = u'select\r\n*from foo\n'
+ p = sqlparse.parse(sql)[0]
+ self.assertEqual(unicode(p), sql)
+
+ def test_within(self):
+ sql = 'foo(col1, col2)'
+ p = sqlparse.parse(sql)[0]
+ col1 = p.tokens[0].tokens[1].tokens[1].tokens[0]
+ self.assert_(col1.within(sqlparse.sql.Function))
+
+ def test_child_of(self):
+ sql = '(col1, col2)'
+ p = sqlparse.parse(sql)[0]
+ self.assert_(p.tokens[0].tokens[1].is_child_of(p.tokens[0]))
+ sql = 'select foo'
+ p = sqlparse.parse(sql)[0]
+ self.assert_(not p.tokens[2].is_child_of(p.tokens[0]))
+ self.assert_(p.tokens[2].is_child_of(p))
+
+ def test_has_ancestor(self):
+ sql = 'foo or (bar, baz)'
+ p = sqlparse.parse(sql)[0]
+ baz = p.tokens[-1].tokens[1].tokens[-1]
+ self.assert_(baz.has_ancestor(p.tokens[-1].tokens[1]))
+ self.assert_(baz.has_ancestor(p.tokens[-1]))
+ self.assert_(baz.has_ancestor(p))
+
+ def test_float(self):
+ t = sqlparse.parse('.5')[0].tokens
+ self.assertEqual(len(t), 1)
+ self.assert_(t[0].ttype is sqlparse.tokens.Number.Float)
+ t = sqlparse.parse('.51')[0].tokens
+ self.assertEqual(len(t), 1)
+ self.assert_(t[0].ttype is sqlparse.tokens.Number.Float)
+ t = sqlparse.parse('1.5')[0].tokens
+ self.assertEqual(len(t), 1)
+ self.assert_(t[0].ttype is sqlparse.tokens.Number.Float)
+ t = sqlparse.parse('12.5')[0].tokens
+ self.assertEqual(len(t), 1)
+ self.assert_(t[0].ttype is sqlparse.tokens.Number.Float)
+
+ def test_placeholder(self):
+ def _get_tokens(sql):
+ return sqlparse.parse(sql)[0].tokens[-1].tokens
+ t = _get_tokens('select * from foo where user = ?')
+ self.assert_(t[-1].ttype is sqlparse.tokens.Name.Placeholder)
+ self.assertEqual(t[-1].value, '?')
+ t = _get_tokens('select * from foo where user = :1')
+ self.assert_(t[-1].ttype is sqlparse.tokens.Name.Placeholder)
+ self.assertEqual(t[-1].value, ':1')
+ t = _get_tokens('select * from foo where user = :name')
+ self.assert_(t[-1].ttype is sqlparse.tokens.Name.Placeholder)
+ self.assertEqual(t[-1].value, ':name')
+ t = _get_tokens('select * from foo where user = %s')
+ self.assert_(t[-1].ttype is sqlparse.tokens.Name.Placeholder)
+ self.assertEqual(t[-1].value, '%s')
+ t = _get_tokens('select * from foo where user = $a')
+ self.assert_(t[-1].ttype is sqlparse.tokens.Name.Placeholder)
+ self.assertEqual(t[-1].value, '$a')
+
+ def test_modulo_not_placeholder(self):
+ tokens = list(sqlparse.lexer.tokenize('x %3'))
+ self.assertEqual(tokens[2][0], sqlparse.tokens.Operator)
+
+ def test_access_symbol(self): # see issue27
+ t = sqlparse.parse('select a.[foo bar] as foo')[0].tokens
+ self.assert_(isinstance(t[-1], sqlparse.sql.Identifier))
+ self.assertEqual(t[-1].get_name(), 'foo')
+ self.assertEqual(t[-1].get_real_name(), '[foo bar]')
+ self.assertEqual(t[-1].get_parent_name(), 'a')
+
+ def test_square_brackets_notation_isnt_too_greedy(self): # see issue153
+ t = sqlparse.parse('[foo], [bar]')[0].tokens
+ self.assert_(isinstance(t[0], sqlparse.sql.IdentifierList))
+ self.assertEqual(len(t[0].tokens), 4)
+ self.assertEqual(t[0].tokens[0].get_real_name(), '[foo]')
+ self.assertEqual(t[0].tokens[-1].get_real_name(), '[bar]')
+
+ def test_keyword_like_identifier(self): # see issue47
+ t = sqlparse.parse('foo.key')[0].tokens
+ self.assertEqual(len(t), 1)
+ self.assert_(isinstance(t[0], sqlparse.sql.Identifier))
+
+ def test_function_parameter(self): # see issue94
+ t = sqlparse.parse('abs(some_col)')[0].tokens[0].get_parameters()
+ self.assertEqual(len(t), 1)
+ self.assert_(isinstance(t[0], sqlparse.sql.Identifier))
+
+ def test_function_param_single_literal(self):
+ t = sqlparse.parse('foo(5)')[0].tokens[0].get_parameters()
+ self.assertEqual(len(t), 1)
+ self.assert_(t[0].ttype is T.Number.Integer)
+
+ def test_nested_function(self):
+ t = sqlparse.parse('foo(bar(5))')[0].tokens[0].get_parameters()
+ self.assertEqual(len(t), 1)
+ self.assert_(type(t[0]) is sqlparse.sql.Function)
+
+
+def test_quoted_identifier():
+ t = sqlparse.parse('select x.y as "z" from foo')[0].tokens
+ assert isinstance(t[2], sqlparse.sql.Identifier)
+ assert t[2].get_name() == 'z'
+ assert t[2].get_real_name() == 'y'
+
+
+@pytest.mark.parametrize('name', [
+ 'foo',
+ '_foo',
+])
+def test_valid_identifier_names(name): # issue175
+ t = sqlparse.parse(name)[0].tokens
+ assert isinstance(t[0], sqlparse.sql.Identifier)
+
+
+def test_psql_quotation_marks(): # issue83
+ # regression: make sure plain $$ work
+ t = sqlparse.split("""
+ CREATE OR REPLACE FUNCTION testfunc1(integer) RETURNS integer AS $$
+ ....
+ $$ LANGUAGE plpgsql;
+ CREATE OR REPLACE FUNCTION testfunc2(integer) RETURNS integer AS $$
+ ....
+ $$ LANGUAGE plpgsql;""")
+ assert len(t) == 2
+ # make sure $SOMETHING$ works too
+ t = sqlparse.split("""
+ CREATE OR REPLACE FUNCTION testfunc1(integer) RETURNS integer AS $PROC_1$
+ ....
+ $PROC_1$ LANGUAGE plpgsql;
+ CREATE OR REPLACE FUNCTION testfunc2(integer) RETURNS integer AS $PROC_2$
+ ....
+ $PROC_2$ LANGUAGE plpgsql;""")
+ assert len(t) == 2
+
+
+def test_double_precision_is_builtin():
+ sql = 'DOUBLE PRECISION'
+ t = sqlparse.parse(sql)[0].tokens
+ assert (len(t) == 1
+ and t[0].ttype == sqlparse.tokens.Name.Builtin
+ and t[0].value == 'DOUBLE PRECISION')
+
+
+@pytest.mark.parametrize('ph', ['?', ':1', ':foo', '%s', '%(foo)s'])
+def test_placeholder(ph):
+ p = sqlparse.parse(ph)[0].tokens
+ assert len(p) == 1
+ assert p[0].ttype is T.Name.Placeholder
+
+
+@pytest.mark.parametrize('num', ['6.67428E-8', '1.988e33', '1e-12'])
+def test_scientific_numbers(num):
+ p = sqlparse.parse(num)[0].tokens
+ assert len(p) == 1
+ assert p[0].ttype is T.Number.Float
+
+
+def test_single_quotes_are_strings():
+ p = sqlparse.parse("'foo'")[0].tokens
+ assert len(p) == 1
+ assert p[0].ttype is T.String.Single
+
+
+def test_double_quotes_are_identifiers():
+ p = sqlparse.parse('"foo"')[0].tokens
+ assert len(p) == 1
+ assert isinstance(p[0], sqlparse.sql.Identifier)
+
+
+def test_single_quotes_with_linebreaks(): # issue118
+ p = sqlparse.parse("'f\nf'")[0].tokens
+ assert len(p) == 1
+ assert p[0].ttype is T.String.Single
+
+
+def test_sqlite_identifiers():
+ # Make sure we still parse sqlite style escapes
+ p = sqlparse.parse('[col1],[col2]')[0].tokens
+ assert (len(p) == 1
+ and isinstance(p[0], sqlparse.sql.IdentifierList)
+ and [id.get_name() for id in p[0].get_identifiers()]
+ == ['[col1]', '[col2]'])
+
+ p = sqlparse.parse('[col1]+[col2]')[0]
+ types = [tok.ttype for tok in p.flatten()]
+ assert types == [T.Name, T.Operator, T.Name]
+
+
+def test_simple_1d_array_index():
+ p = sqlparse.parse('col[1]')[0].tokens
+ assert len(p) == 1
+ assert p[0].get_name() == 'col'
+ indices = list(p[0].get_array_indices())
+ assert (len(indices) == 1 # 1-dimensional index
+ and len(indices[0]) == 1 # index is single token
+ and indices[0][0].value == '1')
+
+
+def test_2d_array_index():
+ p = sqlparse.parse('col[x][(y+1)*2]')[0].tokens
+ assert len(p) == 1
+ assert p[0].get_name() == 'col'
+ assert len(list(p[0].get_array_indices())) == 2 # 2-dimensional index
+
+
+def test_array_index_function_result():
+ p = sqlparse.parse('somefunc()[1]')[0].tokens
+ assert len(p) == 1
+ assert len(list(p[0].get_array_indices())) == 1
+
+
+def test_schema_qualified_array_index():
+ p = sqlparse.parse('schem.col[1]')[0].tokens
+ assert len(p) == 1
+ assert p[0].get_parent_name() == 'schem'
+ assert p[0].get_name() == 'col'
+ assert list(p[0].get_array_indices())[0][0].value == '1'
+
+
+def test_aliased_array_index():
+ p = sqlparse.parse('col[1] x')[0].tokens
+ assert len(p) == 1
+ assert p[0].get_alias() == 'x'
+ assert p[0].get_real_name() == 'col'
+ assert list(p[0].get_array_indices())[0][0].value == '1'
+
+
+def test_array_literal():
+ # See issue #176
+ p = sqlparse.parse('ARRAY[%s, %s]')[0]
+ assert len(p.tokens) == 2
+ assert len(list(p.flatten())) == 7
+
+
+def test_typed_array_definition():
+ # array indices aren't grouped with builtins, but make sure we can extract
+ # indentifer names
+ p = sqlparse.parse('x int, y int[], z int')[0]
+ names = [x.get_name() for x in p.get_sublists()
+ if isinstance(x, sqlparse.sql.Identifier)]
+ assert names == ['x', 'y', 'z']
+
+
+@pytest.mark.parametrize('sql', [
+ 'select 1 -- foo',
+ 'select 1 # foo' # see issue178
+])
+def test_single_line_comments(sql):
+ p = sqlparse.parse(sql)[0]
+ assert len(p.tokens) == 5
+ assert p.tokens[-1].ttype == T.Comment.Single
+
+
+@pytest.mark.parametrize('sql', [
+ 'foo',
+ '@foo',
+ '#foo', # see issue192
+ '##foo'
+])
+def test_names_and_special_names(sql):
+ p = sqlparse.parse(sql)[0]
+ assert len(p.tokens) == 1
+ assert isinstance(p.tokens[0], sqlparse.sql.Identifier)
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/tests/test_pipeline.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/tests/test_pipeline.py b/shell/ext-py/sqlparse-0.1.19/tests/test_pipeline.py
new file mode 100644
index 0000000..3442a5b
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/tests/test_pipeline.py
@@ -0,0 +1,70 @@
+import unittest
+
+from sqlparse.filters import ColumnsSelect
+from sqlparse.lexer import tokenize
+from sqlparse.pipeline import Pipeline
+
+
+class Test(unittest.TestCase):
+
+ def setUp(self):
+ self.pipe = Pipeline()
+ self.pipe.append(tokenize)
+ self.pipe.append(ColumnsSelect())
+
+ def test_1(self):
+ sql = """
+ -- type: script
+ -- return: integer
+
+ INCLUDE "Direntry.make.sql";
+
+ INSERT INTO directories(inode)
+ VALUES(:inode)
+ LIMIT 1"""
+ self.assertEqual([], self.pipe(sql))
+
+ def test_2(self):
+ sql = """
+ SELECT child_entry,asdf AS inode, creation
+ FROM links
+ WHERE parent_dir == :parent_dir AND name == :name
+ LIMIT 1"""
+ self.assertEqual([u'child_entry', u'inode', u'creation'],
+ self.pipe(sql))
+
+ def test_3(self):
+ sql = """
+SELECT
+0 AS st_dev,
+0 AS st_uid,
+0 AS st_gid,
+
+dir_entries.type AS st_mode,
+dir_entries.inode AS st_ino,
+COUNT(links.child_entry) AS st_nlink,
+
+:creation AS st_ctime,
+dir_entries.access AS st_atime,
+dir_entries.modification AS st_mtime,
+-- :creation AS st_ctime,
+-- CAST(STRFTIME('%s',dir_entries.access) AS INTEGER) AS st_atime,
+-- CAST(STRFTIME('%s',dir_entries.modification) AS INTEGER) AS st_mtime,
+
+COALESCE(files.size,0) AS st_size, -- Python-FUSE
+COALESCE(files.size,0) AS size -- PyFilesystem
+
+FROM dir_entries
+LEFT JOIN files
+ON dir_entries.inode == files.inode
+LEFT JOIN links
+ON dir_entries.inode == links.child_entry
+
+WHERE dir_entries.inode == :inode
+
+GROUP BY dir_entries.inode
+LIMIT 1"""
+ self.assertEqual([u'st_dev', u'st_uid', u'st_gid', u'st_mode',
+ u'st_ino', u'st_nlink', u'st_ctime',
+ u'st_atime', u'st_mtime', u'st_size', u'size'],
+ self.pipe(sql))
[2/8] impala git commit: IMPALA-6999: Upgrade to sqlparse-0.1.19 for
Impala shell
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/tests/test_regressions.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/tests/test_regressions.py b/shell/ext-py/sqlparse-0.1.19/tests/test_regressions.py
new file mode 100644
index 0000000..a64b400
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/tests/test_regressions.py
@@ -0,0 +1,276 @@
+# -*- coding: utf-8 -*-
+
+import sys
+
+from tests.utils import TestCaseBase, load_file
+
+import sqlparse
+from sqlparse import sql
+from sqlparse import tokens as T
+
+
+class RegressionTests(TestCaseBase):
+
+ def test_issue9(self):
+ # make sure where doesn't consume parenthesis
+ p = sqlparse.parse('(where 1)')[0]
+ self.assert_(isinstance(p, sql.Statement))
+ self.assertEqual(len(p.tokens), 1)
+ self.assert_(isinstance(p.tokens[0], sql.Parenthesis))
+ prt = p.tokens[0]
+ self.assertEqual(len(prt.tokens), 3)
+ self.assertEqual(prt.tokens[0].ttype, T.Punctuation)
+ self.assertEqual(prt.tokens[-1].ttype, T.Punctuation)
+
+ def test_issue13(self):
+ parsed = sqlparse.parse(("select 'one';\n"
+ "select 'two\\'';\n"
+ "select 'three';"))
+ self.assertEqual(len(parsed), 3)
+ self.assertEqual(str(parsed[1]).strip(), "select 'two\\'';")
+
+ def test_issue26(self):
+ # parse stand-alone comments
+ p = sqlparse.parse('--hello')[0]
+ self.assertEqual(len(p.tokens), 1)
+ self.assert_(p.tokens[0].ttype is T.Comment.Single)
+ p = sqlparse.parse('-- hello')[0]
+ self.assertEqual(len(p.tokens), 1)
+ self.assert_(p.tokens[0].ttype is T.Comment.Single)
+ p = sqlparse.parse('--hello\n')[0]
+ self.assertEqual(len(p.tokens), 1)
+ self.assert_(p.tokens[0].ttype is T.Comment.Single)
+ p = sqlparse.parse('--')[0]
+ self.assertEqual(len(p.tokens), 1)
+ self.assert_(p.tokens[0].ttype is T.Comment.Single)
+ p = sqlparse.parse('--\n')[0]
+ self.assertEqual(len(p.tokens), 1)
+ self.assert_(p.tokens[0].ttype is T.Comment.Single)
+
+ def test_issue34(self):
+ t = sqlparse.parse("create")[0].token_first()
+ self.assertEqual(t.match(T.Keyword.DDL, "create"), True)
+ self.assertEqual(t.match(T.Keyword.DDL, "CREATE"), True)
+
+ def test_issue35(self):
+ # missing space before LIMIT
+ sql = sqlparse.format("select * from foo where bar = 1 limit 1",
+ reindent=True)
+ self.ndiffAssertEqual(sql, "\n".join(["select *",
+ "from foo",
+ "where bar = 1 limit 1"]))
+
+ def test_issue38(self):
+ sql = sqlparse.format("SELECT foo; -- comment",
+ strip_comments=True)
+ self.ndiffAssertEqual(sql, "SELECT foo;")
+ sql = sqlparse.format("/* foo */", strip_comments=True)
+ self.ndiffAssertEqual(sql, "")
+
+ def test_issue39(self):
+ p = sqlparse.parse('select user.id from user')[0]
+ self.assertEqual(len(p.tokens), 7)
+ idt = p.tokens[2]
+ self.assertEqual(idt.__class__, sql.Identifier)
+ self.assertEqual(len(idt.tokens), 3)
+ self.assertEqual(idt.tokens[0].match(T.Name, 'user'), True)
+ self.assertEqual(idt.tokens[1].match(T.Punctuation, '.'), True)
+ self.assertEqual(idt.tokens[2].match(T.Name, 'id'), True)
+
+ def test_issue40(self):
+ # make sure identifier lists in subselects are grouped
+ p = sqlparse.parse(('SELECT id, name FROM '
+ '(SELECT id, name FROM bar) as foo'))[0]
+ self.assertEqual(len(p.tokens), 7)
+ self.assertEqual(p.tokens[2].__class__, sql.IdentifierList)
+ self.assertEqual(p.tokens[-1].__class__, sql.Identifier)
+ self.assertEqual(p.tokens[-1].get_name(), u'foo')
+ sp = p.tokens[-1].tokens[0]
+ self.assertEqual(sp.tokens[3].__class__, sql.IdentifierList)
+ # make sure that formatting works as expected
+ self.ndiffAssertEqual(
+ sqlparse.format(('SELECT id, name FROM '
+ '(SELECT id, name FROM bar)'),
+ reindent=True),
+ ('SELECT id,\n'
+ ' name\n'
+ 'FROM\n'
+ ' (SELECT id,\n'
+ ' name\n'
+ ' FROM bar)'))
+ self.ndiffAssertEqual(
+ sqlparse.format(('SELECT id, name FROM '
+ '(SELECT id, name FROM bar) as foo'),
+ reindent=True),
+ ('SELECT id,\n'
+ ' name\n'
+ 'FROM\n'
+ ' (SELECT id,\n'
+ ' name\n'
+ ' FROM bar) as foo'))
+
+
+def test_issue78():
+ # the bug author provided this nice examples, let's use them!
+ def _get_identifier(sql):
+ p = sqlparse.parse(sql)[0]
+ return p.tokens[2]
+ results = (('get_name', 'z'),
+ ('get_real_name', 'y'),
+ ('get_parent_name', 'x'),
+ ('get_alias', 'z'),
+ ('get_typecast', 'text'))
+ variants = (
+ 'select x.y::text as z from foo',
+ 'select x.y::text as "z" from foo',
+ 'select x."y"::text as z from foo',
+ 'select x."y"::text as "z" from foo',
+ 'select "x".y::text as z from foo',
+ 'select "x".y::text as "z" from foo',
+ 'select "x"."y"::text as z from foo',
+ 'select "x"."y"::text as "z" from foo',
+ )
+ for variant in variants:
+ i = _get_identifier(variant)
+ assert isinstance(i, sql.Identifier)
+ for func_name, result in results:
+ func = getattr(i, func_name)
+ assert func() == result
+
+
+def test_issue83():
+ sql = """
+CREATE OR REPLACE FUNCTION func_a(text)
+ RETURNS boolean LANGUAGE plpgsql STRICT IMMUTABLE AS
+$_$
+BEGIN
+ ...
+END;
+$_$;
+
+CREATE OR REPLACE FUNCTION func_b(text)
+ RETURNS boolean LANGUAGE plpgsql STRICT IMMUTABLE AS
+$_$
+BEGIN
+ ...
+END;
+$_$;
+
+ALTER TABLE..... ;"""
+ t = sqlparse.split(sql)
+ assert len(t) == 3
+
+
+def test_comment_encoding_when_reindent():
+ # There was an UnicodeEncodeError in the reindent filter that
+ # casted every comment followed by a keyword to str.
+ sql = u'select foo -- Comment containing Ümläuts\nfrom bar'
+ formatted = sqlparse.format(sql, reindent=True)
+ assert formatted == sql
+
+
+def test_parse_sql_with_binary():
+ # See https://github.com/andialbrecht/sqlparse/pull/88
+ digest = '\x82|\xcb\x0e\xea\x8aplL4\xa1h\x91\xf8N{'
+ sql = 'select * from foo where bar = \'%s\'' % digest
+ formatted = sqlparse.format(sql, reindent=True)
+ tformatted = 'select *\nfrom foo\nwhere bar = \'%s\'' % digest
+ if sys.version_info < (3,):
+ tformatted = tformatted.decode('unicode-escape')
+ assert formatted == tformatted
+
+
+def test_dont_alias_keywords():
+ # The _group_left_right function had a bug where the check for the
+ # left side wasn't handled correctly. In one case this resulted in
+ # a keyword turning into an identifier.
+ p = sqlparse.parse('FROM AS foo')[0]
+ assert len(p.tokens) == 5
+ assert p.tokens[0].ttype is T.Keyword
+ assert p.tokens[2].ttype is T.Keyword
+
+
+def test_format_accepts_encoding(): # issue20
+ sql = load_file('test_cp1251.sql', 'cp1251')
+ formatted = sqlparse.format(sql, reindent=True, encoding='cp1251')
+ if sys.version_info < (3,):
+ tformatted = u'insert into foo\nvalues (1); -- Песня про надежду\n'
+ else:
+ tformatted = 'insert into foo\nvalues (1); -- Песня про надежду\n'
+ assert formatted == tformatted
+
+
+def test_issue90():
+ sql = ('UPDATE "gallery_photo" SET "owner_id" = 4018, "deleted_at" = NULL,'
+ ' "width" = NULL, "height" = NULL, "rating_votes" = 0,'
+ ' "rating_score" = 0, "thumbnail_width" = NULL,'
+ ' "thumbnail_height" = NULL, "price" = 1, "description" = NULL')
+ formatted = sqlparse.format(sql, reindent=True)
+ tformatted = '\n'.join(['UPDATE "gallery_photo"',
+ 'SET "owner_id" = 4018,',
+ ' "deleted_at" = NULL,',
+ ' "width" = NULL,',
+ ' "height" = NULL,',
+ ' "rating_votes" = 0,',
+ ' "rating_score" = 0,',
+ ' "thumbnail_width" = NULL,',
+ ' "thumbnail_height" = NULL,',
+ ' "price" = 1,',
+ ' "description" = NULL'])
+ assert formatted == tformatted
+
+
+def test_except_formatting():
+ sql = 'SELECT 1 FROM foo WHERE 2 = 3 EXCEPT SELECT 2 FROM bar WHERE 1 = 2'
+ formatted = sqlparse.format(sql, reindent=True)
+ tformatted = '\n'.join([
+ 'SELECT 1',
+ 'FROM foo',
+ 'WHERE 2 = 3',
+ 'EXCEPT',
+ 'SELECT 2',
+ 'FROM bar',
+ 'WHERE 1 = 2'
+ ])
+ assert formatted == tformatted
+
+
+def test_null_with_as():
+ sql = 'SELECT NULL AS c1, NULL AS c2 FROM t1'
+ formatted = sqlparse.format(sql, reindent=True)
+ tformatted = '\n'.join([
+ 'SELECT NULL AS c1,',
+ ' NULL AS c2',
+ 'FROM t1'
+ ])
+ assert formatted == tformatted
+
+
+def test_issue193_splitting_function():
+ sql = """CREATE FUNCTION a(x VARCHAR(20)) RETURNS VARCHAR(20)
+BEGIN
+ DECLARE y VARCHAR(20);
+ RETURN x;
+END;
+SELECT * FROM a.b;"""
+ splitted = sqlparse.split(sql)
+ assert len(splitted) == 2
+
+def test_issue194_splitting_function():
+ sql = """CREATE FUNCTION a(x VARCHAR(20)) RETURNS VARCHAR(20)
+BEGIN
+ DECLARE y VARCHAR(20);
+ IF (1 = 1) THEN
+ SET x = y;
+ END IF;
+ RETURN x;
+END;
+SELECT * FROM a.b;"""
+ splitted = sqlparse.split(sql)
+ assert len(splitted) == 2
+
+
+def test_issue186_get_type():
+ sql = "-- comment\ninsert into foo"
+ p = sqlparse.parse(sql)[0]
+ assert p.get_type() == 'INSERT'
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/tests/test_split.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/tests/test_split.py b/shell/ext-py/sqlparse-0.1.19/tests/test_split.py
new file mode 100644
index 0000000..54e8d04
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/tests/test_split.py
@@ -0,0 +1,144 @@
+# -*- coding: utf-8 -*-
+
+# Tests splitting functions.
+
+import unittest
+
+from tests.utils import load_file, TestCaseBase
+
+import sqlparse
+
+
+class SQLSplitTest(TestCaseBase):
+ """Tests sqlparse.sqlsplit()."""
+
+ _sql1 = 'select * from foo;'
+ _sql2 = 'select * from bar;'
+
+ def test_split_semicolon(self):
+ sql2 = 'select * from foo where bar = \'foo;bar\';'
+ stmts = sqlparse.parse(''.join([self._sql1, sql2]))
+ self.assertEqual(len(stmts), 2)
+ self.ndiffAssertEqual(unicode(stmts[0]), self._sql1)
+ self.ndiffAssertEqual(unicode(stmts[1]), sql2)
+
+ def test_split_backslash(self):
+ stmts = sqlparse.parse(r"select '\\'; select '\''; select '\\\'';")
+ self.assertEqual(len(stmts), 3)
+
+ def test_create_function(self):
+ sql = load_file('function.sql')
+ stmts = sqlparse.parse(sql)
+ self.assertEqual(len(stmts), 1)
+ self.ndiffAssertEqual(unicode(stmts[0]), sql)
+
+ def test_create_function_psql(self):
+ sql = load_file('function_psql.sql')
+ stmts = sqlparse.parse(sql)
+ self.assertEqual(len(stmts), 1)
+ self.ndiffAssertEqual(unicode(stmts[0]), sql)
+
+ def test_create_function_psql3(self):
+ sql = load_file('function_psql3.sql')
+ stmts = sqlparse.parse(sql)
+ self.assertEqual(len(stmts), 1)
+ self.ndiffAssertEqual(unicode(stmts[0]), sql)
+
+ def test_create_function_psql2(self):
+ sql = load_file('function_psql2.sql')
+ stmts = sqlparse.parse(sql)
+ self.assertEqual(len(stmts), 1)
+ self.ndiffAssertEqual(unicode(stmts[0]), sql)
+
+ def test_dashcomments(self):
+ sql = load_file('dashcomment.sql')
+ stmts = sqlparse.parse(sql)
+ self.assertEqual(len(stmts), 3)
+ self.ndiffAssertEqual(''.join(unicode(q) for q in stmts), sql)
+
+ def test_dashcomments_eol(self):
+ stmts = sqlparse.parse('select foo; -- comment\n')
+ self.assertEqual(len(stmts), 1)
+ stmts = sqlparse.parse('select foo; -- comment\r')
+ self.assertEqual(len(stmts), 1)
+ stmts = sqlparse.parse('select foo; -- comment\r\n')
+ self.assertEqual(len(stmts), 1)
+ stmts = sqlparse.parse('select foo; -- comment')
+ self.assertEqual(len(stmts), 1)
+
+ def test_begintag(self):
+ sql = load_file('begintag.sql')
+ stmts = sqlparse.parse(sql)
+ self.assertEqual(len(stmts), 3)
+ self.ndiffAssertEqual(''.join(unicode(q) for q in stmts), sql)
+
+ def test_begintag_2(self):
+ sql = load_file('begintag_2.sql')
+ stmts = sqlparse.parse(sql)
+ self.assertEqual(len(stmts), 1)
+ self.ndiffAssertEqual(''.join(unicode(q) for q in stmts), sql)
+
+ def test_dropif(self):
+ sql = 'DROP TABLE IF EXISTS FOO;\n\nSELECT * FROM BAR;'
+ stmts = sqlparse.parse(sql)
+ self.assertEqual(len(stmts), 2)
+ self.ndiffAssertEqual(''.join(unicode(q) for q in stmts), sql)
+
+ def test_comment_with_umlaut(self):
+ sql = (u'select * from foo;\n'
+ u'-- Testing an umlaut: ä\n'
+ u'select * from bar;')
+ stmts = sqlparse.parse(sql)
+ self.assertEqual(len(stmts), 2)
+ self.ndiffAssertEqual(''.join(unicode(q) for q in stmts), sql)
+
+ def test_comment_end_of_line(self):
+ sql = ('select * from foo; -- foo\n'
+ 'select * from bar;')
+ stmts = sqlparse.parse(sql)
+ self.assertEqual(len(stmts), 2)
+ self.ndiffAssertEqual(''.join(unicode(q) for q in stmts), sql)
+ # make sure the comment belongs to first query
+ self.ndiffAssertEqual(unicode(stmts[0]), 'select * from foo; -- foo\n')
+
+ def test_casewhen(self):
+ sql = ('SELECT case when val = 1 then 2 else null end as foo;\n'
+ 'comment on table actor is \'The actor table.\';')
+ stmts = sqlparse.split(sql)
+ self.assertEqual(len(stmts), 2)
+
+ def test_cursor_declare(self):
+ sql = ('DECLARE CURSOR "foo" AS SELECT 1;\n'
+ 'SELECT 2;')
+ stmts = sqlparse.split(sql)
+ self.assertEqual(len(stmts), 2)
+
+ def test_if_function(self): # see issue 33
+ # don't let IF as a function confuse the splitter
+ sql = ('CREATE TEMPORARY TABLE tmp '
+ 'SELECT IF(a=1, a, b) AS o FROM one; '
+ 'SELECT t FROM two')
+ stmts = sqlparse.split(sql)
+ self.assertEqual(len(stmts), 2)
+
+ def test_split_stream(self):
+ import types
+ from cStringIO import StringIO
+
+ stream = StringIO("SELECT 1; SELECT 2;")
+ stmts = sqlparse.parsestream(stream)
+ self.assertEqual(type(stmts), types.GeneratorType)
+ self.assertEqual(len(list(stmts)), 2)
+
+ def test_encoding_parsestream(self):
+ from cStringIO import StringIO
+ stream = StringIO("SELECT 1; SELECT 2;")
+ stmts = list(sqlparse.parsestream(stream))
+ self.assertEqual(type(stmts[0].tokens[0].value), unicode)
+
+
+def test_split_simple():
+ stmts = sqlparse.split('select * from foo; select * from bar;')
+ assert len(stmts) == 2
+ assert stmts[0] == 'select * from foo;'
+ assert stmts[1] == 'select * from bar;'
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/tests/test_tokenize.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/tests/test_tokenize.py b/shell/ext-py/sqlparse-0.1.19/tests/test_tokenize.py
new file mode 100644
index 0000000..0b23fa8
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/tests/test_tokenize.py
@@ -0,0 +1,190 @@
+# -*- coding: utf-8 -*-
+
+import sys
+import types
+import unittest
+
+import pytest
+
+import sqlparse
+from sqlparse import lexer
+from sqlparse import sql
+from sqlparse.tokens import *
+
+
+class TestTokenize(unittest.TestCase):
+
+ def test_simple(self):
+ s = 'select * from foo;'
+ stream = lexer.tokenize(s)
+ self.assert_(isinstance(stream, types.GeneratorType))
+ tokens = list(stream)
+ self.assertEqual(len(tokens), 8)
+ self.assertEqual(len(tokens[0]), 2)
+ self.assertEqual(tokens[0], (Keyword.DML, u'select'))
+ self.assertEqual(tokens[-1], (Punctuation, u';'))
+
+ def test_backticks(self):
+ s = '`foo`.`bar`'
+ tokens = list(lexer.tokenize(s))
+ self.assertEqual(len(tokens), 3)
+ self.assertEqual(tokens[0], (Name, u'`foo`'))
+
+ def test_linebreaks(self): # issue1
+ s = 'foo\nbar\n'
+ tokens = lexer.tokenize(s)
+ self.assertEqual(''.join(str(x[1]) for x in tokens), s)
+ s = 'foo\rbar\r'
+ tokens = lexer.tokenize(s)
+ self.assertEqual(''.join(str(x[1]) for x in tokens), s)
+ s = 'foo\r\nbar\r\n'
+ tokens = lexer.tokenize(s)
+ self.assertEqual(''.join(str(x[1]) for x in tokens), s)
+ s = 'foo\r\nbar\n'
+ tokens = lexer.tokenize(s)
+ self.assertEqual(''.join(str(x[1]) for x in tokens), s)
+
+ def test_inline_keywords(self): # issue 7
+ s = "create created_foo"
+ tokens = list(lexer.tokenize(s))
+ self.assertEqual(len(tokens), 3)
+ self.assertEqual(tokens[0][0], Keyword.DDL)
+ self.assertEqual(tokens[2][0], Name)
+ self.assertEqual(tokens[2][1], u'created_foo')
+ s = "enddate"
+ tokens = list(lexer.tokenize(s))
+ self.assertEqual(len(tokens), 1)
+ self.assertEqual(tokens[0][0], Name)
+ s = "join_col"
+ tokens = list(lexer.tokenize(s))
+ self.assertEqual(len(tokens), 1)
+ self.assertEqual(tokens[0][0], Name)
+ s = "left join_col"
+ tokens = list(lexer.tokenize(s))
+ self.assertEqual(len(tokens), 3)
+ self.assertEqual(tokens[2][0], Name)
+ self.assertEqual(tokens[2][1], 'join_col')
+
+ def test_negative_numbers(self):
+ s = "values(-1)"
+ tokens = list(lexer.tokenize(s))
+ self.assertEqual(len(tokens), 4)
+ self.assertEqual(tokens[2][0], Number.Integer)
+ self.assertEqual(tokens[2][1], '-1')
+
+ # Somehow this test fails on Python 3.2
+ @pytest.mark.skipif('sys.version_info >= (3,0)')
+ def test_tab_expansion(self):
+ s = "\t"
+ lex = lexer.Lexer()
+ lex.tabsize = 5
+ tokens = list(lex.get_tokens(s))
+ self.assertEqual(tokens[0][1], " " * 5)
+
+
+class TestToken(unittest.TestCase):
+
+ def test_str(self):
+ token = sql.Token(None, 'FoO')
+ self.assertEqual(str(token), 'FoO')
+
+ def test_repr(self):
+ token = sql.Token(Keyword, 'foo')
+ tst = "<Keyword 'foo' at 0x"
+ self.assertEqual(repr(token)[:len(tst)], tst)
+ token = sql.Token(Keyword, '1234567890')
+ tst = "<Keyword '123456...' at 0x"
+ self.assertEqual(repr(token)[:len(tst)], tst)
+
+ def test_flatten(self):
+ token = sql.Token(Keyword, 'foo')
+ gen = token.flatten()
+ self.assertEqual(type(gen), types.GeneratorType)
+ lgen = list(gen)
+ self.assertEqual(lgen, [token])
+
+
+class TestTokenList(unittest.TestCase):
+
+ def test_repr(self):
+ p = sqlparse.parse('foo, bar, baz')[0]
+ tst = "<IdentifierList 'foo, b...' at 0x"
+ self.assertEqual(repr(p.tokens[0])[:len(tst)], tst)
+
+ def test_token_first(self):
+ p = sqlparse.parse(' select foo')[0]
+ first = p.token_first()
+ self.assertEqual(first.value, 'select')
+ self.assertEqual(p.token_first(ignore_whitespace=False).value, ' ')
+ self.assertEqual(sql.TokenList([]).token_first(), None)
+
+ def test_token_matching(self):
+ t1 = sql.Token(Keyword, 'foo')
+ t2 = sql.Token(Punctuation, ',')
+ x = sql.TokenList([t1, t2])
+ self.assertEqual(x.token_matching(0, [lambda t: t.ttype is Keyword]),
+ t1)
+ self.assertEqual(x.token_matching(0,
+ [lambda t: t.ttype is Punctuation]),
+ t2)
+ self.assertEqual(x.token_matching(1, [lambda t: t.ttype is Keyword]),
+ None)
+
+
+class TestStream(unittest.TestCase):
+ def test_simple(self):
+ from cStringIO import StringIO
+
+ stream = StringIO("SELECT 1; SELECT 2;")
+ lex = lexer.Lexer()
+
+ tokens = lex.get_tokens(stream)
+ self.assertEqual(len(list(tokens)), 9)
+
+ stream.seek(0)
+ lex.bufsize = 4
+ tokens = list(lex.get_tokens(stream))
+ self.assertEqual(len(tokens), 9)
+
+ stream.seek(0)
+ lex.bufsize = len(stream.getvalue())
+ tokens = list(lex.get_tokens(stream))
+ self.assertEqual(len(tokens), 9)
+
+ def test_error(self):
+ from cStringIO import StringIO
+
+ stream = StringIO("FOOBAR{")
+
+ lex = lexer.Lexer()
+ lex.bufsize = 4
+ tokens = list(lex.get_tokens(stream))
+ self.assertEqual(len(tokens), 2)
+ self.assertEqual(tokens[1][0], Error)
+
+
+@pytest.mark.parametrize('expr', ['JOIN', 'LEFT JOIN', 'LEFT OUTER JOIN',
+ 'FULL OUTER JOIN', 'NATURAL JOIN',
+ 'CROSS JOIN', 'STRAIGHT JOIN',
+ 'INNER JOIN', 'LEFT INNER JOIN'])
+def test_parse_join(expr):
+ p = sqlparse.parse('%s foo' % expr)[0]
+ assert len(p.tokens) == 3
+ assert p.tokens[0].ttype is Keyword
+
+
+def test_parse_endifloop():
+ p = sqlparse.parse('END IF')[0]
+ assert len(p.tokens) == 1
+ assert p.tokens[0].ttype is Keyword
+ p = sqlparse.parse('END IF')[0]
+ assert len(p.tokens) == 1
+ p = sqlparse.parse('END\t\nIF')[0]
+ assert len(p.tokens) == 1
+ assert p.tokens[0].ttype is Keyword
+ p = sqlparse.parse('END LOOP')[0]
+ assert len(p.tokens) == 1
+ assert p.tokens[0].ttype is Keyword
+ p = sqlparse.parse('END LOOP')[0]
+ assert len(p.tokens) == 1
+ assert p.tokens[0].ttype is Keyword
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/tests/utils.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/tests/utils.py b/shell/ext-py/sqlparse-0.1.19/tests/utils.py
new file mode 100644
index 0000000..9eb46bf
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/tests/utils.py
@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+
+"""Helpers for testing."""
+
+import codecs
+import difflib
+import os
+import unittest
+from StringIO import StringIO
+
+import sqlparse.utils
+
+NL = '\n'
+DIR_PATH = os.path.abspath(os.path.dirname(__file__))
+PARENT_DIR = os.path.dirname(DIR_PATH)
+FILES_DIR = os.path.join(DIR_PATH, 'files')
+
+
+def load_file(filename, encoding='utf-8'):
+ """Opens filename with encoding and return it's contents."""
+ f = codecs.open(os.path.join(FILES_DIR, filename), 'r', encoding)
+ data = f.read()
+ f.close()
+ return data
+
+
+class TestCaseBase(unittest.TestCase):
+ """Base class for test cases with some additional checks."""
+
+ # Adopted from Python's tests.
+ def ndiffAssertEqual(self, first, second):
+ """Like failUnlessEqual except use ndiff for readable output."""
+ if first != second:
+ sfirst = unicode(first)
+ ssecond = unicode(second)
+ # Using the built-in .splitlines() method here will cause incorrect
+ # results when splitting statements that have quoted CR/CR+LF
+ # characters.
+ sfirst = sqlparse.utils.split_unquoted_newlines(sfirst)
+ ssecond = sqlparse.utils.split_unquoted_newlines(ssecond)
+ diff = difflib.ndiff(sfirst, ssecond)
+ fp = StringIO()
+ fp.write(NL)
+ fp.write(NL.join(diff))
+ print fp.getvalue()
+ raise self.failureException, fp.getvalue()
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/tox.ini
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/tox.ini b/shell/ext-py/sqlparse-0.1.19/tox.ini
new file mode 100644
index 0000000..e797ca9
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/tox.ini
@@ -0,0 +1,37 @@
+[tox]
+envlist=py26,py27,py32,py33,py34,pypy
+
+[testenv]
+deps=
+ pytest
+ pytest-cov
+commands=
+ sqlformat --version # Sanity check.
+ py.test --cov=sqlparse/ tests
+
+[testenv:py32]
+changedir={envdir}
+commands=
+ sqlformat --version # Sanity check.
+ rm -rf tests/
+ cp -r {toxinidir}/tests/ tests/
+ 2to3 -w --no-diffs -n tests/
+ py.test --cov={envdir}/lib/python3.2/site-packages/sqlparse/ tests
+
+[testenv:py33]
+changedir={envdir}
+commands=
+ sqlformat --version # Sanity check.
+ rm -rf tests/
+ cp -r {toxinidir}/tests/ tests/
+ 2to3 -w --no-diffs -n tests/
+ py.test --cov={envdir}/lib/python3.3/site-packages/sqlparse/ tests
+
+[testenv:py34]
+changedir={envdir}
+commands=
+ sqlformat --version # Sanity check.
+ rm -rf tests/
+ cp -r {toxinidir}/tests/ tests/
+ 2to3 -w --no-diffs -n tests/
+ py.test --cov={envdir}/lib/python3.4/site-packages/sqlparse/ tests
[6/8] impala git commit: IMPALA-6999: Upgrade to sqlparse-0.1.19 for
Impala shell
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/sqlparse/utils.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/sqlparse/utils.py b/shell/ext-py/sqlparse-0.1.14/sqlparse/utils.py
deleted file mode 100644
index 3a49ac2..0000000
--- a/shell/ext-py/sqlparse-0.1.14/sqlparse/utils.py
+++ /dev/null
@@ -1,137 +0,0 @@
-'''
-Created on 17/05/2012
-
-@author: piranna
-'''
-
-import re
-
-try:
- from collections import OrderedDict
-except ImportError:
- OrderedDict = None
-
-
-if OrderedDict:
- class Cache(OrderedDict):
- """Cache with LRU algorithm using an OrderedDict as basis
- """
- def __init__(self, maxsize=100):
- OrderedDict.__init__(self)
-
- self._maxsize = maxsize
-
- def __getitem__(self, key, *args, **kwargs):
- # Get the key and remove it from the cache, or raise KeyError
- value = OrderedDict.__getitem__(self, key)
- del self[key]
-
- # Insert the (key, value) pair on the front of the cache
- OrderedDict.__setitem__(self, key, value)
-
- # Return the value from the cache
- return value
-
- def __setitem__(self, key, value, *args, **kwargs):
- # Key was inserted before, remove it so we put it at front later
- if key in self:
- del self[key]
-
- # Too much items on the cache, remove the least recent used
- elif len(self) >= self._maxsize:
- self.popitem(False)
-
- # Insert the (key, value) pair on the front of the cache
- OrderedDict.__setitem__(self, key, value, *args, **kwargs)
-
-else:
- class Cache(dict):
- """Cache that reset when gets full
- """
- def __init__(self, maxsize=100):
- dict.__init__(self)
-
- self._maxsize = maxsize
-
- def __setitem__(self, key, value, *args, **kwargs):
- # Reset the cache if we have too much cached entries and start over
- if len(self) >= self._maxsize:
- self.clear()
-
- # Insert the (key, value) pair on the front of the cache
- dict.__setitem__(self, key, value, *args, **kwargs)
-
-
-def memoize_generator(func):
- """Memoize decorator for generators
-
- Store `func` results in a cache according to their arguments as 'memoize'
- does but instead this works on decorators instead of regular functions.
- Obviusly, this is only useful if the generator will always return the same
- values for each specific parameters...
- """
- cache = Cache()
-
- def wrapped_func(*args, **kwargs):
-# params = (args, kwargs)
- params = (args, tuple(sorted(kwargs.items())))
-
- # Look if cached
- try:
- cached = cache[params]
-
- # Not cached, exec and store it
- except KeyError:
- cached = []
-
- for item in func(*args, **kwargs):
- cached.append(item)
- yield item
-
- cache[params] = cached
-
- # Cached, yield its items
- else:
- for item in cached:
- yield item
-
- return wrapped_func
-
-
-# This regular expression replaces the home-cooked parser that was here before.
-# It is much faster, but requires an extra post-processing step to get the
-# desired results (that are compatible with what you would expect from the
-# str.splitlines() method).
-#
-# It matches groups of characters: newlines, quoted strings, or unquoted text,
-# and splits on that basis. The post-processing step puts those back together
-# into the actual lines of SQL.
-SPLIT_REGEX = re.compile(r"""
-(
- (?: # Start of non-capturing group
- (?:\r\n|\r|\n) | # Match any single newline, or
- [^\r\n'"]+ | # Match any character series without quotes or
- # newlines, or
- "(?:[^"\\]|\\.)*" | # Match double-quoted strings, or
- '(?:[^'\\]|\\.)*' # Match single quoted strings
- )
-)
-""", re.VERBOSE)
-
-LINE_MATCH = re.compile(r'(\r\n|\r|\n)')
-
-def split_unquoted_newlines(text):
- """Split a string on all unquoted newlines.
-
- Unlike str.splitlines(), this will ignore CR/LF/CR+LF if the requisite
- character is inside of a string."""
- lines = SPLIT_REGEX.split(text)
- outputlines = ['']
- for line in lines:
- if not line:
- continue
- elif LINE_MATCH.match(line):
- outputlines.append('')
- else:
- outputlines[-1] += line
- return outputlines
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/tests/__init__.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/tests/__init__.py b/shell/ext-py/sqlparse-0.1.14/tests/__init__.py
deleted file mode 100644
index e69de29..0000000
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/tests/files/_Make_DirEntry.sql
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/tests/files/_Make_DirEntry.sql b/shell/ext-py/sqlparse-0.1.14/tests/files/_Make_DirEntry.sql
deleted file mode 100644
index e877bf1..0000000
--- a/shell/ext-py/sqlparse-0.1.14/tests/files/_Make_DirEntry.sql
+++ /dev/null
@@ -1,6 +0,0 @@
--- Make a new dir entry
--- and return its inode
-
-
-INSERT INTO dir_entries(type)
- VALUES(:type)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/tests/files/begintag.sql
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/tests/files/begintag.sql b/shell/ext-py/sqlparse-0.1.14/tests/files/begintag.sql
deleted file mode 100644
index 699b365..0000000
--- a/shell/ext-py/sqlparse-0.1.14/tests/files/begintag.sql
+++ /dev/null
@@ -1,4 +0,0 @@
-begin;
-update foo
- set bar = 1;
-commit;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/tests/files/begintag_2.sql
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/tests/files/begintag_2.sql b/shell/ext-py/sqlparse-0.1.14/tests/files/begintag_2.sql
deleted file mode 100644
index 0de26d6..0000000
--- a/shell/ext-py/sqlparse-0.1.14/tests/files/begintag_2.sql
+++ /dev/null
@@ -1,13 +0,0 @@
-CREATE TRIGGER IF NOT EXISTS remove_if_it_was_the_last_file_link
--- Delete the direntry when is removed it's last static link
- AFTER DELETE ON links
- WHEN NOT EXISTS
- (
- SELECT * FROM links
- WHERE child_entry = OLD.child_entry
- LIMIT 1
- )
-BEGIN
- DELETE FROM dir_entries
- WHERE dir_entries.inode = OLD.child_entry;
-END;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/tests/files/dashcomment.sql
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/tests/files/dashcomment.sql b/shell/ext-py/sqlparse-0.1.14/tests/files/dashcomment.sql
deleted file mode 100644
index 0d5ac62..0000000
--- a/shell/ext-py/sqlparse-0.1.14/tests/files/dashcomment.sql
+++ /dev/null
@@ -1,5 +0,0 @@
-select * from user;
---select * from host;
-select * from user;
-select * -- foo;
-from foo;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/tests/files/function.sql
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/tests/files/function.sql b/shell/ext-py/sqlparse-0.1.14/tests/files/function.sql
deleted file mode 100644
index d19227f..0000000
--- a/shell/ext-py/sqlparse-0.1.14/tests/files/function.sql
+++ /dev/null
@@ -1,13 +0,0 @@
-CREATE OR REPLACE FUNCTION foo(
- p_in1 VARCHAR
- , p_in2 INTEGER
-) RETURNS INTEGER AS
-
- DECLARE
- v_foo INTEGER;
- BEGIN
- SELECT *
- FROM foo
- INTO v_foo;
- RETURN v_foo.id;
- END;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/tests/files/function_psql.sql
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/tests/files/function_psql.sql b/shell/ext-py/sqlparse-0.1.14/tests/files/function_psql.sql
deleted file mode 100644
index e485f7a..0000000
--- a/shell/ext-py/sqlparse-0.1.14/tests/files/function_psql.sql
+++ /dev/null
@@ -1,72 +0,0 @@
-CREATE OR REPLACE FUNCTION public.delete_data (
- p_tabelle VARCHAR
- , p_key VARCHAR
- , p_value INTEGER
-) RETURNS INTEGER AS
-$$
-DECLARE
- p_retval INTEGER;
- v_constraint RECORD;
- v_count INTEGER;
- v_data RECORD;
- v_fieldname VARCHAR;
- v_sql VARCHAR;
- v_key VARCHAR;
- v_value INTEGER;
-BEGIN
- v_sql := 'SELECT COUNT(*) FROM ' || p_tabelle || ' WHERE ' || p_key || ' = ' || p_value;
- --RAISE NOTICE '%', v_sql;
- EXECUTE v_sql INTO v_count;
- IF v_count::integer != 0 THEN
- SELECT att.attname
- INTO v_key
- FROM pg_attribute att
- LEFT JOIN pg_constraint con ON con.conrelid = att.attrelid
- AND con.conkey[1] = att.attnum
- AND con.contype = 'p', pg_type typ, pg_class rel, pg_namespace ns
- WHERE att.attrelid = rel.oid
- AND att.attnum > 0
- AND typ.oid = att.atttypid
- AND att.attisdropped = false
- AND rel.relname = p_tabelle
- AND con.conkey[1] = 1
- AND ns.oid = rel.relnamespace
- AND ns.nspname = 'public'
- ORDER BY att.attnum;
- v_sql := 'SELECT ' || v_key || ' AS id FROM ' || p_tabelle || ' WHERE ' || p_key || ' = ' || p_value;
- FOR v_data IN EXECUTE v_sql
- LOOP
- --RAISE NOTICE ' -> % %', p_tabelle, v_data.id;
- FOR v_constraint IN SELECT t.constraint_name
- , t.constraint_type
- , t.table_name
- , c.column_name
- FROM public.v_table_constraints t
- , public.v_constraint_columns c
- WHERE t.constraint_name = c.constraint_name
- AND t.constraint_type = 'FOREIGN KEY'
- AND c.table_name = p_tabelle
- AND t.table_schema = 'public'
- AND c.table_schema = 'public'
- LOOP
- v_fieldname := substring(v_constraint.constraint_name from 1 for length(v_constraint.constraint_name) - length(v_constraint.column_name) - 1);
- IF (v_constraint.table_name = p_tabelle) AND (p_value = v_data.id) THEN
- --RAISE NOTICE 'Skip (Selbstverweis)';
- CONTINUE;
- ELSE
- PERFORM delete_data(v_constraint.table_name::varchar, v_fieldname::varchar, v_data.id::integer);
- END IF;
- END LOOP;
- END LOOP;
- v_sql := 'DELETE FROM ' || p_tabelle || ' WHERE ' || p_key || ' = ' || p_value;
- --RAISE NOTICE '%', v_sql;
- EXECUTE v_sql;
- p_retval := 1;
- ELSE
- --RAISE NOTICE ' -> Keine Sätze gefunden';
- p_retval := 0;
- END IF;
- RETURN p_retval;
-END;
-$$
-LANGUAGE plpgsql;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/tests/files/function_psql2.sql
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/tests/files/function_psql2.sql b/shell/ext-py/sqlparse-0.1.14/tests/files/function_psql2.sql
deleted file mode 100644
index b5d494c..0000000
--- a/shell/ext-py/sqlparse-0.1.14/tests/files/function_psql2.sql
+++ /dev/null
@@ -1,7 +0,0 @@
-CREATE OR REPLACE FUNCTION update_something() RETURNS void AS
-$body$
-BEGIN
- raise notice 'foo';
-END;
-$body$
-LANGUAGE 'plpgsql' VOLATILE CALLED ON NULL INPUT SECURITY INVOKER;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/tests/files/function_psql3.sql
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/tests/files/function_psql3.sql b/shell/ext-py/sqlparse-0.1.14/tests/files/function_psql3.sql
deleted file mode 100644
index b25d818..0000000
--- a/shell/ext-py/sqlparse-0.1.14/tests/files/function_psql3.sql
+++ /dev/null
@@ -1,8 +0,0 @@
-CREATE OR REPLACE FUNCTION foo() RETURNS integer AS
-$body$
-DECLARE
-BEGIN
- select * from foo;
-END;
-$body$
-LANGUAGE 'plpgsql' VOLATILE CALLED ON NULL INPUT SECURITY INVOKER;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/tests/files/huge_select.sql
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/tests/files/huge_select.sql b/shell/ext-py/sqlparse-0.1.14/tests/files/huge_select.sql
deleted file mode 100644
index ab39823..0000000
--- a/shell/ext-py/sqlparse-0.1.14/tests/files/huge_select.sql
+++ /dev/null
@@ -1 +0,0 @@
-select case when i = 0 then 1 else 0 end as col0, case when i = 1 then 1 else 1 end as col1, case when i = 2 then 1 else 2 end as col2, case when i = 3 then 1 else 3 end as col3, case when i = 4 then 1 else 4 end as col4, case when i = 5 then 1 else 5 end as col5, case when i = 6 then 1 else 6 end as col6, case when i = 7 then 1 else 7 end as col7, case when i = 8 then 1 else 8 end as col8, case when i = 9 then 1 else 9 end as col9, case when i = 10 then 1 else 10 end as col10, case when i = 11 then 1 else 11 end as col11, case when i = 12 then 1 else 12 end as col12, case when i = 13 then 1 else 13 end as col13, case when i = 14 then 1 else 14 end as col14, case when i = 15 then 1 else 15 end as col15, case when i = 16 then 1 else 16 end as col16, case when i = 17 then 1 else 17 end as col17, case when i = 18 then 1 else 18 end as col18, case when i = 19 then 1 else 19 end as col19, case when i = 20 then 1 else 20 end as col20, case when i = 21 then 1 else 21 end as col21, case whe
n i = 22 then 1 else 22 end as col22 from foo UNION select case when i = 0 then 1 else 0 end as col0, case when i = 1 then 1 else 1 end as col1, case when i = 2 then 1 else 2 end as col2, case when i = 3 then 1 else 3 end as col3, case when i = 4 then 1 else 4 end as col4, case when i = 5 then 1 else 5 end as col5, case when i = 6 then 1 else 6 end as col6, case when i = 7 then 1 else 7 end as col7, case when i = 8 then 1 else 8 end as col8, case when i = 9 then 1 else 9 end as col9, case when i = 10 then 1 else 10 end as col10, case when i = 11 then 1 else 11 end as col11, case when i = 12 then 1 else 12 end as col12, case when i = 13 then 1 else 13 end as col13, case when i = 14 then 1 else 14 end as col14, case when i = 15 then 1 else 15 end as col15, case when i = 16 then 1 else 16 end as col16, case when i = 17 then 1 else 17 end as col17, case when i = 18 then 1 else 18 end as col18, case when i = 19 then 1 else 19 end as col19, case when i = 20 then 1 else 20 end as col20, ca
se when i = 21 then 1 else 21 end as col21, case when i = 22 then 1 else 22 end as col22 from foo UNION select case when i = 0 then 1 else 0 end as col0, case when i = 1 then 1 else 1 end as col1, case when i = 2 then 1 else 2 end as col2, case when i = 3 then 1 else 3 end as col3, case when i = 4 then 1 else 4 end as col4, case when i = 5 then 1 else 5 end as col5, case when i = 6 then 1 else 6 end as col6, case when i = 7 then 1 else 7 end as col7, case when i = 8 then 1 else 8 end as col8, case when i = 9 then 1 else 9 end as col9, case when i = 10 then 1 else 10 end as col10, case when i = 11 then 1 else 11 end as col11, case when i = 12 then 1 else 12 end as col12, case when i = 13 then 1 else 13 end as col13, case when i = 14 then 1 else 14 end as col14, case when i = 15 then 1 else 15 end as col15, case when i = 16 then 1 else 16 end as col16, case when i = 17 then 1 else 17 end as col17, case when i = 18 then 1 else 18 end as col18, case when i = 19 then 1 else 19 end as col
19, case when i = 20 then 1 else 20 end as col20, case when i = 21 then 1 else 21 end as col21, case when i = 22 then 1 else 22 end as col22 from foo UNION select case when i = 0 then 1 else 0 end as col0, case when i = 1 then 1 else 1 end as col1, case when i = 2 then 1 else 2 end as col2, case when i = 3 then 1 else 3 end as col3, case when i = 4 then 1 else 4 end as col4, case when i = 5 then 1 else 5 end as col5, case when i = 6 then 1 else 6 end as col6, case when i = 7 then 1 else 7 end as col7, case when i = 8 then 1 else 8 end as col8, case when i = 9 then 1 else 9 end as col9, case when i = 10 then 1 else 10 end as col10, case when i = 11 then 1 else 11 end as col11, case when i = 12 then 1 else 12 end as col12, case when i = 13 then 1 else 13 end as col13, case when i = 14 then 1 else 14 end as col14, case when i = 15 then 1 else 15 end as col15, case when i = 16 then 1 else 16 end as col16, case when i = 17 then 1 else 17 end as col17, case when i = 18 then 1 else 18 end
as col18, case when i = 19 then 1 else 19 end as col19, case when i = 20 then 1 else 20 end as col20, case when i = 21 then 1 else 21 end as col21, case when i = 22 then 1 else 22 end as col22 from foo UNION select case when i = 0 then 1 else 0 end as col0, case when i = 1 then 1 else 1 end as col1, case when i = 2 then 1 else 2 end as col2, case when i = 3 then 1 else 3 end as col3, case when i = 4 then 1 else 4 end as col4, case when i = 5 then 1 else 5 end as col5, case when i = 6 then 1 else 6 end as col6, case when i = 7 then 1 else 7 end as col7, case when i = 8 then 1 else 8 end as col8, case when i = 9 then 1 else 9 end as col9, case when i = 10 then 1 else 10 end as col10, case when i = 11 then 1 else 11 end as col11, case when i = 12 then 1 else 12 end as col12, case when i = 13 then 1 else 13 end as col13, case when i = 14 then 1 else 14 end as col14, case when i = 15 then 1 else 15 end as col15, case when i = 16 then 1 else 16 end as col16, case when i = 17 then 1 else 1
7 end as col17, case when i = 18 then 1 else 18 end as col18, case when i = 19 then 1 else 19 end as col19, case when i = 20 then 1 else 20 end as col20, case when i = 21 then 1 else 21 end as col21, case when i = 22 then 1 else 22 end as col22 from foo UNION select case when i = 0 then 1 else 0 end as col0, case when i = 1 then 1 else 1 end as col1, case when i = 2 then 1 else 2 end as col2, case when i = 3 then 1 else 3 end as col3, case when i = 4 then 1 else 4 end as col4, case when i = 5 then 1 else 5 end as col5, case when i = 6 then 1 else 6 end as col6, case when i = 7 then 1 else 7 end as col7, case when i = 8 then 1 else 8 end as col8, case when i = 9 then 1 else 9 end as col9, case when i = 10 then 1 else 10 end as col10, case when i = 11 then 1 else 11 end as col11, case when i = 12 then 1 else 12 end as col12, case when i = 13 then 1 else 13 end as col13, case when i = 14 then 1 else 14 end as col14, case when i = 15 then 1 else 15 end as col15, case when i = 16 then 1
else 16 end as col16, case when i = 17 then 1 else 17 end as col17, case when i = 18 then 1 else 18 end as col18, case when i = 19 then 1 else 19 end as col19, case when i = 20 then 1 else 20 end as col20, case when i = 21 then 1 else 21 end as col21, case when i = 22 then 1 else 22 end as col22 from foo UNION select case when i = 0 then 1 else 0 end as col0, case when i = 1 then 1 else 1 end as col1, case when i = 2 then 1 else 2 end as col2, case when i = 3 then 1 else 3 end as col3, case when i = 4 then 1 else 4 end as col4, case when i = 5 then 1 else 5 end as col5, case when i = 6 then 1 else 6 end as col6, case when i = 7 then 1 else 7 end as col7, case when i = 8 then 1 else 8 end as col8, case when i = 9 then 1 else 9 end as col9, case when i = 10 then 1 else 10 end as col10, case when i = 11 then 1 else 11 end as col11, case when i = 12 then 1 else 12 end as col12, case when i = 13 then 1 else 13 end as col13, case when i = 14 then 1 else 14 end as col14, case when i = 15 t
hen 1 else 15 end as col15, case when i = 16 then 1 else 16 end as col16, case when i = 17 then 1 else 17 end as col17, case when i = 18 then 1 else 18 end as col18, case when i = 19 then 1 else 19 end as col19, case when i = 20 then 1 else 20 end as col20, case when i = 21 then 1 else 21 end as col21, case when i = 22 then 1 else 22 end as col22 from foo UNION select case when i = 0 then 1 else 0 end as col0, case when i = 1 then 1 else 1 end as col1, case when i = 2 then 1 else 2 end as col2, case when i = 3 then 1 else 3 end as col3, case when i = 4 then 1 else 4 end as col4, case when i = 5 then 1 else 5 end as col5, case when i = 6 then 1 else 6 end as col6, case when i = 7 then 1 else 7 end as col7, case when i = 8 then 1 else 8 end as col8, case when i = 9 then 1 else 9 end as col9, case when i = 10 then 1 else 10 end as col10, case when i = 11 then 1 else 11 end as col11, case when i = 12 then 1 else 12 end as col12, case when i = 13 then 1 else 13 end as col13, case when i
= 14 then 1 else 14 end as col14, case when i = 15 then 1 else 15 end as col15, case when i = 16 then 1 else 16 end as col16, case when i = 17 then 1 else 17 end as col17, case when i = 18 then 1 else 18 end as col18, case when i = 19 then 1 else 19 end as col19, case when i = 20 then 1 else 20 end as col20, case when i = 21 then 1 else 21 end as col21, case when i = 22 then 1 else 22 end as col22 from foo UNION select case when i = 0 then 1 else 0 end as col0, case when i = 1 then 1 else 1 end as col1, case when i = 2 then 1 else 2 end as col2, case when i = 3 then 1 else 3 end as col3, case when i = 4 then 1 else 4 end as col4, case when i = 5 then 1 else 5 end as col5, case when i = 6 then 1 else 6 end as col6, case when i = 7 then 1 else 7 end as col7, case when i = 8 then 1 else 8 end as col8, case when i = 9 then 1 else 9 end as col9, case when i = 10 then 1 else 10 end as col10, case when i = 11 then 1 else 11 end as col11, case when i = 12 then 1 else 12 end as col12, case w
hen i = 13 then 1 else 13 end as col13, case when i = 14 then 1 else 14 end as col14, case when i = 15 then 1 else 15 end as col15, case when i = 16 then 1 else 16 end as col16, case when i = 17 then 1 else 17 end as col17, case when i = 18 then 1 else 18 end as col18, case when i = 19 then 1 else 19 end as col19, case when i = 20 then 1 else 20 end as col20, case when i = 21 then 1 else 21 end as col21, case when i = 22 then 1 else 22 end as col22 from foo
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/tests/files/test_cp1251.sql
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/tests/files/test_cp1251.sql b/shell/ext-py/sqlparse-0.1.14/tests/files/test_cp1251.sql
deleted file mode 100644
index 6c0228b..0000000
--- a/shell/ext-py/sqlparse-0.1.14/tests/files/test_cp1251.sql
+++ /dev/null
@@ -1 +0,0 @@
-insert into foo values (1); -- ����� ��� �������
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/tests/test_filters.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/tests/test_filters.py b/shell/ext-py/sqlparse-0.1.14/tests/test_filters.py
deleted file mode 100644
index d827454..0000000
--- a/shell/ext-py/sqlparse-0.1.14/tests/test_filters.py
+++ /dev/null
@@ -1,78 +0,0 @@
-'''
-Created on 24/03/2012
-
-@author: piranna
-'''
-import unittest
-
-from sqlparse.filters import StripWhitespace, Tokens2Unicode
-from sqlparse.lexer import tokenize
-
-
-class Test__StripWhitespace(unittest.TestCase):
- sql = """INSERT INTO dir_entries(type)VALUES(:type);
-
- INSERT INTO directories(inode)
- VALUES(:inode)
- LIMIT 1"""
-
- sql2 = """SELECT child_entry,asdf AS inode, creation
- FROM links
- WHERE parent_dir == :parent_dir AND name == :name
- LIMIT 1"""
-
- sql3 = """SELECT
- 0 AS st_dev,
- 0 AS st_uid,
- 0 AS st_gid,
-
- dir_entries.type AS st_mode,
- dir_entries.inode AS st_ino,
- COUNT(links.child_entry) AS st_nlink,
-
- :creation AS st_ctime,
- dir_entries.access AS st_atime,
- dir_entries.modification AS st_mtime,
-
- COALESCE(files.size,0) AS st_size,
- COALESCE(files.size,0) AS size
-
-FROM dir_entries
- LEFT JOIN files
- ON dir_entries.inode == files.inode
- LEFT JOIN links
- ON dir_entries.inode == links.child_entry
-
-WHERE dir_entries.inode == :inode
-
-GROUP BY dir_entries.inode
-LIMIT 1"""
-
- def test_StripWhitespace1(self):
- self.assertEqual(
- Tokens2Unicode(StripWhitespace(tokenize(self.sql))),
- 'INSERT INTO dir_entries(type)VALUES(:type);INSERT INTO '
- 'directories(inode)VALUES(:inode)LIMIT 1')
-
- def test_StripWhitespace2(self):
- self.assertEqual(
- Tokens2Unicode(StripWhitespace(tokenize(self.sql2))),
- 'SELECT child_entry,asdf AS inode,creation FROM links WHERE '
- 'parent_dir==:parent_dir AND name==:name LIMIT 1')
-
- def test_StripWhitespace3(self):
- self.assertEqual(
- Tokens2Unicode(StripWhitespace(tokenize(self.sql3))),
- 'SELECT 0 AS st_dev,0 AS st_uid,0 AS st_gid,dir_entries.type AS '
- 'st_mode,dir_entries.inode AS st_ino,COUNT(links.child_entry)AS '
- 'st_nlink,:creation AS st_ctime,dir_entries.access AS st_atime,'
- 'dir_entries.modification AS st_mtime,COALESCE(files.size,0)AS '
- 'st_size,COALESCE(files.size,0)AS size FROM dir_entries LEFT JOIN'
- ' files ON dir_entries.inode==files.inode LEFT JOIN links ON '
- 'dir_entries.inode==links.child_entry WHERE dir_entries.inode=='
- ':inode GROUP BY dir_entries.inode LIMIT 1')
-
-
-if __name__ == "__main__":
- #import sys;sys.argv = ['', 'Test.testName']
- unittest.main()
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/tests/test_format.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/tests/test_format.py b/shell/ext-py/sqlparse-0.1.14/tests/test_format.py
deleted file mode 100644
index b77b7a1..0000000
--- a/shell/ext-py/sqlparse-0.1.14/tests/test_format.py
+++ /dev/null
@@ -1,328 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import pytest
-
-from tests.utils import TestCaseBase
-
-import sqlparse
-from sqlparse.exceptions import SQLParseError
-
-
-class TestFormat(TestCaseBase):
-
- def test_keywordcase(self):
- sql = 'select * from bar; -- select foo\n'
- res = sqlparse.format(sql, keyword_case='upper')
- self.ndiffAssertEqual(res, 'SELECT * FROM bar; -- select foo\n')
- res = sqlparse.format(sql, keyword_case='capitalize')
- self.ndiffAssertEqual(res, 'Select * From bar; -- select foo\n')
- res = sqlparse.format(sql.upper(), keyword_case='lower')
- self.ndiffAssertEqual(res, 'select * from BAR; -- SELECT FOO\n')
- self.assertRaises(SQLParseError, sqlparse.format, sql,
- keyword_case='foo')
-
- def test_identifiercase(self):
- sql = 'select * from bar; -- select foo\n'
- res = sqlparse.format(sql, identifier_case='upper')
- self.ndiffAssertEqual(res, 'select * from BAR; -- select foo\n')
- res = sqlparse.format(sql, identifier_case='capitalize')
- self.ndiffAssertEqual(res, 'select * from Bar; -- select foo\n')
- res = sqlparse.format(sql.upper(), identifier_case='lower')
- self.ndiffAssertEqual(res, 'SELECT * FROM bar; -- SELECT FOO\n')
- self.assertRaises(SQLParseError, sqlparse.format, sql,
- identifier_case='foo')
- sql = 'select * from "foo"."bar"'
- res = sqlparse.format(sql, identifier_case="upper")
- self.ndiffAssertEqual(res, 'select * from "foo"."bar"')
-
- def test_strip_comments_single(self):
- sql = 'select *-- statement starts here\nfrom foo'
- res = sqlparse.format(sql, strip_comments=True)
- self.ndiffAssertEqual(res, 'select * from foo')
- sql = 'select * -- statement starts here\nfrom foo'
- res = sqlparse.format(sql, strip_comments=True)
- self.ndiffAssertEqual(res, 'select * from foo')
- sql = 'select-- foo\nfrom -- bar\nwhere'
- res = sqlparse.format(sql, strip_comments=True)
- self.ndiffAssertEqual(res, 'select from where')
- self.assertRaises(SQLParseError, sqlparse.format, sql,
- strip_comments=None)
-
- def test_strip_comments_multi(self):
- sql = '/* sql starts here */\nselect'
- res = sqlparse.format(sql, strip_comments=True)
- self.ndiffAssertEqual(res, 'select')
- sql = '/* sql starts here */ select'
- res = sqlparse.format(sql, strip_comments=True)
- self.ndiffAssertEqual(res, 'select')
- sql = '/*\n * sql starts here\n */\nselect'
- res = sqlparse.format(sql, strip_comments=True)
- self.ndiffAssertEqual(res, 'select')
- sql = 'select (/* sql starts here */ select 2)'
- res = sqlparse.format(sql, strip_comments=True)
- self.ndiffAssertEqual(res, 'select (select 2)')
-
- def test_strip_ws(self):
- f = lambda sql: sqlparse.format(sql, strip_whitespace=True)
- s = 'select\n* from foo\n\twhere ( 1 = 2 )\n'
- self.ndiffAssertEqual(f(s), 'select * from foo where (1 = 2)')
- s = 'select -- foo\nfrom bar\n'
- self.ndiffAssertEqual(f(s), 'select -- foo\nfrom bar')
- self.assertRaises(SQLParseError, sqlparse.format, s,
- strip_whitespace=None)
-
- def test_preserve_ws(self):
- # preserve at least one whitespace after subgroups
- f = lambda sql: sqlparse.format(sql, strip_whitespace=True)
- s = 'select\n* /* foo */ from bar '
- self.ndiffAssertEqual(f(s), 'select * /* foo */ from bar')
-
- def test_notransform_of_quoted_crlf(self):
- # Make sure that CR/CR+LF characters inside string literals don't get
- # affected by the formatter.
-
- s1 = "SELECT some_column LIKE 'value\r'"
- s2 = "SELECT some_column LIKE 'value\r'\r\nWHERE id = 1\n"
- s3 = "SELECT some_column LIKE 'value\\'\r' WHERE id = 1\r"
- s4 = "SELECT some_column LIKE 'value\\\\\\'\r' WHERE id = 1\r\n"
-
- f = lambda x: sqlparse.format(x)
-
- # Because of the use of
- self.ndiffAssertEqual(f(s1), "SELECT some_column LIKE 'value\r'")
- self.ndiffAssertEqual(f(s2), "SELECT some_column LIKE 'value\r'\nWHERE id = 1\n")
- self.ndiffAssertEqual(f(s3), "SELECT some_column LIKE 'value\\'\r' WHERE id = 1\n")
- self.ndiffAssertEqual(f(s4), "SELECT some_column LIKE 'value\\\\\\'\r' WHERE id = 1\n")
-
- def test_outputformat(self):
- sql = 'select * from foo;'
- self.assertRaises(SQLParseError, sqlparse.format, sql,
- output_format='foo')
-
-
-class TestFormatReindent(TestCaseBase):
-
- def test_option(self):
- self.assertRaises(SQLParseError, sqlparse.format, 'foo',
- reindent=2)
- self.assertRaises(SQLParseError, sqlparse.format, 'foo',
- indent_tabs=2)
- self.assertRaises(SQLParseError, sqlparse.format, 'foo',
- reindent=True, indent_width='foo')
- self.assertRaises(SQLParseError, sqlparse.format, 'foo',
- reindent=True, indent_width=-12)
-
- def test_stmts(self):
- f = lambda sql: sqlparse.format(sql, reindent=True)
- s = 'select foo; select bar'
- self.ndiffAssertEqual(f(s), 'select foo;\n\nselect bar')
- s = 'select foo'
- self.ndiffAssertEqual(f(s), 'select foo')
- s = 'select foo; -- test\n select bar'
- self.ndiffAssertEqual(f(s), 'select foo; -- test\n\nselect bar')
-
- def test_keywords(self):
- f = lambda sql: sqlparse.format(sql, reindent=True)
- s = 'select * from foo union select * from bar;'
- self.ndiffAssertEqual(f(s), '\n'.join(['select *',
- 'from foo',
- 'union',
- 'select *',
- 'from bar;']))
-
- def test_keywords_between(self): # issue 14
- # don't break AND after BETWEEN
- f = lambda sql: sqlparse.format(sql, reindent=True)
- s = 'and foo between 1 and 2 and bar = 3'
- self.ndiffAssertEqual(f(s), '\n'.join(['',
- 'and foo between 1 and 2',
- 'and bar = 3']))
-
- def test_parenthesis(self):
- f = lambda sql: sqlparse.format(sql, reindent=True)
- s = 'select count(*) from (select * from foo);'
- self.ndiffAssertEqual(f(s),
- '\n'.join(['select count(*)',
- 'from',
- ' (select *',
- ' from foo);',
- ])
- )
-
- def test_where(self):
- f = lambda sql: sqlparse.format(sql, reindent=True)
- s = 'select * from foo where bar = 1 and baz = 2 or bzz = 3;'
- self.ndiffAssertEqual(f(s), ('select *\nfrom foo\n'
- 'where bar = 1\n'
- ' and baz = 2\n'
- ' or bzz = 3;'))
- s = 'select * from foo where bar = 1 and (baz = 2 or bzz = 3);'
- self.ndiffAssertEqual(f(s), ('select *\nfrom foo\n'
- 'where bar = 1\n'
- ' and (baz = 2\n'
- ' or bzz = 3);'))
-
- def test_join(self):
- f = lambda sql: sqlparse.format(sql, reindent=True)
- s = 'select * from foo join bar on 1 = 2'
- self.ndiffAssertEqual(f(s), '\n'.join(['select *',
- 'from foo',
- 'join bar on 1 = 2']))
- s = 'select * from foo inner join bar on 1 = 2'
- self.ndiffAssertEqual(f(s), '\n'.join(['select *',
- 'from foo',
- 'inner join bar on 1 = 2']))
- s = 'select * from foo left outer join bar on 1 = 2'
- self.ndiffAssertEqual(f(s), '\n'.join(['select *',
- 'from foo',
- 'left outer join bar on 1 = 2']
- ))
- s = 'select * from foo straight_join bar on 1 = 2'
- self.ndiffAssertEqual(f(s), '\n'.join(['select *',
- 'from foo',
- 'straight_join bar on 1 = 2']
- ))
-
- def test_identifier_list(self):
- f = lambda sql: sqlparse.format(sql, reindent=True)
- s = 'select foo, bar, baz from table1, table2 where 1 = 2'
- self.ndiffAssertEqual(f(s), '\n'.join(['select foo,',
- ' bar,',
- ' baz',
- 'from table1,',
- ' table2',
- 'where 1 = 2']))
- s = 'select a.*, b.id from a, b'
- self.ndiffAssertEqual(f(s), '\n'.join(['select a.*,',
- ' b.id',
- 'from a,',
- ' b']))
-
- def test_identifier_list_with_functions(self):
- f = lambda sql: sqlparse.format(sql, reindent=True)
- s = ("select 'abc' as foo, coalesce(col1, col2)||col3 as bar,"
- "col3 from my_table")
- self.ndiffAssertEqual(f(s), '\n'.join(
- ["select 'abc' as foo,",
- " coalesce(col1, col2)||col3 as bar,",
- " col3",
- "from my_table"]))
-
- def test_case(self):
- f = lambda sql: sqlparse.format(sql, reindent=True)
- s = 'case when foo = 1 then 2 when foo = 3 then 4 else 5 end'
- self.ndiffAssertEqual(f(s), '\n'.join(['case',
- ' when foo = 1 then 2',
- ' when foo = 3 then 4',
- ' else 5',
- 'end']))
-
- def test_case2(self):
- f = lambda sql: sqlparse.format(sql, reindent=True)
- s = 'case(foo) when bar = 1 then 2 else 3 end'
- self.ndiffAssertEqual(f(s), '\n'.join(['case(foo)',
- ' when bar = 1 then 2',
- ' else 3',
- 'end']))
-
- def test_nested_identifier_list(self): # issue4
- f = lambda sql: sqlparse.format(sql, reindent=True)
- s = '(foo as bar, bar1, bar2 as bar3, b4 as b5)'
- self.ndiffAssertEqual(f(s), '\n'.join(['(foo as bar,',
- ' bar1,',
- ' bar2 as bar3,',
- ' b4 as b5)']))
-
- def test_duplicate_linebreaks(self): # issue3
- f = lambda sql: sqlparse.format(sql, reindent=True)
- s = 'select c1 -- column1\nfrom foo'
- self.ndiffAssertEqual(f(s), '\n'.join(['select c1 -- column1',
- 'from foo']))
- s = 'select c1 -- column1\nfrom foo'
- r = sqlparse.format(s, reindent=True, strip_comments=True)
- self.ndiffAssertEqual(r, '\n'.join(['select c1',
- 'from foo']))
- s = 'select c1\nfrom foo\norder by c1'
- self.ndiffAssertEqual(f(s), '\n'.join(['select c1',
- 'from foo',
- 'order by c1']))
- s = 'select c1 from t1 where (c1 = 1) order by c1'
- self.ndiffAssertEqual(f(s), '\n'.join(['select c1',
- 'from t1',
- 'where (c1 = 1)',
- 'order by c1']))
-
- def test_keywordfunctions(self): # issue36
- f = lambda sql: sqlparse.format(sql, reindent=True)
- s = 'select max(a) b, foo, bar'
- self.ndiffAssertEqual(f(s), '\n'.join(['select max(a) b,',
- ' foo,',
- ' bar']))
-
- def test_identifier_and_functions(self): # issue45
- f = lambda sql: sqlparse.format(sql, reindent=True)
- s = 'select foo.bar, nvl(1) from dual'
- self.ndiffAssertEqual(f(s), '\n'.join(['select foo.bar,',
- ' nvl(1)',
- 'from dual']))
-
-
-class TestOutputFormat(TestCaseBase):
-
- def test_python(self):
- sql = 'select * from foo;'
- f = lambda sql: sqlparse.format(sql, output_format='python')
- self.ndiffAssertEqual(f(sql), "sql = 'select * from foo;'")
- f = lambda sql: sqlparse.format(sql, output_format='python',
- reindent=True)
- self.ndiffAssertEqual(f(sql), ("sql = ('select * '\n"
- " 'from foo;')"))
-
- def test_php(self):
- sql = 'select * from foo;'
- f = lambda sql: sqlparse.format(sql, output_format='php')
- self.ndiffAssertEqual(f(sql), '$sql = "select * from foo;";')
- f = lambda sql: sqlparse.format(sql, output_format='php',
- reindent=True)
- self.ndiffAssertEqual(f(sql), ('$sql = "select * ";\n'
- '$sql .= "from foo;";'))
-
- def test_sql(self): # "sql" is an allowed option but has no effect
- sql = 'select * from foo;'
- f = lambda sql: sqlparse.format(sql, output_format='sql')
- self.ndiffAssertEqual(f(sql), 'select * from foo;')
-
-
-def test_format_column_ordering(): # issue89
- sql = 'select * from foo order by c1 desc, c2, c3;'
- formatted = sqlparse.format(sql, reindent=True)
- expected = '\n'.join(['select *',
- 'from foo',
- 'order by c1 desc,',
- ' c2,',
- ' c3;'])
- assert formatted == expected
-
-
-def test_truncate_strings():
- sql = 'update foo set value = \'' + 'x' * 1000 + '\';'
- formatted = sqlparse.format(sql, truncate_strings=10)
- assert formatted == 'update foo set value = \'xxxxxxxxxx[...]\';'
- formatted = sqlparse.format(sql, truncate_strings=3, truncate_char='YYY')
- assert formatted == 'update foo set value = \'xxxYYY\';'
-
-
-def test_truncate_strings_invalid_option():
- pytest.raises(SQLParseError, sqlparse.format,
- 'foo', truncate_strings='bar')
- pytest.raises(SQLParseError, sqlparse.format,
- 'foo', truncate_strings=-1)
- pytest.raises(SQLParseError, sqlparse.format,
- 'foo', truncate_strings=0)
-
-
-@pytest.mark.parametrize('sql', ['select verrrylongcolumn from foo',
- 'select "verrrylongcolumn" from "foo"'])
-def test_truncate_strings_doesnt_truncate_identifiers(sql):
- formatted = sqlparse.format(sql, truncate_strings=2)
- assert formatted == sql
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/tests/test_functions.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/tests/test_functions.py b/shell/ext-py/sqlparse-0.1.14/tests/test_functions.py
deleted file mode 100644
index 52e2ce7..0000000
--- a/shell/ext-py/sqlparse-0.1.14/tests/test_functions.py
+++ /dev/null
@@ -1,164 +0,0 @@
-'''
-Created on 13/02/2012
-
-@author: piranna
-'''
-from unittest import main, TestCase
-
-from sqlparse.filters import IncludeStatement, Tokens2Unicode
-from sqlparse.lexer import tokenize
-
-import sys
-sys.path.insert(0, '..')
-
-from sqlparse.filters import compact
-from sqlparse.functions import getcolumns, getlimit, IsType
-
-
-class Test_IncludeStatement(TestCase):
- sql = """-- type: script
- -- return: integer
-
- INCLUDE "_Make_DirEntry.sql";
-
- INSERT INTO directories(inode)
- VALUES(:inode)
- LIMIT 1"""
-
- def test_includeStatement(self):
- stream = tokenize(self.sql)
- includeStatement = IncludeStatement('tests/files',
- raiseexceptions=True)
- stream = includeStatement.process(None, stream)
- stream = compact(stream)
-
- result = Tokens2Unicode(stream)
-
- self.assertEqual(
- result, (
- 'INSERT INTO dir_entries(type)VALUES(:type);INSERT INTO '
- 'directories(inode)VALUES(:inode)LIMIT 1'))
-
-
-class Test_SQL(TestCase):
- sql = """-- type: script
- -- return: integer
-
- INSERT INTO directories(inode)
- VALUES(:inode)
- LIMIT 1"""
-
- sql2 = """SELECT child_entry,asdf AS inode, creation
- FROM links
- WHERE parent_dir == :parent_dir AND name == :name
- LIMIT 1"""
-
- sql3 = """SELECT
- 0 AS st_dev,
- 0 AS st_uid,
- 0 AS st_gid,
-
- dir_entries.type AS st_mode,
- dir_entries.inode AS st_ino,
- COUNT(links.child_entry) AS st_nlink,
-
- :creation AS st_ctime,
- dir_entries.access AS st_atime,
- dir_entries.modification AS st_mtime,
--- :creation AS st_ctime,
--- CAST(STRFTIME('%s',dir_entries.access) AS INTEGER) AS st_atime,
--- CAST(STRFTIME('%s',dir_entries.modification) AS INTEGER) AS st_mtime,
-
- COALESCE(files.size,0) AS st_size, -- Python-FUSE
- COALESCE(files.size,0) AS size -- PyFilesystem
-
-FROM dir_entries
- LEFT JOIN files
- ON dir_entries.inode == files.inode
- LEFT JOIN links
- ON dir_entries.inode == links.child_entry
-
-WHERE dir_entries.inode == :inode
-
-GROUP BY dir_entries.inode
-LIMIT 1"""
-
-
-class Test_Compact(Test_SQL):
- def test_compact1(self):
- stream = compact(tokenize(self.sql))
-
- result = Tokens2Unicode(stream)
-
- self.assertEqual(result,
- 'INSERT INTO directories(inode)VALUES(:inode)LIMIT 1')
-
- def test_compact2(self):
- stream = tokenize(self.sql2)
-
- result = compact(stream)
-
- self.assertEqual(
- Tokens2Unicode(result),
- 'SELECT child_entry,asdf AS inode,creation FROM links WHERE '
- 'parent_dir==:parent_dir AND name==:name LIMIT 1')
-
- def test_compact3(self):
- stream = tokenize(self.sql3)
-
- result = compact(stream)
-
- self.assertEqual(
- Tokens2Unicode(result),
- 'SELECT 0 AS st_dev,0 AS st_uid,0 AS st_gid,dir_entries.type AS '
- 'st_mode,dir_entries.inode AS st_ino,COUNT(links.child_entry)AS '
- 'st_nlink,:creation AS st_ctime,dir_entries.access AS st_atime,'
- 'dir_entries.modification AS st_mtime,COALESCE(files.size,0)AS '
- 'st_size,COALESCE(files.size,0)AS size FROM dir_entries LEFT JOIN'
- ' files ON dir_entries.inode==files.inode LEFT JOIN links ON '
- 'dir_entries.inode==links.child_entry WHERE dir_entries.inode=='
- ':inode GROUP BY dir_entries.inode LIMIT 1')
-
-
-class Test_GetColumns(Test_SQL):
- def test_getcolumns1(self):
- columns = getcolumns(tokenize(self.sql))
- self.assertEqual(columns, [])
-
- def test_getcolumns2(self):
- columns = getcolumns(tokenize(self.sql2))
- self.assertEqual(columns, ['child_entry', 'inode', 'creation'])
-
- def test_getcolumns3(self):
- columns = getcolumns(tokenize(self.sql3))
- self.assertEqual(columns, ['st_dev', 'st_uid', 'st_gid', 'st_mode',
- 'st_ino', 'st_nlink', 'st_ctime',
- 'st_atime', 'st_mtime', 'st_size', 'size'])
-
-
-class Test_GetLimit(Test_SQL):
- def test_getlimit1(self):
- limit = getlimit(tokenize(self.sql))
- self.assertEqual(limit, 1)
-
- def test_getlimit2(self):
- limit = getlimit(tokenize(self.sql2))
- self.assertEqual(limit, 1)
-
- def test_getlimit3(self):
- limit = getlimit(tokenize(self.sql3))
- self.assertEqual(limit, 1)
-
-
-class Test_IsType(Test_SQL):
- def test_istype2(self):
- stream = tokenize(self.sql2)
- self.assertTrue(IsType('SELECT')(stream))
-
- stream = tokenize(self.sql2)
- self.assertFalse(IsType('INSERT')(stream))
-
-
-if __name__ == "__main__":
- #import sys;sys.argv = ['', 'Test.testName']
- main()
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/tests/test_grouping.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/tests/test_grouping.py b/shell/ext-py/sqlparse-0.1.14/tests/test_grouping.py
deleted file mode 100644
index 86d4c7a..0000000
--- a/shell/ext-py/sqlparse-0.1.14/tests/test_grouping.py
+++ /dev/null
@@ -1,341 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import pytest
-
-import sqlparse
-from sqlparse import sql
-from sqlparse import tokens as T
-
-from tests.utils import TestCaseBase
-
-
-class TestGrouping(TestCaseBase):
-
- def test_parenthesis(self):
- s = 'select (select (x3) x2) and (y2) bar'
- parsed = sqlparse.parse(s)[0]
- self.ndiffAssertEqual(s, str(parsed))
- self.assertEqual(len(parsed.tokens), 9)
- self.assert_(isinstance(parsed.tokens[2], sql.Parenthesis))
- self.assert_(isinstance(parsed.tokens[-3], sql.Parenthesis))
- self.assertEqual(len(parsed.tokens[2].tokens), 7)
- self.assert_(isinstance(parsed.tokens[2].tokens[3], sql.Parenthesis))
- self.assertEqual(len(parsed.tokens[2].tokens[3].tokens), 3)
-
- def test_comments(self):
- s = '/*\n * foo\n */ \n bar'
- parsed = sqlparse.parse(s)[0]
- self.ndiffAssertEqual(s, unicode(parsed))
- self.assertEqual(len(parsed.tokens), 2)
-
- def test_assignment(self):
- s = 'foo := 1;'
- parsed = sqlparse.parse(s)[0]
- self.assertEqual(len(parsed.tokens), 1)
- self.assert_(isinstance(parsed.tokens[0], sql.Assignment))
- s = 'foo := 1'
- parsed = sqlparse.parse(s)[0]
- self.assertEqual(len(parsed.tokens), 1)
- self.assert_(isinstance(parsed.tokens[0], sql.Assignment))
-
- def test_identifiers(self):
- s = 'select foo.bar from "myscheme"."table" where fail. order'
- parsed = sqlparse.parse(s)[0]
- self.ndiffAssertEqual(s, unicode(parsed))
- self.assert_(isinstance(parsed.tokens[2], sql.Identifier))
- self.assert_(isinstance(parsed.tokens[6], sql.Identifier))
- self.assert_(isinstance(parsed.tokens[8], sql.Where))
- s = 'select * from foo where foo.id = 1'
- parsed = sqlparse.parse(s)[0]
- self.ndiffAssertEqual(s, unicode(parsed))
- self.assert_(isinstance(parsed.tokens[-1].tokens[-1].tokens[0],
- sql.Identifier))
- s = 'select * from (select "foo"."id" from foo)'
- parsed = sqlparse.parse(s)[0]
- self.ndiffAssertEqual(s, unicode(parsed))
- self.assert_(isinstance(parsed.tokens[-1].tokens[3], sql.Identifier))
-
- s = "INSERT INTO `test` VALUES('foo', 'bar');"
- parsed = sqlparse.parse(s)[0]
- types = [l.ttype for l in parsed.tokens if not l.is_whitespace()]
- self.assertEquals(types, [T.DML, T.Keyword, None,
- T.Keyword, None, T.Punctuation])
-
- s = "select 1.0*(a+b) as col, sum(c)/sum(d) from myschema.mytable"
- parsed = sqlparse.parse(s)[0]
- self.assertEqual(len(parsed.tokens), 7)
- self.assert_(isinstance(parsed.tokens[2], sql.IdentifierList))
- self.assertEqual(len(parsed.tokens[2].tokens), 4)
- identifiers = list(parsed.tokens[2].get_identifiers())
- self.assertEqual(len(identifiers), 2)
- self.assertEquals(identifiers[0].get_alias(), u"col")
-
- def test_identifier_wildcard(self):
- p = sqlparse.parse('a.*, b.id')[0]
- self.assert_(isinstance(p.tokens[0], sql.IdentifierList))
- self.assert_(isinstance(p.tokens[0].tokens[0], sql.Identifier))
- self.assert_(isinstance(p.tokens[0].tokens[-1], sql.Identifier))
-
- def test_identifier_name_wildcard(self):
- p = sqlparse.parse('a.*')[0]
- t = p.tokens[0]
- self.assertEqual(t.get_name(), '*')
- self.assertEqual(t.is_wildcard(), True)
-
- def test_identifier_invalid(self):
- p = sqlparse.parse('a.')[0]
- self.assert_(isinstance(p.tokens[0], sql.Identifier))
- self.assertEqual(p.tokens[0].has_alias(), False)
- self.assertEqual(p.tokens[0].get_name(), None)
- self.assertEqual(p.tokens[0].get_real_name(), None)
- self.assertEqual(p.tokens[0].get_parent_name(), 'a')
-
- def test_identifier_as_invalid(self): # issue8
- p = sqlparse.parse('foo as select *')[0]
- self.assert_(len(p.tokens), 5)
- self.assert_(isinstance(p.tokens[0], sql.Identifier))
- self.assertEqual(len(p.tokens[0].tokens), 1)
- self.assertEqual(p.tokens[2].ttype, T.Keyword)
-
- def test_identifier_function(self):
- p = sqlparse.parse('foo() as bar')[0]
- self.assert_(isinstance(p.tokens[0], sql.Identifier))
- self.assert_(isinstance(p.tokens[0].tokens[0], sql.Function))
- p = sqlparse.parse('foo()||col2 bar')[0]
- self.assert_(isinstance(p.tokens[0], sql.Identifier))
- self.assert_(isinstance(p.tokens[0].tokens[0], sql.Function))
-
- def test_identifier_extended(self): # issue 15
- p = sqlparse.parse('foo+100')[0]
- self.assert_(isinstance(p.tokens[0], sql.Identifier))
- p = sqlparse.parse('foo + 100')[0]
- self.assert_(isinstance(p.tokens[0], sql.Identifier))
- p = sqlparse.parse('foo*100')[0]
- self.assert_(isinstance(p.tokens[0], sql.Identifier))
-
- def test_identifier_list(self):
- p = sqlparse.parse('a, b, c')[0]
- self.assert_(isinstance(p.tokens[0], sql.IdentifierList))
- p = sqlparse.parse('(a, b, c)')[0]
- self.assert_(isinstance(p.tokens[0].tokens[1], sql.IdentifierList))
-
- def test_identifier_list_case(self):
- p = sqlparse.parse('a, case when 1 then 2 else 3 end as b, c')[0]
- self.assert_(isinstance(p.tokens[0], sql.IdentifierList))
- p = sqlparse.parse('(a, case when 1 then 2 else 3 end as b, c)')[0]
- self.assert_(isinstance(p.tokens[0].tokens[1], sql.IdentifierList))
-
- def test_identifier_list_other(self): # issue2
- p = sqlparse.parse("select *, null, 1, 'foo', bar from mytable, x")[0]
- self.assert_(isinstance(p.tokens[2], sql.IdentifierList))
- l = p.tokens[2]
- self.assertEqual(len(l.tokens), 13)
-
- def test_where(self):
- s = 'select * from foo where bar = 1 order by id desc'
- p = sqlparse.parse(s)[0]
- self.ndiffAssertEqual(s, unicode(p))
- self.assertTrue(len(p.tokens), 16)
- s = 'select x from (select y from foo where bar = 1) z'
- p = sqlparse.parse(s)[0]
- self.ndiffAssertEqual(s, unicode(p))
- self.assertTrue(isinstance(p.tokens[-3].tokens[-2], sql.Where))
-
- def test_typecast(self):
- s = 'select foo::integer from bar'
- p = sqlparse.parse(s)[0]
- self.ndiffAssertEqual(s, unicode(p))
- self.assertEqual(p.tokens[2].get_typecast(), 'integer')
- self.assertEqual(p.tokens[2].get_name(), 'foo')
- s = 'select (current_database())::information_schema.sql_identifier'
- p = sqlparse.parse(s)[0]
- self.ndiffAssertEqual(s, unicode(p))
- self.assertEqual(p.tokens[2].get_typecast(),
- 'information_schema.sql_identifier')
-
- def test_alias(self):
- s = 'select foo as bar from mytable'
- p = sqlparse.parse(s)[0]
- self.ndiffAssertEqual(s, unicode(p))
- self.assertEqual(p.tokens[2].get_real_name(), 'foo')
- self.assertEqual(p.tokens[2].get_alias(), 'bar')
- s = 'select foo from mytable t1'
- p = sqlparse.parse(s)[0]
- self.ndiffAssertEqual(s, unicode(p))
- self.assertEqual(p.tokens[6].get_real_name(), 'mytable')
- self.assertEqual(p.tokens[6].get_alias(), 't1')
- s = 'select foo::integer as bar from mytable'
- p = sqlparse.parse(s)[0]
- self.ndiffAssertEqual(s, unicode(p))
- self.assertEqual(p.tokens[2].get_alias(), 'bar')
- s = ('SELECT DISTINCT '
- '(current_database())::information_schema.sql_identifier AS view')
- p = sqlparse.parse(s)[0]
- self.ndiffAssertEqual(s, unicode(p))
- self.assertEqual(p.tokens[4].get_alias(), 'view')
-
- def test_alias_case(self): # see issue46
- p = sqlparse.parse('CASE WHEN 1 THEN 2 ELSE 3 END foo')[0]
- self.assertEqual(len(p.tokens), 1)
- self.assertEqual(p.tokens[0].get_alias(), 'foo')
-
- def test_idlist_function(self): # see issue10 too
- p = sqlparse.parse('foo(1) x, bar')[0]
- self.assert_(isinstance(p.tokens[0], sql.IdentifierList))
-
- def test_comparison_exclude(self):
- # make sure operators are not handled too lazy
- p = sqlparse.parse('(=)')[0]
- self.assert_(isinstance(p.tokens[0], sql.Parenthesis))
- self.assert_(not isinstance(p.tokens[0].tokens[1], sql.Comparison))
- p = sqlparse.parse('(a=1)')[0]
- self.assert_(isinstance(p.tokens[0].tokens[1], sql.Comparison))
- p = sqlparse.parse('(a>=1)')[0]
- self.assert_(isinstance(p.tokens[0].tokens[1], sql.Comparison))
-
- def test_function(self):
- p = sqlparse.parse('foo()')[0]
- self.assert_(isinstance(p.tokens[0], sql.Function))
- p = sqlparse.parse('foo(null, bar)')[0]
- self.assert_(isinstance(p.tokens[0], sql.Function))
- self.assertEqual(len(list(p.tokens[0].get_parameters())), 2)
-
- def test_varchar(self):
- p = sqlparse.parse('"text" Varchar(50) NOT NULL')[0]
- self.assert_(isinstance(p.tokens[2], sql.Function))
-
-
-class TestStatement(TestCaseBase):
-
- def test_get_type(self):
- f = lambda sql: sqlparse.parse(sql)[0]
- self.assertEqual(f('select * from foo').get_type(), 'SELECT')
- self.assertEqual(f('update foo').get_type(), 'UPDATE')
- self.assertEqual(f(' update foo').get_type(), 'UPDATE')
- self.assertEqual(f('\nupdate foo').get_type(), 'UPDATE')
- self.assertEqual(f('foo').get_type(), 'UNKNOWN')
- # Statements that have a whitespace after the closing semicolon
- # are parsed as two statements where later only consists of the
- # trailing whitespace.
- self.assertEqual(f('\n').get_type(), 'UNKNOWN')
-
-
-def test_identifier_with_operators(): # issue 53
- p = sqlparse.parse('foo||bar')[0]
- assert len(p.tokens) == 1
- assert isinstance(p.tokens[0], sql.Identifier)
- # again with whitespaces
- p = sqlparse.parse('foo || bar')[0]
- assert len(p.tokens) == 1
- assert isinstance(p.tokens[0], sql.Identifier)
-
-
-def test_identifier_with_op_trailing_ws():
- # make sure trailing whitespace isn't grouped with identifier
- p = sqlparse.parse('foo || bar ')[0]
- assert len(p.tokens) == 2
- assert isinstance(p.tokens[0], sql.Identifier)
- assert p.tokens[1].ttype is T.Whitespace
-
-
-def test_identifier_with_string_literals():
- p = sqlparse.parse('foo + \'bar\'')[0]
- assert len(p.tokens) == 1
- assert isinstance(p.tokens[0], sql.Identifier)
-
-
-# This test seems to be wrong. It was introduced when fixing #53, but #111
-# showed that this shouldn't be an identifier at all. I'm leaving this
-# commented in the source for a while.
-# def test_identifier_string_concat():
-# p = sqlparse.parse('\'foo\' || bar')[0]
-# assert len(p.tokens) == 1
-# assert isinstance(p.tokens[0], sql.Identifier)
-
-
-def test_identifier_consumes_ordering(): # issue89
- p = sqlparse.parse('select * from foo order by c1 desc, c2, c3')[0]
- assert isinstance(p.tokens[-1], sql.IdentifierList)
- ids = list(p.tokens[-1].get_identifiers())
- assert len(ids) == 3
- assert ids[0].get_name() == 'c1'
- assert ids[0].get_ordering() == 'DESC'
- assert ids[1].get_name() == 'c2'
- assert ids[1].get_ordering() is None
-
-
-def test_comparison_with_keywords(): # issue90
- # in fact these are assignments, but for now we don't distinguish them
- p = sqlparse.parse('foo = NULL')[0]
- assert len(p.tokens) == 1
- assert isinstance(p.tokens[0], sql.Comparison)
- assert len(p.tokens[0].tokens) == 5
- assert p.tokens[0].left.value == 'foo'
- assert p.tokens[0].right.value == 'NULL'
- # make sure it's case-insensitive
- p = sqlparse.parse('foo = null')[0]
- assert len(p.tokens) == 1
- assert isinstance(p.tokens[0], sql.Comparison)
-
-
-def test_comparison_with_floats(): # issue145
- p = sqlparse.parse('foo = 25.5')[0]
- assert len(p.tokens) == 1
- assert isinstance(p.tokens[0], sql.Comparison)
- assert len(p.tokens[0].tokens) == 5
- assert p.tokens[0].left.value == 'foo'
- assert p.tokens[0].right.value == '25.5'
-
-
-def test_comparison_with_parenthesis(): # issue23
- p = sqlparse.parse('(3 + 4) = 7')[0]
- assert len(p.tokens) == 1
- assert isinstance(p.tokens[0], sql.Comparison)
- comp = p.tokens[0]
- assert isinstance(comp.left, sql.Parenthesis)
- assert comp.right.ttype is T.Number.Integer
-
-
-def test_comparison_with_strings(): # issue148
- p = sqlparse.parse('foo = \'bar\'')[0]
- assert len(p.tokens) == 1
- assert isinstance(p.tokens[0], sql.Comparison)
- assert p.tokens[0].right.value == '\'bar\''
- assert p.tokens[0].right.ttype == T.String.Single
-
-
-@pytest.mark.parametrize('start', ['FOR', 'FOREACH'])
-def test_forloops(start):
- p = sqlparse.parse('%s foo in bar LOOP foobar END LOOP' % start)[0]
- assert (len(p.tokens)) == 1
- assert isinstance(p.tokens[0], sql.For)
-
-
-def test_nested_for():
- p = sqlparse.parse('FOR foo LOOP FOR bar LOOP END LOOP END LOOP')[0]
- assert len(p.tokens) == 1
- for1 = p.tokens[0]
- assert for1.tokens[0].value == 'FOR'
- assert for1.tokens[-1].value == 'END LOOP'
- for2 = for1.tokens[6]
- assert isinstance(for2, sql.For)
- assert for2.tokens[0].value == 'FOR'
- assert for2.tokens[-1].value == 'END LOOP'
-
-
-def test_begin():
- p = sqlparse.parse('BEGIN foo END')[0]
- assert len(p.tokens) == 1
- assert isinstance(p.tokens[0], sql.Begin)
-
-
-def test_nested_begin():
- p = sqlparse.parse('BEGIN foo BEGIN bar END END')[0]
- assert len(p.tokens) == 1
- outer = p.tokens[0]
- assert outer.tokens[0].value == 'BEGIN'
- assert outer.tokens[-1].value == 'END'
- inner = outer.tokens[4]
- assert inner.tokens[0].value == 'BEGIN'
- assert inner.tokens[-1].value == 'END'
- assert isinstance(inner, sql.Begin)
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/tests/test_parse.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/tests/test_parse.py b/shell/ext-py/sqlparse-0.1.14/tests/test_parse.py
deleted file mode 100644
index d77bb43..0000000
--- a/shell/ext-py/sqlparse-0.1.14/tests/test_parse.py
+++ /dev/null
@@ -1,191 +0,0 @@
-# -*- coding: utf-8 -*-
-
-"""Tests sqlparse function."""
-
-import pytest
-
-from tests.utils import TestCaseBase
-
-import sqlparse
-import sqlparse.sql
-
-from sqlparse import tokens as T
-
-
-class SQLParseTest(TestCaseBase):
- """Tests sqlparse.parse()."""
-
- def test_tokenize(self):
- sql = 'select * from foo;'
- stmts = sqlparse.parse(sql)
- self.assertEqual(len(stmts), 1)
- self.assertEqual(str(stmts[0]), sql)
-
- def test_multistatement(self):
- sql1 = 'select * from foo;'
- sql2 = 'select * from bar;'
- stmts = sqlparse.parse(sql1 + sql2)
- self.assertEqual(len(stmts), 2)
- self.assertEqual(str(stmts[0]), sql1)
- self.assertEqual(str(stmts[1]), sql2)
-
- def test_newlines(self):
- sql = u'select\n*from foo;'
- p = sqlparse.parse(sql)[0]
- self.assertEqual(unicode(p), sql)
- sql = u'select\r\n*from foo'
- p = sqlparse.parse(sql)[0]
- self.assertEqual(unicode(p), sql)
- sql = u'select\r*from foo'
- p = sqlparse.parse(sql)[0]
- self.assertEqual(unicode(p), sql)
- sql = u'select\r\n*from foo\n'
- p = sqlparse.parse(sql)[0]
- self.assertEqual(unicode(p), sql)
-
- def test_within(self):
- sql = 'foo(col1, col2)'
- p = sqlparse.parse(sql)[0]
- col1 = p.tokens[0].tokens[1].tokens[1].tokens[0]
- self.assert_(col1.within(sqlparse.sql.Function))
-
- def test_child_of(self):
- sql = '(col1, col2)'
- p = sqlparse.parse(sql)[0]
- self.assert_(p.tokens[0].tokens[1].is_child_of(p.tokens[0]))
- sql = 'select foo'
- p = sqlparse.parse(sql)[0]
- self.assert_(not p.tokens[2].is_child_of(p.tokens[0]))
- self.assert_(p.tokens[2].is_child_of(p))
-
- def test_has_ancestor(self):
- sql = 'foo or (bar, baz)'
- p = sqlparse.parse(sql)[0]
- baz = p.tokens[-1].tokens[1].tokens[-1]
- self.assert_(baz.has_ancestor(p.tokens[-1].tokens[1]))
- self.assert_(baz.has_ancestor(p.tokens[-1]))
- self.assert_(baz.has_ancestor(p))
-
- def test_float(self):
- t = sqlparse.parse('.5')[0].tokens
- self.assertEqual(len(t), 1)
- self.assert_(t[0].ttype is sqlparse.tokens.Number.Float)
- t = sqlparse.parse('.51')[0].tokens
- self.assertEqual(len(t), 1)
- self.assert_(t[0].ttype is sqlparse.tokens.Number.Float)
- t = sqlparse.parse('1.5')[0].tokens
- self.assertEqual(len(t), 1)
- self.assert_(t[0].ttype is sqlparse.tokens.Number.Float)
- t = sqlparse.parse('12.5')[0].tokens
- self.assertEqual(len(t), 1)
- self.assert_(t[0].ttype is sqlparse.tokens.Number.Float)
-
- def test_placeholder(self):
- def _get_tokens(sql):
- return sqlparse.parse(sql)[0].tokens[-1].tokens
- t = _get_tokens('select * from foo where user = ?')
- self.assert_(t[-1].ttype is sqlparse.tokens.Name.Placeholder)
- self.assertEqual(t[-1].value, '?')
- t = _get_tokens('select * from foo where user = :1')
- self.assert_(t[-1].ttype is sqlparse.tokens.Name.Placeholder)
- self.assertEqual(t[-1].value, ':1')
- t = _get_tokens('select * from foo where user = :name')
- self.assert_(t[-1].ttype is sqlparse.tokens.Name.Placeholder)
- self.assertEqual(t[-1].value, ':name')
- t = _get_tokens('select * from foo where user = %s')
- self.assert_(t[-1].ttype is sqlparse.tokens.Name.Placeholder)
- self.assertEqual(t[-1].value, '%s')
- t = _get_tokens('select * from foo where user = $a')
- self.assert_(t[-1].ttype is sqlparse.tokens.Name.Placeholder)
- self.assertEqual(t[-1].value, '$a')
-
- def test_modulo_not_placeholder(self):
- tokens = list(sqlparse.lexer.tokenize('x %3'))
- self.assertEqual(tokens[2][0], sqlparse.tokens.Operator)
-
- def test_access_symbol(self): # see issue27
- t = sqlparse.parse('select a.[foo bar] as foo')[0].tokens
- self.assert_(isinstance(t[-1], sqlparse.sql.Identifier))
- self.assertEqual(t[-1].get_name(), 'foo')
- self.assertEqual(t[-1].get_real_name(), '[foo bar]')
- self.assertEqual(t[-1].get_parent_name(), 'a')
-
- def test_keyword_like_identifier(self): # see issue47
- t = sqlparse.parse('foo.key')[0].tokens
- self.assertEqual(len(t), 1)
- self.assert_(isinstance(t[0], sqlparse.sql.Identifier))
-
- def test_function_parameter(self): # see issue94
- t = sqlparse.parse('abs(some_col)')[0].tokens[0].get_parameters()
- self.assertEqual(len(t), 1)
- self.assert_(isinstance(t[0], sqlparse.sql.Identifier))
-
- def test_function_param_single_literal(self):
- t = sqlparse.parse('foo(5)')[0].tokens[0].get_parameters()
- self.assertEqual(len(t), 1)
- self.assert_(t[0].ttype is T.Number.Integer)
-
- def test_nested_function(self):
- t = sqlparse.parse('foo(bar(5))')[0].tokens[0].get_parameters()
- self.assertEqual(len(t), 1)
- self.assert_(type(t[0]) is sqlparse.sql.Function)
-
-
-def test_quoted_identifier():
- t = sqlparse.parse('select x.y as "z" from foo')[0].tokens
- assert isinstance(t[2], sqlparse.sql.Identifier)
- assert t[2].get_name() == 'z'
- assert t[2].get_real_name() == 'y'
-
-
-def test_psql_quotation_marks(): # issue83
- # regression: make sure plain $$ work
- t = sqlparse.split("""
- CREATE OR REPLACE FUNCTION testfunc1(integer) RETURNS integer AS $$
- ....
- $$ LANGUAGE plpgsql;
- CREATE OR REPLACE FUNCTION testfunc2(integer) RETURNS integer AS $$
- ....
- $$ LANGUAGE plpgsql;""")
- assert len(t) == 2
- # make sure $SOMETHING$ works too
- t = sqlparse.split("""
- CREATE OR REPLACE FUNCTION testfunc1(integer) RETURNS integer AS $PROC_1$
- ....
- $PROC_1$ LANGUAGE plpgsql;
- CREATE OR REPLACE FUNCTION testfunc2(integer) RETURNS integer AS $PROC_2$
- ....
- $PROC_2$ LANGUAGE plpgsql;""")
- assert len(t) == 2
-
-
-@pytest.mark.parametrize('ph', ['?', ':1', ':foo', '%s', '%(foo)s'])
-def test_placeholder(ph):
- p = sqlparse.parse(ph)[0].tokens
- assert len(p) == 1
- assert p[0].ttype is T.Name.Placeholder
-
-
-@pytest.mark.parametrize('num', ['6.67428E-8', '1.988e33', '1e-12'])
-def test_scientific_numbers(num):
- p = sqlparse.parse(num)[0].tokens
- assert len(p) == 1
- assert p[0].ttype is T.Number.Float
-
-
-def test_single_quotes_are_strings():
- p = sqlparse.parse("'foo'")[0].tokens
- assert len(p) == 1
- assert p[0].ttype is T.String.Single
-
-
-def test_double_quotes_are_identifiers():
- p = sqlparse.parse('"foo"')[0].tokens
- assert len(p) == 1
- assert isinstance(p[0], sqlparse.sql.Identifier)
-
-
-def test_single_quotes_with_linebreaks(): # issue118
- p = sqlparse.parse("'f\nf'")[0].tokens
- assert len(p) == 1
- assert p[0].ttype is T.String.Single
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/tests/test_pipeline.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/tests/test_pipeline.py b/shell/ext-py/sqlparse-0.1.14/tests/test_pipeline.py
deleted file mode 100644
index 3442a5b..0000000
--- a/shell/ext-py/sqlparse-0.1.14/tests/test_pipeline.py
+++ /dev/null
@@ -1,70 +0,0 @@
-import unittest
-
-from sqlparse.filters import ColumnsSelect
-from sqlparse.lexer import tokenize
-from sqlparse.pipeline import Pipeline
-
-
-class Test(unittest.TestCase):
-
- def setUp(self):
- self.pipe = Pipeline()
- self.pipe.append(tokenize)
- self.pipe.append(ColumnsSelect())
-
- def test_1(self):
- sql = """
- -- type: script
- -- return: integer
-
- INCLUDE "Direntry.make.sql";
-
- INSERT INTO directories(inode)
- VALUES(:inode)
- LIMIT 1"""
- self.assertEqual([], self.pipe(sql))
-
- def test_2(self):
- sql = """
- SELECT child_entry,asdf AS inode, creation
- FROM links
- WHERE parent_dir == :parent_dir AND name == :name
- LIMIT 1"""
- self.assertEqual([u'child_entry', u'inode', u'creation'],
- self.pipe(sql))
-
- def test_3(self):
- sql = """
-SELECT
-0 AS st_dev,
-0 AS st_uid,
-0 AS st_gid,
-
-dir_entries.type AS st_mode,
-dir_entries.inode AS st_ino,
-COUNT(links.child_entry) AS st_nlink,
-
-:creation AS st_ctime,
-dir_entries.access AS st_atime,
-dir_entries.modification AS st_mtime,
--- :creation AS st_ctime,
--- CAST(STRFTIME('%s',dir_entries.access) AS INTEGER) AS st_atime,
--- CAST(STRFTIME('%s',dir_entries.modification) AS INTEGER) AS st_mtime,
-
-COALESCE(files.size,0) AS st_size, -- Python-FUSE
-COALESCE(files.size,0) AS size -- PyFilesystem
-
-FROM dir_entries
-LEFT JOIN files
-ON dir_entries.inode == files.inode
-LEFT JOIN links
-ON dir_entries.inode == links.child_entry
-
-WHERE dir_entries.inode == :inode
-
-GROUP BY dir_entries.inode
-LIMIT 1"""
- self.assertEqual([u'st_dev', u'st_uid', u'st_gid', u'st_mode',
- u'st_ino', u'st_nlink', u'st_ctime',
- u'st_atime', u'st_mtime', u'st_size', u'size'],
- self.pipe(sql))
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/tests/test_regressions.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/tests/test_regressions.py b/shell/ext-py/sqlparse-0.1.14/tests/test_regressions.py
deleted file mode 100644
index e30ddf0..0000000
--- a/shell/ext-py/sqlparse-0.1.14/tests/test_regressions.py
+++ /dev/null
@@ -1,246 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import sys
-
-from tests.utils import TestCaseBase, load_file
-
-import sqlparse
-from sqlparse import sql
-from sqlparse import tokens as T
-
-
-class RegressionTests(TestCaseBase):
-
- def test_issue9(self):
- # make sure where doesn't consume parenthesis
- p = sqlparse.parse('(where 1)')[0]
- self.assert_(isinstance(p, sql.Statement))
- self.assertEqual(len(p.tokens), 1)
- self.assert_(isinstance(p.tokens[0], sql.Parenthesis))
- prt = p.tokens[0]
- self.assertEqual(len(prt.tokens), 3)
- self.assertEqual(prt.tokens[0].ttype, T.Punctuation)
- self.assertEqual(prt.tokens[-1].ttype, T.Punctuation)
-
- def test_issue13(self):
- parsed = sqlparse.parse(("select 'one';\n"
- "select 'two\\'';\n"
- "select 'three';"))
- self.assertEqual(len(parsed), 3)
- self.assertEqual(str(parsed[1]).strip(), "select 'two\\'';")
-
- def test_issue26(self):
- # parse stand-alone comments
- p = sqlparse.parse('--hello')[0]
- self.assertEqual(len(p.tokens), 1)
- self.assert_(p.tokens[0].ttype is T.Comment.Single)
- p = sqlparse.parse('-- hello')[0]
- self.assertEqual(len(p.tokens), 1)
- self.assert_(p.tokens[0].ttype is T.Comment.Single)
- p = sqlparse.parse('--hello\n')[0]
- self.assertEqual(len(p.tokens), 1)
- self.assert_(p.tokens[0].ttype is T.Comment.Single)
- p = sqlparse.parse('--')[0]
- self.assertEqual(len(p.tokens), 1)
- self.assert_(p.tokens[0].ttype is T.Comment.Single)
- p = sqlparse.parse('--\n')[0]
- self.assertEqual(len(p.tokens), 1)
- self.assert_(p.tokens[0].ttype is T.Comment.Single)
-
- def test_issue34(self):
- t = sqlparse.parse("create")[0].token_first()
- self.assertEqual(t.match(T.Keyword.DDL, "create"), True)
- self.assertEqual(t.match(T.Keyword.DDL, "CREATE"), True)
-
- def test_issue35(self):
- # missing space before LIMIT
- sql = sqlparse.format("select * from foo where bar = 1 limit 1",
- reindent=True)
- self.ndiffAssertEqual(sql, "\n".join(["select *",
- "from foo",
- "where bar = 1 limit 1"]))
-
- def test_issue38(self):
- sql = sqlparse.format("SELECT foo; -- comment",
- strip_comments=True)
- self.ndiffAssertEqual(sql, "SELECT foo;")
- sql = sqlparse.format("/* foo */", strip_comments=True)
- self.ndiffAssertEqual(sql, "")
-
- def test_issue39(self):
- p = sqlparse.parse('select user.id from user')[0]
- self.assertEqual(len(p.tokens), 7)
- idt = p.tokens[2]
- self.assertEqual(idt.__class__, sql.Identifier)
- self.assertEqual(len(idt.tokens), 3)
- self.assertEqual(idt.tokens[0].match(T.Name, 'user'), True)
- self.assertEqual(idt.tokens[1].match(T.Punctuation, '.'), True)
- self.assertEqual(idt.tokens[2].match(T.Name, 'id'), True)
-
- def test_issue40(self):
- # make sure identifier lists in subselects are grouped
- p = sqlparse.parse(('SELECT id, name FROM '
- '(SELECT id, name FROM bar) as foo'))[0]
- self.assertEqual(len(p.tokens), 7)
- self.assertEqual(p.tokens[2].__class__, sql.IdentifierList)
- self.assertEqual(p.tokens[-1].__class__, sql.Identifier)
- self.assertEqual(p.tokens[-1].get_name(), u'foo')
- sp = p.tokens[-1].tokens[0]
- self.assertEqual(sp.tokens[3].__class__, sql.IdentifierList)
- # make sure that formatting works as expected
- self.ndiffAssertEqual(
- sqlparse.format(('SELECT id, name FROM '
- '(SELECT id, name FROM bar)'),
- reindent=True),
- ('SELECT id,\n'
- ' name\n'
- 'FROM\n'
- ' (SELECT id,\n'
- ' name\n'
- ' FROM bar)'))
- self.ndiffAssertEqual(
- sqlparse.format(('SELECT id, name FROM '
- '(SELECT id, name FROM bar) as foo'),
- reindent=True),
- ('SELECT id,\n'
- ' name\n'
- 'FROM\n'
- ' (SELECT id,\n'
- ' name\n'
- ' FROM bar) as foo'))
-
-
-def test_issue78():
- # the bug author provided this nice examples, let's use them!
- def _get_identifier(sql):
- p = sqlparse.parse(sql)[0]
- return p.tokens[2]
- results = (('get_name', 'z'),
- ('get_real_name', 'y'),
- ('get_parent_name', 'x'),
- ('get_alias', 'z'),
- ('get_typecast', 'text'))
- variants = (
- 'select x.y::text as z from foo',
- 'select x.y::text as "z" from foo',
- 'select x."y"::text as z from foo',
- 'select x."y"::text as "z" from foo',
- 'select "x".y::text as z from foo',
- 'select "x".y::text as "z" from foo',
- 'select "x"."y"::text as z from foo',
- 'select "x"."y"::text as "z" from foo',
- )
- for variant in variants:
- i = _get_identifier(variant)
- assert isinstance(i, sql.Identifier)
- for func_name, result in results:
- func = getattr(i, func_name)
- assert func() == result
-
-
-def test_issue83():
- sql = """
-CREATE OR REPLACE FUNCTION func_a(text)
- RETURNS boolean LANGUAGE plpgsql STRICT IMMUTABLE AS
-$_$
-BEGIN
- ...
-END;
-$_$;
-
-CREATE OR REPLACE FUNCTION func_b(text)
- RETURNS boolean LANGUAGE plpgsql STRICT IMMUTABLE AS
-$_$
-BEGIN
- ...
-END;
-$_$;
-
-ALTER TABLE..... ;"""
- t = sqlparse.split(sql)
- assert len(t) == 3
-
-
-def test_comment_encoding_when_reindent():
- # There was an UnicodeEncodeError in the reindent filter that
- # casted every comment followed by a keyword to str.
- sql = u'select foo -- Comment containing Ümläuts\nfrom bar'
- formatted = sqlparse.format(sql, reindent=True)
- assert formatted == sql
-
-
-def test_parse_sql_with_binary():
- # See https://github.com/andialbrecht/sqlparse/pull/88
- digest = '\x82|\xcb\x0e\xea\x8aplL4\xa1h\x91\xf8N{'
- sql = 'select * from foo where bar = \'%s\'' % digest
- formatted = sqlparse.format(sql, reindent=True)
- tformatted = 'select *\nfrom foo\nwhere bar = \'%s\'' % digest
- if sys.version_info < (3,):
- tformatted = tformatted.decode('unicode-escape')
- assert formatted == tformatted
-
-
-def test_dont_alias_keywords():
- # The _group_left_right function had a bug where the check for the
- # left side wasn't handled correctly. In one case this resulted in
- # a keyword turning into an identifier.
- p = sqlparse.parse('FROM AS foo')[0]
- assert len(p.tokens) == 5
- assert p.tokens[0].ttype is T.Keyword
- assert p.tokens[2].ttype is T.Keyword
-
-
-def test_format_accepts_encoding(): # issue20
- sql = load_file('test_cp1251.sql', 'cp1251')
- formatted = sqlparse.format(sql, reindent=True, encoding='cp1251')
- if sys.version_info < (3,):
- tformatted = u'insert into foo\nvalues (1); -- Песня про надежду\n'
- else:
- tformatted = 'insert into foo\nvalues (1); -- Песня про надежду\n'
- assert formatted == tformatted
-
-
-def test_issue90():
- sql = ('UPDATE "gallery_photo" SET "owner_id" = 4018, "deleted_at" = NULL,'
- ' "width" = NULL, "height" = NULL, "rating_votes" = 0,'
- ' "rating_score" = 0, "thumbnail_width" = NULL,'
- ' "thumbnail_height" = NULL, "price" = 1, "description" = NULL')
- formatted = sqlparse.format(sql, reindent=True)
- tformatted = '\n'.join(['UPDATE "gallery_photo"',
- 'SET "owner_id" = 4018,',
- ' "deleted_at" = NULL,',
- ' "width" = NULL,',
- ' "height" = NULL,',
- ' "rating_votes" = 0,',
- ' "rating_score" = 0,',
- ' "thumbnail_width" = NULL,',
- ' "thumbnail_height" = NULL,',
- ' "price" = 1,',
- ' "description" = NULL'])
- assert formatted == tformatted
-
-
-def test_except_formatting():
- sql = 'SELECT 1 FROM foo WHERE 2 = 3 EXCEPT SELECT 2 FROM bar WHERE 1 = 2'
- formatted = sqlparse.format(sql, reindent=True)
- tformatted = '\n'.join([
- 'SELECT 1',
- 'FROM foo',
- 'WHERE 2 = 3',
- 'EXCEPT',
- 'SELECT 2',
- 'FROM bar',
- 'WHERE 1 = 2'
- ])
- assert formatted == tformatted
-
-
-def test_null_with_as():
- sql = 'SELECT NULL AS c1, NULL AS c2 FROM t1'
- formatted = sqlparse.format(sql, reindent=True)
- tformatted = '\n'.join([
- 'SELECT NULL AS c1,',
- ' NULL AS c2',
- 'FROM t1'
- ])
- assert formatted == tformatted
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/tests/test_split.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/tests/test_split.py b/shell/ext-py/sqlparse-0.1.14/tests/test_split.py
deleted file mode 100644
index 54e8d04..0000000
--- a/shell/ext-py/sqlparse-0.1.14/tests/test_split.py
+++ /dev/null
@@ -1,144 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Tests splitting functions.
-
-import unittest
-
-from tests.utils import load_file, TestCaseBase
-
-import sqlparse
-
-
-class SQLSplitTest(TestCaseBase):
- """Tests sqlparse.sqlsplit()."""
-
- _sql1 = 'select * from foo;'
- _sql2 = 'select * from bar;'
-
- def test_split_semicolon(self):
- sql2 = 'select * from foo where bar = \'foo;bar\';'
- stmts = sqlparse.parse(''.join([self._sql1, sql2]))
- self.assertEqual(len(stmts), 2)
- self.ndiffAssertEqual(unicode(stmts[0]), self._sql1)
- self.ndiffAssertEqual(unicode(stmts[1]), sql2)
-
- def test_split_backslash(self):
- stmts = sqlparse.parse(r"select '\\'; select '\''; select '\\\'';")
- self.assertEqual(len(stmts), 3)
-
- def test_create_function(self):
- sql = load_file('function.sql')
- stmts = sqlparse.parse(sql)
- self.assertEqual(len(stmts), 1)
- self.ndiffAssertEqual(unicode(stmts[0]), sql)
-
- def test_create_function_psql(self):
- sql = load_file('function_psql.sql')
- stmts = sqlparse.parse(sql)
- self.assertEqual(len(stmts), 1)
- self.ndiffAssertEqual(unicode(stmts[0]), sql)
-
- def test_create_function_psql3(self):
- sql = load_file('function_psql3.sql')
- stmts = sqlparse.parse(sql)
- self.assertEqual(len(stmts), 1)
- self.ndiffAssertEqual(unicode(stmts[0]), sql)
-
- def test_create_function_psql2(self):
- sql = load_file('function_psql2.sql')
- stmts = sqlparse.parse(sql)
- self.assertEqual(len(stmts), 1)
- self.ndiffAssertEqual(unicode(stmts[0]), sql)
-
- def test_dashcomments(self):
- sql = load_file('dashcomment.sql')
- stmts = sqlparse.parse(sql)
- self.assertEqual(len(stmts), 3)
- self.ndiffAssertEqual(''.join(unicode(q) for q in stmts), sql)
-
- def test_dashcomments_eol(self):
- stmts = sqlparse.parse('select foo; -- comment\n')
- self.assertEqual(len(stmts), 1)
- stmts = sqlparse.parse('select foo; -- comment\r')
- self.assertEqual(len(stmts), 1)
- stmts = sqlparse.parse('select foo; -- comment\r\n')
- self.assertEqual(len(stmts), 1)
- stmts = sqlparse.parse('select foo; -- comment')
- self.assertEqual(len(stmts), 1)
-
- def test_begintag(self):
- sql = load_file('begintag.sql')
- stmts = sqlparse.parse(sql)
- self.assertEqual(len(stmts), 3)
- self.ndiffAssertEqual(''.join(unicode(q) for q in stmts), sql)
-
- def test_begintag_2(self):
- sql = load_file('begintag_2.sql')
- stmts = sqlparse.parse(sql)
- self.assertEqual(len(stmts), 1)
- self.ndiffAssertEqual(''.join(unicode(q) for q in stmts), sql)
-
- def test_dropif(self):
- sql = 'DROP TABLE IF EXISTS FOO;\n\nSELECT * FROM BAR;'
- stmts = sqlparse.parse(sql)
- self.assertEqual(len(stmts), 2)
- self.ndiffAssertEqual(''.join(unicode(q) for q in stmts), sql)
-
- def test_comment_with_umlaut(self):
- sql = (u'select * from foo;\n'
- u'-- Testing an umlaut: ä\n'
- u'select * from bar;')
- stmts = sqlparse.parse(sql)
- self.assertEqual(len(stmts), 2)
- self.ndiffAssertEqual(''.join(unicode(q) for q in stmts), sql)
-
- def test_comment_end_of_line(self):
- sql = ('select * from foo; -- foo\n'
- 'select * from bar;')
- stmts = sqlparse.parse(sql)
- self.assertEqual(len(stmts), 2)
- self.ndiffAssertEqual(''.join(unicode(q) for q in stmts), sql)
- # make sure the comment belongs to first query
- self.ndiffAssertEqual(unicode(stmts[0]), 'select * from foo; -- foo\n')
-
- def test_casewhen(self):
- sql = ('SELECT case when val = 1 then 2 else null end as foo;\n'
- 'comment on table actor is \'The actor table.\';')
- stmts = sqlparse.split(sql)
- self.assertEqual(len(stmts), 2)
-
- def test_cursor_declare(self):
- sql = ('DECLARE CURSOR "foo" AS SELECT 1;\n'
- 'SELECT 2;')
- stmts = sqlparse.split(sql)
- self.assertEqual(len(stmts), 2)
-
- def test_if_function(self): # see issue 33
- # don't let IF as a function confuse the splitter
- sql = ('CREATE TEMPORARY TABLE tmp '
- 'SELECT IF(a=1, a, b) AS o FROM one; '
- 'SELECT t FROM two')
- stmts = sqlparse.split(sql)
- self.assertEqual(len(stmts), 2)
-
- def test_split_stream(self):
- import types
- from cStringIO import StringIO
-
- stream = StringIO("SELECT 1; SELECT 2;")
- stmts = sqlparse.parsestream(stream)
- self.assertEqual(type(stmts), types.GeneratorType)
- self.assertEqual(len(list(stmts)), 2)
-
- def test_encoding_parsestream(self):
- from cStringIO import StringIO
- stream = StringIO("SELECT 1; SELECT 2;")
- stmts = list(sqlparse.parsestream(stream))
- self.assertEqual(type(stmts[0].tokens[0].value), unicode)
-
-
-def test_split_simple():
- stmts = sqlparse.split('select * from foo; select * from bar;')
- assert len(stmts) == 2
- assert stmts[0] == 'select * from foo;'
- assert stmts[1] == 'select * from bar;'
[7/8] impala git commit: IMPALA-6999: Upgrade to sqlparse-0.1.19 for
Impala shell
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/sqlparse/filters.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/sqlparse/filters.py b/shell/ext-py/sqlparse-0.1.14/sqlparse/filters.py
deleted file mode 100644
index e576a26..0000000
--- a/shell/ext-py/sqlparse-0.1.14/sqlparse/filters.py
+++ /dev/null
@@ -1,716 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import re
-
-from os.path import abspath, join
-
-from sqlparse import sql, tokens as T
-from sqlparse.engine import FilterStack
-from sqlparse.lexer import tokenize
-from sqlparse.pipeline import Pipeline
-from sqlparse.tokens import (Comment, Comparison, Keyword, Name, Punctuation,
- String, Whitespace)
-from sqlparse.utils import memoize_generator
-from sqlparse.utils import split_unquoted_newlines
-
-
-# --------------------------
-# token process
-
-class _CaseFilter:
-
- ttype = None
-
- def __init__(self, case=None):
- if case is None:
- case = 'upper'
- assert case in ['lower', 'upper', 'capitalize']
- self.convert = getattr(unicode, case)
-
- def process(self, stack, stream):
- for ttype, value in stream:
- if ttype in self.ttype:
- value = self.convert(value)
- yield ttype, value
-
-
-class KeywordCaseFilter(_CaseFilter):
- ttype = T.Keyword
-
-
-class IdentifierCaseFilter(_CaseFilter):
- ttype = (T.Name, T.String.Symbol)
-
- def process(self, stack, stream):
- for ttype, value in stream:
- if ttype in self.ttype and not value.strip()[0] == '"':
- value = self.convert(value)
- yield ttype, value
-
-
-class TruncateStringFilter:
-
- def __init__(self, width, char):
- self.width = max(width, 1)
- self.char = unicode(char)
-
- def process(self, stack, stream):
- for ttype, value in stream:
- if ttype is T.Literal.String.Single:
- if value[:2] == '\'\'':
- inner = value[2:-2]
- quote = u'\'\''
- else:
- inner = value[1:-1]
- quote = u'\''
- if len(inner) > self.width:
- value = u''.join((quote, inner[:self.width], self.char,
- quote))
- yield ttype, value
-
-
-class GetComments:
- """Get the comments from a stack"""
- def process(self, stack, stream):
- for token_type, value in stream:
- if token_type in Comment:
- yield token_type, value
-
-
-class StripComments:
- """Strip the comments from a stack"""
- def process(self, stack, stream):
- for token_type, value in stream:
- if token_type not in Comment:
- yield token_type, value
-
-
-def StripWhitespace(stream):
- "Strip the useless whitespaces from a stream leaving only the minimal ones"
- last_type = None
- has_space = False
- ignore_group = frozenset((Comparison, Punctuation))
-
- for token_type, value in stream:
- # We got a previous token (not empty first ones)
- if last_type:
- if token_type in Whitespace:
- has_space = True
- continue
-
- # Ignore first empty spaces and dot-commas
- elif token_type in (Whitespace, Whitespace.Newline, ignore_group):
- continue
-
- # Yield a whitespace if it can't be ignored
- if has_space:
- if not ignore_group.intersection((last_type, token_type)):
- yield Whitespace, ' '
- has_space = False
-
- # Yield the token and set its type for checking with the next one
- yield token_type, value
- last_type = token_type
-
-
-class IncludeStatement:
- """Filter that enable a INCLUDE statement"""
-
- def __init__(self, dirpath=".", maxrecursive=10, raiseexceptions=False):
- if maxrecursive <= 0:
- raise ValueError('Max recursion limit reached')
-
- self.dirpath = abspath(dirpath)
- self.maxRecursive = maxrecursive
- self.raiseexceptions = raiseexceptions
-
- self.detected = False
-
- @memoize_generator
- def process(self, stack, stream):
- # Run over all tokens in the stream
- for token_type, value in stream:
- # INCLUDE statement found, set detected mode
- if token_type in Name and value.upper() == 'INCLUDE':
- self.detected = True
- continue
-
- # INCLUDE statement was found, parse it
- elif self.detected:
- # Omit whitespaces
- if token_type in Whitespace:
- continue
-
- # Found file path to include
- if token_type in String.Symbol:
-# if token_type in tokens.String.Symbol:
-
- # Get path of file to include
- path = join(self.dirpath, value[1:-1])
-
- try:
- f = open(path)
- raw_sql = f.read()
- f.close()
-
- # There was a problem loading the include file
- except IOError, err:
- # Raise the exception to the interpreter
- if self.raiseexceptions:
- raise
-
- # Put the exception as a comment on the SQL code
- yield Comment, u'-- IOError: %s\n' % err
-
- else:
- # Create new FilterStack to parse readed file
- # and add all its tokens to the main stack recursively
- try:
- filtr = IncludeStatement(self.dirpath,
- self.maxRecursive - 1,
- self.raiseexceptions)
-
- # Max recursion limit reached
- except ValueError, err:
- # Raise the exception to the interpreter
- if self.raiseexceptions:
- raise
-
- # Put the exception as a comment on the SQL code
- yield Comment, u'-- ValueError: %s\n' % err
-
- stack = FilterStack()
- stack.preprocess.append(filtr)
-
- for tv in stack.run(raw_sql):
- yield tv
-
- # Set normal mode
- self.detected = False
-
- # Don't include any token while in detected mode
- continue
-
- # Normal token
- yield token_type, value
-
-
-# ----------------------
-# statement process
-
-class StripCommentsFilter:
-
- def _get_next_comment(self, tlist):
- # TODO(andi) Comment types should be unified, see related issue38
- token = tlist.token_next_by_instance(0, sql.Comment)
- if token is None:
- token = tlist.token_next_by_type(0, T.Comment)
- return token
-
- def _process(self, tlist):
- token = self._get_next_comment(tlist)
- while token:
- tidx = tlist.token_index(token)
- prev = tlist.token_prev(tidx, False)
- next_ = tlist.token_next(tidx, False)
- # Replace by whitespace if prev and next exist and if they're not
- # whitespaces. This doesn't apply if prev or next is a paranthesis.
- if (prev is not None and next_ is not None
- and not prev.is_whitespace() and not next_.is_whitespace()
- and not (prev.match(T.Punctuation, '(')
- or next_.match(T.Punctuation, ')'))):
- tlist.tokens[tidx] = sql.Token(T.Whitespace, ' ')
- else:
- tlist.tokens.pop(tidx)
- token = self._get_next_comment(tlist)
-
- def process(self, stack, stmt):
- [self.process(stack, sgroup) for sgroup in stmt.get_sublists()]
- self._process(stmt)
-
-
-class StripWhitespaceFilter:
-
- def _stripws(self, tlist):
- func_name = '_stripws_%s' % tlist.__class__.__name__.lower()
- func = getattr(self, func_name, self._stripws_default)
- func(tlist)
-
- def _stripws_default(self, tlist):
- last_was_ws = False
- for token in tlist.tokens:
- if token.is_whitespace():
- if last_was_ws:
- token.value = ''
- else:
- token.value = ' '
- last_was_ws = token.is_whitespace()
-
- def _stripws_identifierlist(self, tlist):
- # Removes newlines before commas, see issue140
- last_nl = None
- for token in tlist.tokens[:]:
- if (token.ttype is T.Punctuation
- and token.value == ','
- and last_nl is not None):
- tlist.tokens.remove(last_nl)
- if token.is_whitespace():
- last_nl = token
- else:
- last_nl = None
- return self._stripws_default(tlist)
-
- def _stripws_parenthesis(self, tlist):
- if tlist.tokens[1].is_whitespace():
- tlist.tokens.pop(1)
- if tlist.tokens[-2].is_whitespace():
- tlist.tokens.pop(-2)
- self._stripws_default(tlist)
-
- def process(self, stack, stmt, depth=0):
- [self.process(stack, sgroup, depth + 1)
- for sgroup in stmt.get_sublists()]
- self._stripws(stmt)
- if depth == 0 and stmt.tokens[-1].is_whitespace():
- stmt.tokens.pop(-1)
-
-
-class ReindentFilter:
-
- def __init__(self, width=2, char=' ', line_width=None):
- self.width = width
- self.char = char
- self.indent = 0
- self.offset = 0
- self.line_width = line_width
- self._curr_stmt = None
- self._last_stmt = None
-
- def _flatten_up_to_token(self, token):
- """Yields all tokens up to token plus the next one."""
- # helper for _get_offset
- iterator = self._curr_stmt.flatten()
- for t in iterator:
- yield t
- if t == token:
- raise StopIteration
-
- def _get_offset(self, token):
- raw = ''.join(map(unicode, self._flatten_up_to_token(token)))
- line = raw.splitlines()[-1]
- # Now take current offset into account and return relative offset.
- full_offset = len(line) - len(self.char * (self.width * self.indent))
- return full_offset - self.offset
-
- def nl(self):
- # TODO: newline character should be configurable
- space = (self.char * ((self.indent * self.width) + self.offset))
- # Detect runaway indenting due to parsing errors
- if len(space) > 200:
- # something seems to be wrong, flip back
- self.indent = self.offset = 0
- space = (self.char * ((self.indent * self.width) + self.offset))
- ws = '\n' + space
- return sql.Token(T.Whitespace, ws)
-
- def _split_kwds(self, tlist):
- split_words = ('FROM', 'STRAIGHT_JOIN$', 'JOIN$', 'AND', 'OR',
- 'GROUP', 'ORDER', 'UNION', 'VALUES',
- 'SET', 'BETWEEN', 'EXCEPT')
-
- def _next_token(i):
- t = tlist.token_next_match(i, T.Keyword, split_words,
- regex=True)
- if t and t.value.upper() == 'BETWEEN':
- t = _next_token(tlist.token_index(t) + 1)
- if t and t.value.upper() == 'AND':
- t = _next_token(tlist.token_index(t) + 1)
- return t
-
- idx = 0
- token = _next_token(idx)
- while token:
- prev = tlist.token_prev(tlist.token_index(token), False)
- offset = 1
- if prev and prev.is_whitespace():
- tlist.tokens.pop(tlist.token_index(prev))
- offset += 1
- if (prev
- and isinstance(prev, sql.Comment)
- and (unicode(prev).endswith('\n')
- or unicode(prev).endswith('\r'))):
- nl = tlist.token_next(token)
- else:
- nl = self.nl()
- tlist.insert_before(token, nl)
- token = _next_token(tlist.token_index(nl) + offset)
-
- def _split_statements(self, tlist):
- idx = 0
- token = tlist.token_next_by_type(idx, (T.Keyword.DDL, T.Keyword.DML))
- while token:
- prev = tlist.token_prev(tlist.token_index(token), False)
- if prev and prev.is_whitespace():
- tlist.tokens.pop(tlist.token_index(prev))
- # only break if it's not the first token
- if prev:
- nl = self.nl()
- tlist.insert_before(token, nl)
- token = tlist.token_next_by_type(tlist.token_index(token) + 1,
- (T.Keyword.DDL, T.Keyword.DML))
-
- def _process(self, tlist):
- func_name = '_process_%s' % tlist.__class__.__name__.lower()
- func = getattr(self, func_name, self._process_default)
- func(tlist)
-
- def _process_where(self, tlist):
- token = tlist.token_next_match(0, T.Keyword, 'WHERE')
- try:
- tlist.insert_before(token, self.nl())
- except ValueError: # issue121, errors in statement
- pass
- self.indent += 1
- self._process_default(tlist)
- self.indent -= 1
-
- def _process_parenthesis(self, tlist):
- first = tlist.token_next(0)
- indented = False
- if first and first.ttype in (T.Keyword.DML, T.Keyword.DDL):
- self.indent += 1
- tlist.tokens.insert(0, self.nl())
- indented = True
- num_offset = self._get_offset(
- tlist.token_next_match(0, T.Punctuation, '('))
- self.offset += num_offset
- self._process_default(tlist, stmts=not indented)
- if indented:
- self.indent -= 1
- self.offset -= num_offset
-
- def _process_identifierlist(self, tlist):
- identifiers = list(tlist.get_identifiers())
- if len(identifiers) > 1 and not tlist.within(sql.Function):
- first = list(identifiers[0].flatten())[0]
- if self.char == '\t':
- # when using tabs we don't count the actual word length
- # in spaces.
- num_offset = 1
- else:
- num_offset = self._get_offset(first) - len(first.value)
- self.offset += num_offset
- for token in identifiers[1:]:
- tlist.insert_before(token, self.nl())
- for token in tlist.tokens:
- if isinstance(token, sql.Comment):
- tlist.insert_after(token, self.nl())
- self.offset -= num_offset
- self._process_default(tlist)
-
- def _process_case(self, tlist):
- is_first = True
- num_offset = None
- case = tlist.tokens[0]
- outer_offset = self._get_offset(case) - len(case.value)
- self.offset += outer_offset
- for cond, value in tlist.get_cases():
- if is_first:
- tcond = list(cond[0].flatten())[0]
- is_first = False
- num_offset = self._get_offset(tcond) - len(tcond.value)
- self.offset += num_offset
- continue
- if cond is None:
- token = value[0]
- else:
- token = cond[0]
- tlist.insert_before(token, self.nl())
- # Line breaks on group level are done. Now let's add an offset of
- # 5 (=length of "when", "then", "else") and process subgroups.
- self.offset += 5
- self._process_default(tlist)
- self.offset -= 5
- if num_offset is not None:
- self.offset -= num_offset
- end = tlist.token_next_match(0, T.Keyword, 'END')
- tlist.insert_before(end, self.nl())
- self.offset -= outer_offset
-
- def _process_default(self, tlist, stmts=True, kwds=True):
- if stmts:
- self._split_statements(tlist)
- if kwds:
- self._split_kwds(tlist)
- [self._process(sgroup) for sgroup in tlist.get_sublists()]
-
- def process(self, stack, stmt):
- if isinstance(stmt, sql.Statement):
- self._curr_stmt = stmt
- self._process(stmt)
- if isinstance(stmt, sql.Statement):
- if self._last_stmt is not None:
- if unicode(self._last_stmt).endswith('\n'):
- nl = '\n'
- else:
- nl = '\n\n'
- stmt.tokens.insert(
- 0, sql.Token(T.Whitespace, nl))
- if self._last_stmt != stmt:
- self._last_stmt = stmt
-
-
-# FIXME: Doesn't work ;)
-class RightMarginFilter:
-
- keep_together = (
- # sql.TypeCast, sql.Identifier, sql.Alias,
- )
-
- def __init__(self, width=79):
- self.width = width
- self.line = ''
-
- def _process(self, stack, group, stream):
- for token in stream:
- if token.is_whitespace() and '\n' in token.value:
- if token.value.endswith('\n'):
- self.line = ''
- else:
- self.line = token.value.splitlines()[-1]
- elif (token.is_group()
- and not token.__class__ in self.keep_together):
- token.tokens = self._process(stack, token, token.tokens)
- else:
- val = unicode(token)
- if len(self.line) + len(val) > self.width:
- match = re.search('^ +', self.line)
- if match is not None:
- indent = match.group()
- else:
- indent = ''
- yield sql.Token(T.Whitespace, '\n%s' % indent)
- self.line = indent
- self.line += val
- yield token
-
- def process(self, stack, group):
- return
- group.tokens = self._process(stack, group, group.tokens)
-
-
-class ColumnsSelect:
- """Get the columns names of a SELECT query"""
- def process(self, stack, stream):
- mode = 0
- oldValue = ""
- parenthesis = 0
-
- for token_type, value in stream:
- # Ignore comments
- if token_type in Comment:
- continue
-
- # We have not detected a SELECT statement
- if mode == 0:
- if token_type in Keyword and value == 'SELECT':
- mode = 1
-
- # We have detected a SELECT statement
- elif mode == 1:
- if value == 'FROM':
- if oldValue:
- yield oldValue
-
- mode = 3 # Columns have been checked
-
- elif value == 'AS':
- oldValue = ""
- mode = 2
-
- elif (token_type == Punctuation
- and value == ',' and not parenthesis):
- if oldValue:
- yield oldValue
- oldValue = ""
-
- elif token_type not in Whitespace:
- if value == '(':
- parenthesis += 1
- elif value == ')':
- parenthesis -= 1
-
- oldValue += value
-
- # We are processing an AS keyword
- elif mode == 2:
- # We check also for Keywords because a bug in SQLParse
- if token_type == Name or token_type == Keyword:
- yield value
- mode = 1
-
-
-# ---------------------------
-# postprocess
-
-class SerializerUnicode:
-
- def process(self, stack, stmt):
- raw = unicode(stmt)
- lines = split_unquoted_newlines(raw)
- res = '\n'.join(line.rstrip() for line in lines)
- return res
-
-
-def Tokens2Unicode(stream):
- result = ""
-
- for _, value in stream:
- result += unicode(value)
-
- return result
-
-
-class OutputFilter:
- varname_prefix = ''
-
- def __init__(self, varname='sql'):
- self.varname = self.varname_prefix + varname
- self.count = 0
-
- def _process(self, stream, varname, has_nl):
- raise NotImplementedError
-
- def process(self, stack, stmt):
- self.count += 1
- if self.count > 1:
- varname = '%s%d' % (self.varname, self.count)
- else:
- varname = self.varname
-
- has_nl = len(unicode(stmt).strip().splitlines()) > 1
- stmt.tokens = self._process(stmt.tokens, varname, has_nl)
- return stmt
-
-
-class OutputPythonFilter(OutputFilter):
- def _process(self, stream, varname, has_nl):
- # SQL query asignation to varname
- if self.count > 1:
- yield sql.Token(T.Whitespace, '\n')
- yield sql.Token(T.Name, varname)
- yield sql.Token(T.Whitespace, ' ')
- yield sql.Token(T.Operator, '=')
- yield sql.Token(T.Whitespace, ' ')
- if has_nl:
- yield sql.Token(T.Operator, '(')
- yield sql.Token(T.Text, "'")
-
- # Print the tokens on the quote
- for token in stream:
- # Token is a new line separator
- if token.is_whitespace() and '\n' in token.value:
- # Close quote and add a new line
- yield sql.Token(T.Text, " '")
- yield sql.Token(T.Whitespace, '\n')
-
- # Quote header on secondary lines
- yield sql.Token(T.Whitespace, ' ' * (len(varname) + 4))
- yield sql.Token(T.Text, "'")
-
- # Indentation
- after_lb = token.value.split('\n', 1)[1]
- if after_lb:
- yield sql.Token(T.Whitespace, after_lb)
- continue
-
- # Token has escape chars
- elif "'" in token.value:
- token.value = token.value.replace("'", "\\'")
-
- # Put the token
- yield sql.Token(T.Text, token.value)
-
- # Close quote
- yield sql.Token(T.Text, "'")
- if has_nl:
- yield sql.Token(T.Operator, ')')
-
-
-class OutputPHPFilter(OutputFilter):
- varname_prefix = '$'
-
- def _process(self, stream, varname, has_nl):
- # SQL query asignation to varname (quote header)
- if self.count > 1:
- yield sql.Token(T.Whitespace, '\n')
- yield sql.Token(T.Name, varname)
- yield sql.Token(T.Whitespace, ' ')
- if has_nl:
- yield sql.Token(T.Whitespace, ' ')
- yield sql.Token(T.Operator, '=')
- yield sql.Token(T.Whitespace, ' ')
- yield sql.Token(T.Text, '"')
-
- # Print the tokens on the quote
- for token in stream:
- # Token is a new line separator
- if token.is_whitespace() and '\n' in token.value:
- # Close quote and add a new line
- yield sql.Token(T.Text, ' ";')
- yield sql.Token(T.Whitespace, '\n')
-
- # Quote header on secondary lines
- yield sql.Token(T.Name, varname)
- yield sql.Token(T.Whitespace, ' ')
- yield sql.Token(T.Operator, '.=')
- yield sql.Token(T.Whitespace, ' ')
- yield sql.Token(T.Text, '"')
-
- # Indentation
- after_lb = token.value.split('\n', 1)[1]
- if after_lb:
- yield sql.Token(T.Whitespace, after_lb)
- continue
-
- # Token has escape chars
- elif '"' in token.value:
- token.value = token.value.replace('"', '\\"')
-
- # Put the token
- yield sql.Token(T.Text, token.value)
-
- # Close quote
- yield sql.Token(T.Text, '"')
- yield sql.Token(T.Punctuation, ';')
-
-
-class Limit:
- """Get the LIMIT of a query.
-
- If not defined, return -1 (SQL specification for no LIMIT query)
- """
- def process(self, stack, stream):
- index = 7
- stream = list(stream)
- stream.reverse()
-
- # Run over all tokens in the stream from the end
- for token_type, value in stream:
- index -= 1
-
-# if index and token_type in Keyword:
- if index and token_type in Keyword and value == 'LIMIT':
- return stream[4 - index][1]
-
- return -1
-
-
-def compact(stream):
- """Function that return a compacted version of the stream"""
- pipe = Pipeline()
-
- pipe.append(StripComments())
- pipe.append(StripWhitespace)
-
- return pipe(stream)
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/sqlparse/formatter.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/sqlparse/formatter.py b/shell/ext-py/sqlparse-0.1.14/sqlparse/formatter.py
deleted file mode 100644
index 811f5af..0000000
--- a/shell/ext-py/sqlparse-0.1.14/sqlparse/formatter.py
+++ /dev/null
@@ -1,137 +0,0 @@
-# Copyright (C) 2008 Andi Albrecht, albrecht.andi@gmail.com
-#
-# This module is part of python-sqlparse and is released under
-# the BSD License: http://www.opensource.org/licenses/bsd-license.php.
-
-"""SQL formatter"""
-
-from sqlparse import filters
-from sqlparse.exceptions import SQLParseError
-
-
-def validate_options(options):
- """Validates options."""
- kwcase = options.get('keyword_case', None)
- if kwcase not in [None, 'upper', 'lower', 'capitalize']:
- raise SQLParseError('Invalid value for keyword_case: %r' % kwcase)
-
- idcase = options.get('identifier_case', None)
- if idcase not in [None, 'upper', 'lower', 'capitalize']:
- raise SQLParseError('Invalid value for identifier_case: %r' % idcase)
-
- ofrmt = options.get('output_format', None)
- if ofrmt not in [None, 'sql', 'python', 'php']:
- raise SQLParseError('Unknown output format: %r' % ofrmt)
-
- strip_comments = options.get('strip_comments', False)
- if strip_comments not in [True, False]:
- raise SQLParseError('Invalid value for strip_comments: %r'
- % strip_comments)
-
- strip_ws = options.get('strip_whitespace', False)
- if strip_ws not in [True, False]:
- raise SQLParseError('Invalid value for strip_whitespace: %r'
- % strip_ws)
-
- truncate_strings = options.get('truncate_strings', None)
- if truncate_strings is not None:
- try:
- truncate_strings = int(truncate_strings)
- except (ValueError, TypeError):
- raise SQLParseError('Invalid value for truncate_strings: %r'
- % truncate_strings)
- if truncate_strings <= 1:
- raise SQLParseError('Invalid value for truncate_strings: %r'
- % truncate_strings)
- options['truncate_strings'] = truncate_strings
- options['truncate_char'] = options.get('truncate_char', '[...]')
-
- reindent = options.get('reindent', False)
- if reindent not in [True, False]:
- raise SQLParseError('Invalid value for reindent: %r'
- % reindent)
- elif reindent:
- options['strip_whitespace'] = True
- indent_tabs = options.get('indent_tabs', False)
- if indent_tabs not in [True, False]:
- raise SQLParseError('Invalid value for indent_tabs: %r' % indent_tabs)
- elif indent_tabs:
- options['indent_char'] = '\t'
- else:
- options['indent_char'] = ' '
- indent_width = options.get('indent_width', 2)
- try:
- indent_width = int(indent_width)
- except (TypeError, ValueError):
- raise SQLParseError('indent_width requires an integer')
- if indent_width < 1:
- raise SQLParseError('indent_width requires an positive integer')
- options['indent_width'] = indent_width
-
- right_margin = options.get('right_margin', None)
- if right_margin is not None:
- try:
- right_margin = int(right_margin)
- except (TypeError, ValueError):
- raise SQLParseError('right_margin requires an integer')
- if right_margin < 10:
- raise SQLParseError('right_margin requires an integer > 10')
- options['right_margin'] = right_margin
-
- return options
-
-
-def build_filter_stack(stack, options):
- """Setup and return a filter stack.
-
- Args:
- stack: :class:`~sqlparse.filters.FilterStack` instance
- options: Dictionary with options validated by validate_options.
- """
- # Token filter
- if options.get('keyword_case', None):
- stack.preprocess.append(
- filters.KeywordCaseFilter(options['keyword_case']))
-
- if options.get('identifier_case', None):
- stack.preprocess.append(
- filters.IdentifierCaseFilter(options['identifier_case']))
-
- if options.get('truncate_strings', None) is not None:
- stack.preprocess.append(filters.TruncateStringFilter(
- width=options['truncate_strings'], char=options['truncate_char']))
-
- # After grouping
- if options.get('strip_comments', False):
- stack.enable_grouping()
- stack.stmtprocess.append(filters.StripCommentsFilter())
-
- if (options.get('strip_whitespace', False)
- or options.get('reindent', False)):
- stack.enable_grouping()
- stack.stmtprocess.append(filters.StripWhitespaceFilter())
-
- if options.get('reindent', False):
- stack.enable_grouping()
- stack.stmtprocess.append(
- filters.ReindentFilter(char=options['indent_char'],
- width=options['indent_width']))
-
- if options.get('right_margin', False):
- stack.enable_grouping()
- stack.stmtprocess.append(
- filters.RightMarginFilter(width=options['right_margin']))
-
- # Serializer
- if options.get('output_format'):
- frmt = options['output_format']
- if frmt.lower() == 'php':
- fltr = filters.OutputPHPFilter()
- elif frmt.lower() == 'python':
- fltr = filters.OutputPythonFilter()
- else:
- fltr = None
- if fltr is not None:
- stack.postprocess.append(fltr)
-
- return stack
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/sqlparse/functions.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/sqlparse/functions.py b/shell/ext-py/sqlparse-0.1.14/sqlparse/functions.py
deleted file mode 100644
index e54457e..0000000
--- a/shell/ext-py/sqlparse-0.1.14/sqlparse/functions.py
+++ /dev/null
@@ -1,44 +0,0 @@
-'''
-Created on 17/05/2012
-
-@author: piranna
-
-Several utility functions to extract info from the SQL sentences
-'''
-
-from sqlparse.filters import ColumnsSelect, Limit
-from sqlparse.pipeline import Pipeline
-from sqlparse.tokens import Keyword, Whitespace
-
-
-def getlimit(stream):
- """Function that return the LIMIT of a input SQL """
- pipe = Pipeline()
-
- pipe.append(Limit())
-
- result = pipe(stream)
- try:
- return int(result)
- except ValueError:
- return result
-
-
-def getcolumns(stream):
- """Function that return the colums of a SELECT query"""
- pipe = Pipeline()
-
- pipe.append(ColumnsSelect())
-
- return pipe(stream)
-
-
-class IsType(object):
- """Functor that return is the statement is of a specific type"""
- def __init__(self, type):
- self.type = type
-
- def __call__(self, stream):
- for token_type, value in stream:
- if token_type not in Whitespace:
- return token_type in Keyword and value == self.type
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/sqlparse/keywords.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/sqlparse/keywords.py b/shell/ext-py/sqlparse-0.1.14/sqlparse/keywords.py
deleted file mode 100644
index 0fb7c00..0000000
--- a/shell/ext-py/sqlparse-0.1.14/sqlparse/keywords.py
+++ /dev/null
@@ -1,571 +0,0 @@
-from sqlparse import tokens
-
-KEYWORDS = {
- 'ABORT': tokens.Keyword,
- 'ABS': tokens.Keyword,
- 'ABSOLUTE': tokens.Keyword,
- 'ACCESS': tokens.Keyword,
- 'ADA': tokens.Keyword,
- 'ADD': tokens.Keyword,
- 'ADMIN': tokens.Keyword,
- 'AFTER': tokens.Keyword,
- 'AGGREGATE': tokens.Keyword,
- 'ALIAS': tokens.Keyword,
- 'ALL': tokens.Keyword,
- 'ALLOCATE': tokens.Keyword,
- 'ANALYSE': tokens.Keyword,
- 'ANALYZE': tokens.Keyword,
- 'ANY': tokens.Keyword,
- 'ARE': tokens.Keyword,
- 'ASC': tokens.Keyword.Order,
- 'ASENSITIVE': tokens.Keyword,
- 'ASSERTION': tokens.Keyword,
- 'ASSIGNMENT': tokens.Keyword,
- 'ASYMMETRIC': tokens.Keyword,
- 'AT': tokens.Keyword,
- 'ATOMIC': tokens.Keyword,
- 'AUTHORIZATION': tokens.Keyword,
- 'AVG': tokens.Keyword,
-
- 'BACKWARD': tokens.Keyword,
- 'BEFORE': tokens.Keyword,
- 'BEGIN': tokens.Keyword,
- 'BETWEEN': tokens.Keyword,
- 'BITVAR': tokens.Keyword,
- 'BIT_LENGTH': tokens.Keyword,
- 'BOTH': tokens.Keyword,
- 'BREADTH': tokens.Keyword,
-
- # 'C': tokens.Keyword, # most likely this is an alias
- 'CACHE': tokens.Keyword,
- 'CALL': tokens.Keyword,
- 'CALLED': tokens.Keyword,
- 'CARDINALITY': tokens.Keyword,
- 'CASCADE': tokens.Keyword,
- 'CASCADED': tokens.Keyword,
- 'CAST': tokens.Keyword,
- 'CATALOG': tokens.Keyword,
- 'CATALOG_NAME': tokens.Keyword,
- 'CHAIN': tokens.Keyword,
- 'CHARACTERISTICS': tokens.Keyword,
- 'CHARACTER_LENGTH': tokens.Keyword,
- 'CHARACTER_SET_CATALOG': tokens.Keyword,
- 'CHARACTER_SET_NAME': tokens.Keyword,
- 'CHARACTER_SET_SCHEMA': tokens.Keyword,
- 'CHAR_LENGTH': tokens.Keyword,
- 'CHECK': tokens.Keyword,
- 'CHECKED': tokens.Keyword,
- 'CHECKPOINT': tokens.Keyword,
- 'CLASS': tokens.Keyword,
- 'CLASS_ORIGIN': tokens.Keyword,
- 'CLOB': tokens.Keyword,
- 'CLOSE': tokens.Keyword,
- 'CLUSTER': tokens.Keyword,
- 'COALESCE': tokens.Keyword,
- 'COBOL': tokens.Keyword,
- 'COLLATE': tokens.Keyword,
- 'COLLATION': tokens.Keyword,
- 'COLLATION_CATALOG': tokens.Keyword,
- 'COLLATION_NAME': tokens.Keyword,
- 'COLLATION_SCHEMA': tokens.Keyword,
- 'COLLECT': tokens.Keyword,
- 'COLUMN': tokens.Keyword,
- 'COLUMN_NAME': tokens.Keyword,
- 'COMMAND_FUNCTION': tokens.Keyword,
- 'COMMAND_FUNCTION_CODE': tokens.Keyword,
- 'COMMENT': tokens.Keyword,
- 'COMMIT': tokens.Keyword.DML,
- 'COMMITTED': tokens.Keyword,
- 'COMPLETION': tokens.Keyword,
- 'CONDITION_NUMBER': tokens.Keyword,
- 'CONNECT': tokens.Keyword,
- 'CONNECTION': tokens.Keyword,
- 'CONNECTION_NAME': tokens.Keyword,
- 'CONSTRAINT': tokens.Keyword,
- 'CONSTRAINTS': tokens.Keyword,
- 'CONSTRAINT_CATALOG': tokens.Keyword,
- 'CONSTRAINT_NAME': tokens.Keyword,
- 'CONSTRAINT_SCHEMA': tokens.Keyword,
- 'CONSTRUCTOR': tokens.Keyword,
- 'CONTAINS': tokens.Keyword,
- 'CONTINUE': tokens.Keyword,
- 'CONVERSION': tokens.Keyword,
- 'CONVERT': tokens.Keyword,
- 'COPY': tokens.Keyword,
- 'CORRESPONTING': tokens.Keyword,
- 'COUNT': tokens.Keyword,
- 'CREATEDB': tokens.Keyword,
- 'CREATEUSER': tokens.Keyword,
- 'CROSS': tokens.Keyword,
- 'CUBE': tokens.Keyword,
- 'CURRENT': tokens.Keyword,
- 'CURRENT_DATE': tokens.Keyword,
- 'CURRENT_PATH': tokens.Keyword,
- 'CURRENT_ROLE': tokens.Keyword,
- 'CURRENT_TIME': tokens.Keyword,
- 'CURRENT_TIMESTAMP': tokens.Keyword,
- 'CURRENT_USER': tokens.Keyword,
- 'CURSOR': tokens.Keyword,
- 'CURSOR_NAME': tokens.Keyword,
- 'CYCLE': tokens.Keyword,
-
- 'DATA': tokens.Keyword,
- 'DATABASE': tokens.Keyword,
- 'DATETIME_INTERVAL_CODE': tokens.Keyword,
- 'DATETIME_INTERVAL_PRECISION': tokens.Keyword,
- 'DAY': tokens.Keyword,
- 'DEALLOCATE': tokens.Keyword,
- 'DECLARE': tokens.Keyword,
- 'DEFAULT': tokens.Keyword,
- 'DEFAULTS': tokens.Keyword,
- 'DEFERRABLE': tokens.Keyword,
- 'DEFERRED': tokens.Keyword,
- 'DEFINED': tokens.Keyword,
- 'DEFINER': tokens.Keyword,
- 'DELIMITER': tokens.Keyword,
- 'DELIMITERS': tokens.Keyword,
- 'DEREF': tokens.Keyword,
- 'DESC': tokens.Keyword.Order,
- 'DESCRIBE': tokens.Keyword,
- 'DESCRIPTOR': tokens.Keyword,
- 'DESTROY': tokens.Keyword,
- 'DESTRUCTOR': tokens.Keyword,
- 'DETERMINISTIC': tokens.Keyword,
- 'DIAGNOSTICS': tokens.Keyword,
- 'DICTIONARY': tokens.Keyword,
- 'DISCONNECT': tokens.Keyword,
- 'DISPATCH': tokens.Keyword,
- 'DO': tokens.Keyword,
- 'DOMAIN': tokens.Keyword,
- 'DYNAMIC': tokens.Keyword,
- 'DYNAMIC_FUNCTION': tokens.Keyword,
- 'DYNAMIC_FUNCTION_CODE': tokens.Keyword,
-
- 'EACH': tokens.Keyword,
- 'ENCODING': tokens.Keyword,
- 'ENCRYPTED': tokens.Keyword,
- 'END-EXEC': tokens.Keyword,
- 'EQUALS': tokens.Keyword,
- 'ESCAPE': tokens.Keyword,
- 'EVERY': tokens.Keyword,
- 'EXCEPT': tokens.Keyword,
- 'ESCEPTION': tokens.Keyword,
- 'EXCLUDING': tokens.Keyword,
- 'EXCLUSIVE': tokens.Keyword,
- 'EXEC': tokens.Keyword,
- 'EXECUTE': tokens.Keyword,
- 'EXISTING': tokens.Keyword,
- 'EXISTS': tokens.Keyword,
- 'EXTERNAL': tokens.Keyword,
- 'EXTRACT': tokens.Keyword,
-
- 'FALSE': tokens.Keyword,
- 'FETCH': tokens.Keyword,
- 'FINAL': tokens.Keyword,
- 'FIRST': tokens.Keyword,
- 'FORCE': tokens.Keyword,
- 'FOREACH': tokens.Keyword,
- 'FOREIGN': tokens.Keyword,
- 'FORTRAN': tokens.Keyword,
- 'FORWARD': tokens.Keyword,
- 'FOUND': tokens.Keyword,
- 'FREE': tokens.Keyword,
- 'FREEZE': tokens.Keyword,
- 'FULL': tokens.Keyword,
- 'FUNCTION': tokens.Keyword,
-
- # 'G': tokens.Keyword,
- 'GENERAL': tokens.Keyword,
- 'GENERATED': tokens.Keyword,
- 'GET': tokens.Keyword,
- 'GLOBAL': tokens.Keyword,
- 'GO': tokens.Keyword,
- 'GOTO': tokens.Keyword,
- 'GRANT': tokens.Keyword,
- 'GRANTED': tokens.Keyword,
- 'GROUPING': tokens.Keyword,
-
- 'HANDLER': tokens.Keyword,
- 'HAVING': tokens.Keyword,
- 'HIERARCHY': tokens.Keyword,
- 'HOLD': tokens.Keyword,
- 'HOST': tokens.Keyword,
-
- 'IDENTITY': tokens.Keyword,
- 'IGNORE': tokens.Keyword,
- 'ILIKE': tokens.Keyword,
- 'IMMEDIATE': tokens.Keyword,
- 'IMMUTABLE': tokens.Keyword,
-
- 'IMPLEMENTATION': tokens.Keyword,
- 'IMPLICIT': tokens.Keyword,
- 'INCLUDING': tokens.Keyword,
- 'INCREMENT': tokens.Keyword,
- 'INDEX': tokens.Keyword,
-
- 'INDITCATOR': tokens.Keyword,
- 'INFIX': tokens.Keyword,
- 'INHERITS': tokens.Keyword,
- 'INITIALIZE': tokens.Keyword,
- 'INITIALLY': tokens.Keyword,
- 'INOUT': tokens.Keyword,
- 'INPUT': tokens.Keyword,
- 'INSENSITIVE': tokens.Keyword,
- 'INSTANTIABLE': tokens.Keyword,
- 'INSTEAD': tokens.Keyword,
- 'INTERSECT': tokens.Keyword,
- 'INTO': tokens.Keyword,
- 'INVOKER': tokens.Keyword,
- 'IS': tokens.Keyword,
- 'ISNULL': tokens.Keyword,
- 'ISOLATION': tokens.Keyword,
- 'ITERATE': tokens.Keyword,
-
- # 'K': tokens.Keyword,
- 'KEY': tokens.Keyword,
- 'KEY_MEMBER': tokens.Keyword,
- 'KEY_TYPE': tokens.Keyword,
-
- 'LANCOMPILER': tokens.Keyword,
- 'LANGUAGE': tokens.Keyword,
- 'LARGE': tokens.Keyword,
- 'LAST': tokens.Keyword,
- 'LATERAL': tokens.Keyword,
- 'LEADING': tokens.Keyword,
- 'LENGTH': tokens.Keyword,
- 'LESS': tokens.Keyword,
- 'LEVEL': tokens.Keyword,
- 'LIMIT': tokens.Keyword,
- 'LISTEN': tokens.Keyword,
- 'LOAD': tokens.Keyword,
- 'LOCAL': tokens.Keyword,
- 'LOCALTIME': tokens.Keyword,
- 'LOCALTIMESTAMP': tokens.Keyword,
- 'LOCATION': tokens.Keyword,
- 'LOCATOR': tokens.Keyword,
- 'LOCK': tokens.Keyword,
- 'LOWER': tokens.Keyword,
-
- # 'M': tokens.Keyword,
- 'MAP': tokens.Keyword,
- 'MATCH': tokens.Keyword,
- 'MAXVALUE': tokens.Keyword,
- 'MESSAGE_LENGTH': tokens.Keyword,
- 'MESSAGE_OCTET_LENGTH': tokens.Keyword,
- 'MESSAGE_TEXT': tokens.Keyword,
- 'METHOD': tokens.Keyword,
- 'MINUTE': tokens.Keyword,
- 'MINVALUE': tokens.Keyword,
- 'MOD': tokens.Keyword,
- 'MODE': tokens.Keyword,
- 'MODIFIES': tokens.Keyword,
- 'MODIFY': tokens.Keyword,
- 'MONTH': tokens.Keyword,
- 'MORE': tokens.Keyword,
- 'MOVE': tokens.Keyword,
- 'MUMPS': tokens.Keyword,
-
- 'NAMES': tokens.Keyword,
- 'NATIONAL': tokens.Keyword,
- 'NATURAL': tokens.Keyword,
- 'NCHAR': tokens.Keyword,
- 'NCLOB': tokens.Keyword,
- 'NEW': tokens.Keyword,
- 'NEXT': tokens.Keyword,
- 'NO': tokens.Keyword,
- 'NOCREATEDB': tokens.Keyword,
- 'NOCREATEUSER': tokens.Keyword,
- 'NONE': tokens.Keyword,
- 'NOT': tokens.Keyword,
- 'NOTHING': tokens.Keyword,
- 'NOTIFY': tokens.Keyword,
- 'NOTNULL': tokens.Keyword,
- 'NULL': tokens.Keyword,
- 'NULLABLE': tokens.Keyword,
- 'NULLIF': tokens.Keyword,
-
- 'OBJECT': tokens.Keyword,
- 'OCTET_LENGTH': tokens.Keyword,
- 'OF': tokens.Keyword,
- 'OFF': tokens.Keyword,
- 'OFFSET': tokens.Keyword,
- 'OIDS': tokens.Keyword,
- 'OLD': tokens.Keyword,
- 'ONLY': tokens.Keyword,
- 'OPEN': tokens.Keyword,
- 'OPERATION': tokens.Keyword,
- 'OPERATOR': tokens.Keyword,
- 'OPTION': tokens.Keyword,
- 'OPTIONS': tokens.Keyword,
- 'ORDINALITY': tokens.Keyword,
- 'OUT': tokens.Keyword,
- 'OUTPUT': tokens.Keyword,
- 'OVERLAPS': tokens.Keyword,
- 'OVERLAY': tokens.Keyword,
- 'OVERRIDING': tokens.Keyword,
- 'OWNER': tokens.Keyword,
-
- 'PAD': tokens.Keyword,
- 'PARAMETER': tokens.Keyword,
- 'PARAMETERS': tokens.Keyword,
- 'PARAMETER_MODE': tokens.Keyword,
- 'PARAMATER_NAME': tokens.Keyword,
- 'PARAMATER_ORDINAL_POSITION': tokens.Keyword,
- 'PARAMETER_SPECIFIC_CATALOG': tokens.Keyword,
- 'PARAMETER_SPECIFIC_NAME': tokens.Keyword,
- 'PARAMATER_SPECIFIC_SCHEMA': tokens.Keyword,
- 'PARTIAL': tokens.Keyword,
- 'PASCAL': tokens.Keyword,
- 'PENDANT': tokens.Keyword,
- 'PLACING': tokens.Keyword,
- 'PLI': tokens.Keyword,
- 'POSITION': tokens.Keyword,
- 'POSTFIX': tokens.Keyword,
- 'PRECISION': tokens.Keyword,
- 'PREFIX': tokens.Keyword,
- 'PREORDER': tokens.Keyword,
- 'PREPARE': tokens.Keyword,
- 'PRESERVE': tokens.Keyword,
- 'PRIMARY': tokens.Keyword,
- 'PRIOR': tokens.Keyword,
- 'PRIVILEGES': tokens.Keyword,
- 'PROCEDURAL': tokens.Keyword,
- 'PROCEDURE': tokens.Keyword,
- 'PUBLIC': tokens.Keyword,
-
- 'RAISE': tokens.Keyword,
- 'READ': tokens.Keyword,
- 'READS': tokens.Keyword,
- 'RECHECK': tokens.Keyword,
- 'RECURSIVE': tokens.Keyword,
- 'REF': tokens.Keyword,
- 'REFERENCES': tokens.Keyword,
- 'REFERENCING': tokens.Keyword,
- 'REINDEX': tokens.Keyword,
- 'RELATIVE': tokens.Keyword,
- 'RENAME': tokens.Keyword,
- 'REPEATABLE': tokens.Keyword,
- 'RESET': tokens.Keyword,
- 'RESTART': tokens.Keyword,
- 'RESTRICT': tokens.Keyword,
- 'RESULT': tokens.Keyword,
- 'RETURN': tokens.Keyword,
- 'RETURNED_LENGTH': tokens.Keyword,
- 'RETURNED_OCTET_LENGTH': tokens.Keyword,
- 'RETURNED_SQLSTATE': tokens.Keyword,
- 'RETURNS': tokens.Keyword,
- 'REVOKE': tokens.Keyword,
- 'RIGHT': tokens.Keyword,
- 'ROLE': tokens.Keyword,
- 'ROLLBACK': tokens.Keyword.DML,
- 'ROLLUP': tokens.Keyword,
- 'ROUTINE': tokens.Keyword,
- 'ROUTINE_CATALOG': tokens.Keyword,
- 'ROUTINE_NAME': tokens.Keyword,
- 'ROUTINE_SCHEMA': tokens.Keyword,
- 'ROW': tokens.Keyword,
- 'ROWS': tokens.Keyword,
- 'ROW_COUNT': tokens.Keyword,
- 'RULE': tokens.Keyword,
-
- 'SAVE_POINT': tokens.Keyword,
- 'SCALE': tokens.Keyword,
- 'SCHEMA': tokens.Keyword,
- 'SCHEMA_NAME': tokens.Keyword,
- 'SCOPE': tokens.Keyword,
- 'SCROLL': tokens.Keyword,
- 'SEARCH': tokens.Keyword,
- 'SECOND': tokens.Keyword,
- 'SECURITY': tokens.Keyword,
- 'SELF': tokens.Keyword,
- 'SENSITIVE': tokens.Keyword,
- 'SERIALIZABLE': tokens.Keyword,
- 'SERVER_NAME': tokens.Keyword,
- 'SESSION': tokens.Keyword,
- 'SESSION_USER': tokens.Keyword,
- 'SETOF': tokens.Keyword,
- 'SETS': tokens.Keyword,
- 'SHARE': tokens.Keyword,
- 'SHOW': tokens.Keyword,
- 'SIMILAR': tokens.Keyword,
- 'SIMPLE': tokens.Keyword,
- 'SIZE': tokens.Keyword,
- 'SOME': tokens.Keyword,
- 'SOURCE': tokens.Keyword,
- 'SPACE': tokens.Keyword,
- 'SPECIFIC': tokens.Keyword,
- 'SPECIFICTYPE': tokens.Keyword,
- 'SPECIFIC_NAME': tokens.Keyword,
- 'SQL': tokens.Keyword,
- 'SQLCODE': tokens.Keyword,
- 'SQLERROR': tokens.Keyword,
- 'SQLEXCEPTION': tokens.Keyword,
- 'SQLSTATE': tokens.Keyword,
- 'SQLWARNING': tokens.Keyword,
- 'STABLE': tokens.Keyword,
- 'START': tokens.Keyword.DML,
- 'STATE': tokens.Keyword,
- 'STATEMENT': tokens.Keyword,
- 'STATIC': tokens.Keyword,
- 'STATISTICS': tokens.Keyword,
- 'STDIN': tokens.Keyword,
- 'STDOUT': tokens.Keyword,
- 'STORAGE': tokens.Keyword,
- 'STRICT': tokens.Keyword,
- 'STRUCTURE': tokens.Keyword,
- 'STYPE': tokens.Keyword,
- 'SUBCLASS_ORIGIN': tokens.Keyword,
- 'SUBLIST': tokens.Keyword,
- 'SUBSTRING': tokens.Keyword,
- 'SUM': tokens.Keyword,
- 'SYMMETRIC': tokens.Keyword,
- 'SYSID': tokens.Keyword,
- 'SYSTEM': tokens.Keyword,
- 'SYSTEM_USER': tokens.Keyword,
-
- 'TABLE': tokens.Keyword,
- 'TABLE_NAME': tokens.Keyword,
- 'TEMP': tokens.Keyword,
- 'TEMPLATE': tokens.Keyword,
- 'TEMPORARY': tokens.Keyword,
- 'TERMINATE': tokens.Keyword,
- 'THAN': tokens.Keyword,
- 'TIMESTAMP': tokens.Keyword,
- 'TIMEZONE_HOUR': tokens.Keyword,
- 'TIMEZONE_MINUTE': tokens.Keyword,
- 'TO': tokens.Keyword,
- 'TOAST': tokens.Keyword,
- 'TRAILING': tokens.Keyword,
- 'TRANSATION': tokens.Keyword,
- 'TRANSACTIONS_COMMITTED': tokens.Keyword,
- 'TRANSACTIONS_ROLLED_BACK': tokens.Keyword,
- 'TRANSATION_ACTIVE': tokens.Keyword,
- 'TRANSFORM': tokens.Keyword,
- 'TRANSFORMS': tokens.Keyword,
- 'TRANSLATE': tokens.Keyword,
- 'TRANSLATION': tokens.Keyword,
- 'TREAT': tokens.Keyword,
- 'TRIGGER': tokens.Keyword,
- 'TRIGGER_CATALOG': tokens.Keyword,
- 'TRIGGER_NAME': tokens.Keyword,
- 'TRIGGER_SCHEMA': tokens.Keyword,
- 'TRIM': tokens.Keyword,
- 'TRUE': tokens.Keyword,
- 'TRUNCATE': tokens.Keyword,
- 'TRUSTED': tokens.Keyword,
- 'TYPE': tokens.Keyword,
-
- 'UNCOMMITTED': tokens.Keyword,
- 'UNDER': tokens.Keyword,
- 'UNENCRYPTED': tokens.Keyword,
- 'UNION': tokens.Keyword,
- 'UNIQUE': tokens.Keyword,
- 'UNKNOWN': tokens.Keyword,
- 'UNLISTEN': tokens.Keyword,
- 'UNNAMED': tokens.Keyword,
- 'UNNEST': tokens.Keyword,
- 'UNTIL': tokens.Keyword,
- 'UPPER': tokens.Keyword,
- 'USAGE': tokens.Keyword,
- 'USE': tokens.Keyword,
- 'USER': tokens.Keyword,
- 'USER_DEFINED_TYPE_CATALOG': tokens.Keyword,
- 'USER_DEFINED_TYPE_NAME': tokens.Keyword,
- 'USER_DEFINED_TYPE_SCHEMA': tokens.Keyword,
- 'USING': tokens.Keyword,
-
- 'VACUUM': tokens.Keyword,
- 'VALID': tokens.Keyword,
- 'VALIDATOR': tokens.Keyword,
- 'VALUES': tokens.Keyword,
- 'VARIABLE': tokens.Keyword,
- 'VERBOSE': tokens.Keyword,
- 'VERSION': tokens.Keyword,
- 'VIEW': tokens.Keyword,
- 'VOLATILE': tokens.Keyword,
-
- 'WHENEVER': tokens.Keyword,
- 'WITH': tokens.Keyword,
- 'WITHOUT': tokens.Keyword,
- 'WORK': tokens.Keyword,
- 'WRITE': tokens.Keyword,
-
- 'YEAR': tokens.Keyword,
-
- 'ZONE': tokens.Keyword,
-
-
- 'ARRAY': tokens.Name.Builtin,
- 'BIGINT': tokens.Name.Builtin,
- 'BINARY': tokens.Name.Builtin,
- 'BIT': tokens.Name.Builtin,
- 'BLOB': tokens.Name.Builtin,
- 'BOOLEAN': tokens.Name.Builtin,
- 'CHAR': tokens.Name.Builtin,
- 'CHARACTER': tokens.Name.Builtin,
- 'DATE': tokens.Name.Builtin,
- 'DEC': tokens.Name.Builtin,
- 'DECIMAL': tokens.Name.Builtin,
- 'FLOAT': tokens.Name.Builtin,
- 'INT': tokens.Name.Builtin,
- 'INTEGER': tokens.Name.Builtin,
- 'INTERVAL': tokens.Name.Builtin,
- 'LONG': tokens.Name.Builtin,
- 'NUMBER': tokens.Name.Builtin,
- 'NUMERIC': tokens.Name.Builtin,
- 'REAL': tokens.Name.Builtin,
- 'SERIAL': tokens.Name.Builtin,
- 'SMALLINT': tokens.Name.Builtin,
- 'VARCHAR': tokens.Name.Builtin,
- 'VARCHAR2': tokens.Name.Builtin,
- 'VARYING': tokens.Name.Builtin,
- 'INT8': tokens.Name.Builtin,
- 'SERIAL8': tokens.Name.Builtin,
- 'TEXT': tokens.Name.Builtin,
-}
-
-
-KEYWORDS_COMMON = {
- 'SELECT': tokens.Keyword.DML,
- 'INSERT': tokens.Keyword.DML,
- 'DELETE': tokens.Keyword.DML,
- 'UPDATE': tokens.Keyword.DML,
- 'REPLACE': tokens.Keyword.DML,
- 'MERGE': tokens.Keyword.DML,
- 'DROP': tokens.Keyword.DDL,
- 'CREATE': tokens.Keyword.DDL,
- 'ALTER': tokens.Keyword.DDL,
-
- 'WHERE': tokens.Keyword,
- 'FROM': tokens.Keyword,
- 'INNER': tokens.Keyword,
- 'JOIN': tokens.Keyword,
- 'STRAIGHT_JOIN': tokens.Keyword,
- 'AND': tokens.Keyword,
- 'OR': tokens.Keyword,
- 'LIKE': tokens.Keyword,
- 'ON': tokens.Keyword,
- 'IN': tokens.Keyword,
- 'SET': tokens.Keyword,
-
- 'BY': tokens.Keyword,
- 'GROUP': tokens.Keyword,
- 'ORDER': tokens.Keyword,
- 'LEFT': tokens.Keyword,
- 'OUTER': tokens.Keyword,
- 'FULL': tokens.Keyword,
-
- 'IF': tokens.Keyword,
- 'END': tokens.Keyword,
- 'THEN': tokens.Keyword,
- 'LOOP': tokens.Keyword,
- 'AS': tokens.Keyword,
- 'ELSE': tokens.Keyword,
- 'FOR': tokens.Keyword,
-
- 'CASE': tokens.Keyword,
- 'WHEN': tokens.Keyword,
- 'MIN': tokens.Keyword,
- 'MAX': tokens.Keyword,
- 'DISTINCT': tokens.Keyword,
-}
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/sqlparse/lexer.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/sqlparse/lexer.py b/shell/ext-py/sqlparse-0.1.14/sqlparse/lexer.py
deleted file mode 100644
index b7a33ff..0000000
--- a/shell/ext-py/sqlparse-0.1.14/sqlparse/lexer.py
+++ /dev/null
@@ -1,350 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright (C) 2008 Andi Albrecht, albrecht.andi@gmail.com
-#
-# This module is part of python-sqlparse and is released under
-# the BSD License: http://www.opensource.org/licenses/bsd-license.php.
-
-"""SQL Lexer"""
-
-# This code is based on the SqlLexer in pygments.
-# http://pygments.org/
-# It's separated from the rest of pygments to increase performance
-# and to allow some customizations.
-
-import re
-import sys
-
-from sqlparse import tokens
-from sqlparse.keywords import KEYWORDS, KEYWORDS_COMMON
-from cStringIO import StringIO
-
-
-class include(str):
- pass
-
-
-class combined(tuple):
- """Indicates a state combined from multiple states."""
-
- def __new__(cls, *args):
- return tuple.__new__(cls, args)
-
- def __init__(self, *args):
- # tuple.__init__ doesn't do anything
- pass
-
-
-def is_keyword(value):
- test = value.upper()
- return KEYWORDS_COMMON.get(test, KEYWORDS.get(test, tokens.Name)), value
-
-
-def apply_filters(stream, filters, lexer=None):
- """
- Use this method to apply an iterable of filters to
- a stream. If lexer is given it's forwarded to the
- filter, otherwise the filter receives `None`.
- """
-
- def _apply(filter_, stream):
- for token in filter_.filter(lexer, stream):
- yield token
-
- for filter_ in filters:
- stream = _apply(filter_, stream)
- return stream
-
-
-class LexerMeta(type):
- """
- Metaclass for Lexer, creates the self._tokens attribute from
- self.tokens on the first instantiation.
- """
-
- def _process_state(cls, unprocessed, processed, state):
- assert type(state) is str, "wrong state name %r" % state
- assert state[0] != '#', "invalid state name %r" % state
- if state in processed:
- return processed[state]
- tokenlist = processed[state] = []
- rflags = cls.flags
- for tdef in unprocessed[state]:
- if isinstance(tdef, include):
- # it's a state reference
- assert tdef != state, "circular state reference %r" % state
- tokenlist.extend(cls._process_state(
- unprocessed, processed, str(tdef)))
- continue
-
- assert type(tdef) is tuple, "wrong rule def %r" % tdef
-
- try:
- rex = re.compile(tdef[0], rflags).match
- except Exception, err:
- raise ValueError(("uncompilable regex %r in state"
- " %r of %r: %s"
- % (tdef[0], state, cls, err)))
-
- assert type(tdef[1]) is tokens._TokenType or callable(tdef[1]), \
- ('token type must be simple type or callable, not %r'
- % (tdef[1],))
-
- if len(tdef) == 2:
- new_state = None
- else:
- tdef2 = tdef[2]
- if isinstance(tdef2, str):
- # an existing state
- if tdef2 == '#pop':
- new_state = -1
- elif tdef2 in unprocessed:
- new_state = (tdef2,)
- elif tdef2 == '#push':
- new_state = tdef2
- elif tdef2[:5] == '#pop:':
- new_state = -int(tdef2[5:])
- else:
- assert False, 'unknown new state %r' % tdef2
- elif isinstance(tdef2, combined):
- # combine a new state from existing ones
- new_state = '_tmp_%d' % cls._tmpname
- cls._tmpname += 1
- itokens = []
- for istate in tdef2:
- assert istate != state, \
- 'circular state ref %r' % istate
- itokens.extend(cls._process_state(unprocessed,
- processed, istate))
- processed[new_state] = itokens
- new_state = (new_state,)
- elif isinstance(tdef2, tuple):
- # push more than one state
- for state in tdef2:
- assert (state in unprocessed or
- state in ('#pop', '#push')), \
- 'unknown new state ' + state
- new_state = tdef2
- else:
- assert False, 'unknown new state def %r' % tdef2
- tokenlist.append((rex, tdef[1], new_state))
- return tokenlist
-
- def process_tokendef(cls):
- cls._all_tokens = {}
- cls._tmpname = 0
- processed = cls._all_tokens[cls.__name__] = {}
- #tokendefs = tokendefs or cls.tokens[name]
- for state in cls.tokens.keys():
- cls._process_state(cls.tokens, processed, state)
- return processed
-
- def __call__(cls, *args, **kwds):
- if not hasattr(cls, '_tokens'):
- cls._all_tokens = {}
- cls._tmpname = 0
- if hasattr(cls, 'token_variants') and cls.token_variants:
- # don't process yet
- pass
- else:
- cls._tokens = cls.process_tokendef()
-
- return type.__call__(cls, *args, **kwds)
-
-
-class Lexer(object):
-
- __metaclass__ = LexerMeta
-
- encoding = 'utf-8'
- stripall = False
- stripnl = False
- tabsize = 0
- flags = re.IGNORECASE | re.UNICODE
-
- tokens = {
- 'root': [
- (r'--.*?(\r\n|\r|\n)', tokens.Comment.Single),
- # $ matches *before* newline, therefore we have two patterns
- # to match Comment.Single
- (r'--.*?$', tokens.Comment.Single),
- (r'(\r\n|\r|\n)', tokens.Newline),
- (r'\s+', tokens.Whitespace),
- (r'/\*', tokens.Comment.Multiline, 'multiline-comments'),
- (r':=', tokens.Assignment),
- (r'::', tokens.Punctuation),
- (r'[*]', tokens.Wildcard),
- (r'CASE\b', tokens.Keyword), # extended CASE(foo)
- (r"`(``|[^`])*`", tokens.Name),
- (r"´(´´|[^´])*´", tokens.Name),
- (r'\$([^\W\d]\w*)?\$', tokens.Name.Builtin),
- (r'\?{1}', tokens.Name.Placeholder),
- (r'%\(\w+\)s', tokens.Name.Placeholder),
- (r'%s', tokens.Name.Placeholder),
- (r'[$:?]\w+', tokens.Name.Placeholder),
- # FIXME(andi): VALUES shouldn't be listed here
- # see https://github.com/andialbrecht/sqlparse/pull/64
- (r'VALUES', tokens.Keyword),
- (r'@[^\W\d_]\w+', tokens.Name),
- (r'[^\W\d_]\w*(?=[.(])', tokens.Name), # see issue39
- (r'[-]?0x[0-9a-fA-F]+', tokens.Number.Hexadecimal),
- (r'[-]?[0-9]*(\.[0-9]+)?[eE][-]?[0-9]+', tokens.Number.Float),
- (r'[-]?[0-9]*\.[0-9]+', tokens.Number.Float),
- (r'[-]?[0-9]+', tokens.Number.Integer),
- (r"'(''|\\\\|\\'|[^'])*'", tokens.String.Single),
- # not a real string literal in ANSI SQL:
- (r'(""|".*?[^\\]")', tokens.String.Symbol),
- (r'(\[.*[^\]]\])', tokens.Name),
- (r'((LEFT\s+|RIGHT\s+|FULL\s+)?(INNER\s+|OUTER\s+|STRAIGHT\s+)?|(CROSS\s+|NATURAL\s+)?)?JOIN\b', tokens.Keyword),
- (r'END(\s+IF|\s+LOOP)?\b', tokens.Keyword),
- (r'NOT NULL\b', tokens.Keyword),
- (r'CREATE(\s+OR\s+REPLACE)?\b', tokens.Keyword.DDL),
- (r'(?<=\.)[^\W\d_]\w*', tokens.Name),
- (r'[^\W\d_]\w*', is_keyword),
- (r'[;:()\[\],\.]', tokens.Punctuation),
- (r'[<>=~!]+', tokens.Operator.Comparison),
- (r'[+/@#%^&|`?^-]+', tokens.Operator),
- ],
- 'multiline-comments': [
- (r'/\*', tokens.Comment.Multiline, 'multiline-comments'),
- (r'\*/', tokens.Comment.Multiline, '#pop'),
- (r'[^/\*]+', tokens.Comment.Multiline),
- (r'[/*]', tokens.Comment.Multiline),
- ]}
-
- def __init__(self):
- self.filters = []
-
- def add_filter(self, filter_, **options):
- from sqlparse.filters import Filter
- if not isinstance(filter_, Filter):
- filter_ = filter_(**options)
- self.filters.append(filter_)
-
- def _decode(self, text):
- if sys.version_info[0] == 3:
- if isinstance(text, str):
- return text
- if self.encoding == 'guess':
- try:
- text = text.decode('utf-8')
- if text.startswith(u'\ufeff'):
- text = text[len(u'\ufeff'):]
- except UnicodeDecodeError:
- text = text.decode('latin1')
- else:
- try:
- text = text.decode(self.encoding)
- except UnicodeDecodeError:
- text = text.decode('unicode-escape')
-
- if self.tabsize > 0:
- text = text.expandtabs(self.tabsize)
- return text
-
- def get_tokens(self, text, unfiltered=False):
- """
- Return an iterable of (tokentype, value) pairs generated from
- `text`. If `unfiltered` is set to `True`, the filtering mechanism
- is bypassed even if filters are defined.
-
- Also preprocess the text, i.e. expand tabs and strip it if
- wanted and applies registered filters.
- """
- if isinstance(text, basestring):
- if self.stripall:
- text = text.strip()
- elif self.stripnl:
- text = text.strip('\n')
-
- if sys.version_info[0] < 3 and isinstance(text, unicode):
- text = StringIO(text.encode('utf-8'))
- self.encoding = 'utf-8'
- else:
- text = StringIO(text)
-
- def streamer():
- for i, t, v in self.get_tokens_unprocessed(text):
- yield t, v
- stream = streamer()
- if not unfiltered:
- stream = apply_filters(stream, self.filters, self)
- return stream
-
- def get_tokens_unprocessed(self, stream, stack=('root',)):
- """
- Split ``text`` into (tokentype, text) pairs.
-
- ``stack`` is the inital stack (default: ``['root']``)
- """
- pos = 0
- tokendefs = self._tokens # see __call__, pylint:disable=E1101
- statestack = list(stack)
- statetokens = tokendefs[statestack[-1]]
- known_names = {}
-
- text = stream.read()
- text = self._decode(text)
-
- while 1:
- for rexmatch, action, new_state in statetokens:
- m = rexmatch(text, pos)
- if m:
- # print rex.pattern
- value = m.group()
- if value in known_names:
- yield pos, known_names[value], value
- elif type(action) is tokens._TokenType:
- yield pos, action, value
- elif hasattr(action, '__call__'):
- ttype, value = action(value)
- known_names[value] = ttype
- yield pos, ttype, value
- else:
- for item in action(self, m):
- yield item
- pos = m.end()
- if new_state is not None:
- # state transition
- if isinstance(new_state, tuple):
- for state in new_state:
- if state == '#pop':
- statestack.pop()
- elif state == '#push':
- statestack.append(statestack[-1])
- else:
- statestack.append(state)
- elif isinstance(new_state, int):
- # pop
- del statestack[new_state:]
- elif new_state == '#push':
- statestack.append(statestack[-1])
- else:
- assert False, "wrong state def: %r" % new_state
- statetokens = tokendefs[statestack[-1]]
- break
- else:
- try:
- if text[pos] == '\n':
- # at EOL, reset state to "root"
- pos += 1
- statestack = ['root']
- statetokens = tokendefs['root']
- yield pos, tokens.Text, u'\n'
- continue
- yield pos, tokens.Error, text[pos]
- pos += 1
- except IndexError:
- break
-
-
-def tokenize(sql, encoding=None):
- """Tokenize sql.
-
- Tokenize *sql* using the :class:`Lexer` and return a 2-tuple stream
- of ``(token type, value)`` items.
- """
- lexer = Lexer()
- if encoding is not None:
- lexer.encoding = encoding
- return lexer.get_tokens(sql)
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/sqlparse/pipeline.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/sqlparse/pipeline.py b/shell/ext-py/sqlparse-0.1.14/sqlparse/pipeline.py
deleted file mode 100644
index 34dad19..0000000
--- a/shell/ext-py/sqlparse-0.1.14/sqlparse/pipeline.py
+++ /dev/null
@@ -1,31 +0,0 @@
-# Copyright (C) 2011 Jesus Leganes "piranna", piranna@gmail.com
-#
-# This module is part of python-sqlparse and is released under
-# the BSD License: http://www.opensource.org/licenses/bsd-license.php.
-
-from types import GeneratorType
-
-
-class Pipeline(list):
- """Pipeline to process filters sequentially"""
-
- def __call__(self, stream):
- """Run the pipeline
-
- Return a static (non generator) version of the result
- """
-
- # Run the stream over all the filters on the pipeline
- for filter in self:
- # Functions and callable objects (objects with '__call__' method)
- if callable(filter):
- stream = filter(stream)
-
- # Normal filters (objects with 'process' method)
- else:
- stream = filter.process(None, stream)
-
- # If last filter return a generator, staticalize it inside a list
- if isinstance(stream, GeneratorType):
- return list(stream)
- return stream
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/sqlparse/sql.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/sqlparse/sql.py b/shell/ext-py/sqlparse-0.1.14/sqlparse/sql.py
deleted file mode 100644
index b8e4090..0000000
--- a/shell/ext-py/sqlparse-0.1.14/sqlparse/sql.py
+++ /dev/null
@@ -1,639 +0,0 @@
-# -*- coding: utf-8 -*-
-
-"""This module contains classes representing syntactical elements of SQL."""
-
-import re
-import sys
-
-from sqlparse import tokens as T
-
-
-class Token(object):
- """Base class for all other classes in this module.
-
- It represents a single token and has two instance attributes:
- ``value`` is the unchange value of the token and ``ttype`` is
- the type of the token.
- """
-
- __slots__ = ('value', 'ttype', 'parent', 'normalized', 'is_keyword')
-
- def __init__(self, ttype, value):
- self.value = value
- if ttype in T.Keyword:
- self.normalized = value.upper()
- else:
- self.normalized = value
- self.ttype = ttype
- self.is_keyword = ttype in T.Keyword
- self.parent = None
-
- def __str__(self):
- if sys.version_info[0] == 3:
- return self.value
- else:
- return unicode(self).encode('utf-8')
-
- def __repr__(self):
- short = self._get_repr_value()
- if sys.version_info[0] < 3:
- short = short.encode('utf-8')
- return '<%s \'%s\' at 0x%07x>' % (self._get_repr_name(),
- short, id(self))
-
- def __unicode__(self):
- """Returns a unicode representation of this object."""
- return self.value or ''
-
- def to_unicode(self):
- """Returns a unicode representation of this object.
-
- .. deprecated:: 0.1.5
- Use ``unicode(token)`` (for Python 3: ``str(token)``) instead.
- """
- return unicode(self)
-
- def _get_repr_name(self):
- return str(self.ttype).split('.')[-1]
-
- def _get_repr_value(self):
- raw = unicode(self)
- if len(raw) > 7:
- raw = raw[:6] + u'...'
- return re.sub('\s+', ' ', raw)
-
- def flatten(self):
- """Resolve subgroups."""
- yield self
-
- def match(self, ttype, values, regex=False):
- """Checks whether the token matches the given arguments.
-
- *ttype* is a token type. If this token doesn't match the given token
- type.
- *values* is a list of possible values for this token. The values
- are OR'ed together so if only one of the values matches ``True``
- is returned. Except for keyword tokens the comparison is
- case-sensitive. For convenience it's ok to pass in a single string.
- If *regex* is ``True`` (default is ``False``) the given values are
- treated as regular expressions.
- """
- type_matched = self.ttype is ttype
- if not type_matched or values is None:
- return type_matched
-
- if regex:
- if isinstance(values, basestring):
- values = set([values])
-
- if self.ttype is T.Keyword:
- values = set(re.compile(v, re.IGNORECASE) for v in values)
- else:
- values = set(re.compile(v) for v in values)
-
- for pattern in values:
- if pattern.search(self.value):
- return True
- return False
-
- if isinstance(values, basestring):
- if self.is_keyword:
- return values.upper() == self.normalized
- return values == self.value
-
- if self.is_keyword:
- for v in values:
- if v.upper() == self.normalized:
- return True
- return False
-
- return self.value in values
-
- def is_group(self):
- """Returns ``True`` if this object has children."""
- return False
-
- def is_whitespace(self):
- """Return ``True`` if this token is a whitespace token."""
- return self.ttype and self.ttype in T.Whitespace
-
- def within(self, group_cls):
- """Returns ``True`` if this token is within *group_cls*.
-
- Use this method for example to check if an identifier is within
- a function: ``t.within(sql.Function)``.
- """
- parent = self.parent
- while parent:
- if isinstance(parent, group_cls):
- return True
- parent = parent.parent
- return False
-
- def is_child_of(self, other):
- """Returns ``True`` if this token is a direct child of *other*."""
- return self.parent == other
-
- def has_ancestor(self, other):
- """Returns ``True`` if *other* is in this tokens ancestry."""
- parent = self.parent
- while parent:
- if parent == other:
- return True
- parent = parent.parent
- return False
-
-
-class TokenList(Token):
- """A group of tokens.
-
- It has an additional instance attribute ``tokens`` which holds a
- list of child-tokens.
- """
-
- __slots__ = ('value', 'ttype', 'tokens')
-
- def __init__(self, tokens=None):
- if tokens is None:
- tokens = []
- self.tokens = tokens
- Token.__init__(self, None, self._to_string())
-
- def __unicode__(self):
- return self._to_string()
-
- def __str__(self):
- str_ = self._to_string()
- if sys.version_info[0] < 2:
- str_ = str_.encode('utf-8')
- return str_
-
- def _to_string(self):
- if sys.version_info[0] == 3:
- return ''.join(x.value for x in self.flatten())
- else:
- return ''.join(unicode(x) for x in self.flatten())
-
- def _get_repr_name(self):
- return self.__class__.__name__
-
- def _pprint_tree(self, max_depth=None, depth=0):
- """Pretty-print the object tree."""
- indent = ' ' * (depth * 2)
- for idx, token in enumerate(self.tokens):
- if token.is_group():
- pre = ' +-'
- else:
- pre = ' | '
- print '%s%s%d %s \'%s\'' % (indent, pre, idx,
- token._get_repr_name(),
- token._get_repr_value())
- if (token.is_group() and (max_depth is None or depth < max_depth)):
- token._pprint_tree(max_depth, depth + 1)
-
- def _remove_quotes(self, val):
- """Helper that removes surrounding quotes from strings."""
- if not val:
- return val
- if val[0] in ('"', '\'') and val[-1] == val[0]:
- val = val[1:-1]
- return val
-
- def get_token_at_offset(self, offset):
- """Returns the token that is on position offset."""
- idx = 0
- for token in self.flatten():
- end = idx + len(token.value)
- if idx <= offset <= end:
- return token
- idx = end
-
- def flatten(self):
- """Generator yielding ungrouped tokens.
-
- This method is recursively called for all child tokens.
- """
- for token in self.tokens:
- if isinstance(token, TokenList):
- for item in token.flatten():
- yield item
- else:
- yield token
-
-# def __iter__(self):
-# return self
-#
-# def next(self):
-# for token in self.tokens:
-# yield token
-
- def is_group(self):
- return True
-
- def get_sublists(self):
-# return [x for x in self.tokens if isinstance(x, TokenList)]
- for x in self.tokens:
- if isinstance(x, TokenList):
- yield x
-
- @property
- def _groupable_tokens(self):
- return self.tokens
-
- def token_first(self, ignore_whitespace=True):
- """Returns the first child token.
-
- If *ignore_whitespace* is ``True`` (the default), whitespace
- tokens are ignored.
- """
- for token in self.tokens:
- if ignore_whitespace and token.is_whitespace():
- continue
- return token
-
- def token_next_by_instance(self, idx, clss):
- """Returns the next token matching a class.
-
- *idx* is where to start searching in the list of child tokens.
- *clss* is a list of classes the token should be an instance of.
-
- If no matching token can be found ``None`` is returned.
- """
- if not isinstance(clss, (list, tuple)):
- clss = (clss,)
-
- for token in self.tokens[idx:]:
- if isinstance(token, clss):
- return token
-
- def token_next_by_type(self, idx, ttypes):
- """Returns next matching token by it's token type."""
- if not isinstance(ttypes, (list, tuple)):
- ttypes = [ttypes]
-
- for token in self.tokens[idx:]:
- if token.ttype in ttypes:
- return token
-
- def token_next_match(self, idx, ttype, value, regex=False):
- """Returns next token where it's ``match`` method returns ``True``."""
- if not isinstance(idx, int):
- idx = self.token_index(idx)
-
- for n in xrange(idx, len(self.tokens)):
- token = self.tokens[n]
- if token.match(ttype, value, regex):
- return token
-
- def token_not_matching(self, idx, funcs):
- for token in self.tokens[idx:]:
- passed = False
- for func in funcs:
- if func(token):
- passed = True
- break
-
- if not passed:
- return token
-
- def token_matching(self, idx, funcs):
- for token in self.tokens[idx:]:
- for func in funcs:
- if func(token):
- return token
-
- def token_prev(self, idx, skip_ws=True):
- """Returns the previous token relative to *idx*.
-
- If *skip_ws* is ``True`` (the default) whitespace tokens are ignored.
- ``None`` is returned if there's no previous token.
- """
- if idx is None:
- return None
-
- if not isinstance(idx, int):
- idx = self.token_index(idx)
-
- while idx:
- idx -= 1
- if self.tokens[idx].is_whitespace() and skip_ws:
- continue
- return self.tokens[idx]
-
- def token_next(self, idx, skip_ws=True):
- """Returns the next token relative to *idx*.
-
- If *skip_ws* is ``True`` (the default) whitespace tokens are ignored.
- ``None`` is returned if there's no next token.
- """
- if idx is None:
- return None
-
- if not isinstance(idx, int):
- idx = self.token_index(idx)
-
- while idx < len(self.tokens) - 1:
- idx += 1
- if self.tokens[idx].is_whitespace() and skip_ws:
- continue
- return self.tokens[idx]
-
- def token_index(self, token):
- """Return list index of token."""
- return self.tokens.index(token)
-
- def tokens_between(self, start, end, exclude_end=False):
- """Return all tokens between (and including) start and end.
-
- If *exclude_end* is ``True`` (default is ``False``) the end token
- is included too.
- """
- # FIXME(andi): rename exclude_end to inlcude_end
- if exclude_end:
- offset = 0
- else:
- offset = 1
- end_idx = self.token_index(end) + offset
- start_idx = self.token_index(start)
- return self.tokens[start_idx:end_idx]
-
- def group_tokens(self, grp_cls, tokens, ignore_ws=False):
- """Replace tokens by an instance of *grp_cls*."""
- idx = self.token_index(tokens[0])
- if ignore_ws:
- while tokens and tokens[-1].is_whitespace():
- tokens = tokens[:-1]
- for t in tokens:
- self.tokens.remove(t)
- grp = grp_cls(tokens)
- for token in tokens:
- token.parent = grp
- grp.parent = self
- self.tokens.insert(idx, grp)
- return grp
-
- def insert_before(self, where, token):
- """Inserts *token* before *where*."""
- self.tokens.insert(self.token_index(where), token)
-
- def insert_after(self, where, token, skip_ws=True):
- """Inserts *token* after *where*."""
- next_token = self.token_next(where, skip_ws=skip_ws)
- if next_token is None:
- self.tokens.append(token)
- else:
- self.tokens.insert(self.token_index(next_token), token)
-
- def has_alias(self):
- """Returns ``True`` if an alias is present."""
- return self.get_alias() is not None
-
- def get_alias(self):
- """Returns the alias for this identifier or ``None``."""
- kw = self.token_next_match(0, T.Keyword, 'AS')
- if kw is not None:
- alias = self.token_next(self.token_index(kw))
- if alias is None:
- return None
- else:
- next_ = self.token_next_by_instance(0, Identifier)
- if next_ is None:
- next_ = self.token_next_by_type(0, T.String.Symbol)
- if next_ is None:
- return None
- alias = next_
- if isinstance(alias, Identifier):
- return alias.get_name()
- return self._remove_quotes(unicode(alias))
-
- def get_name(self):
- """Returns the name of this identifier.
-
- This is either it's alias or it's real name. The returned valued can
- be considered as the name under which the object corresponding to
- this identifier is known within the current statement.
- """
- alias = self.get_alias()
- if alias is not None:
- return alias
- return self.get_real_name()
-
- def get_real_name(self):
- """Returns the real name (object name) of this identifier."""
- # a.b
- dot = self.token_next_match(0, T.Punctuation, '.')
- if dot is None:
- next_ = self.token_next_by_type(0, T.Name)
- if next_ is not None:
- return self._remove_quotes(next_.value)
- return None
-
- next_ = self.token_next_by_type(self.token_index(dot),
- (T.Name, T.Wildcard, T.String.Symbol))
- if next_ is None: # invalid identifier, e.g. "a."
- return None
- return self._remove_quotes(next_.value)
-
-
-class Statement(TokenList):
- """Represents a SQL statement."""
-
- __slots__ = ('value', 'ttype', 'tokens')
-
- def get_type(self):
- """Returns the type of a statement.
-
- The returned value is a string holding an upper-cased reprint of
- the first DML or DDL keyword. If the first token in this group
- isn't a DML or DDL keyword "UNKNOWN" is returned.
- """
- first_token = self.token_first()
- if first_token is None:
- # An "empty" statement that either has not tokens at all
- # or only whitespace tokens.
- return 'UNKNOWN'
-
- elif first_token.ttype in (T.Keyword.DML, T.Keyword.DDL):
- return first_token.normalized
-
- return 'UNKNOWN'
-
-
-class Identifier(TokenList):
- """Represents an identifier.
-
- Identifiers may have aliases or typecasts.
- """
-
- __slots__ = ('value', 'ttype', 'tokens')
-
- def get_parent_name(self):
- """Return name of the parent object if any.
-
- A parent object is identified by the first occuring dot.
- """
- dot = self.token_next_match(0, T.Punctuation, '.')
- if dot is None:
- return None
- prev_ = self.token_prev(self.token_index(dot))
- if prev_ is None: # something must be verry wrong here..
- return None
- return self._remove_quotes(prev_.value)
-
- def is_wildcard(self):
- """Return ``True`` if this identifier contains a wildcard."""
- token = self.token_next_by_type(0, T.Wildcard)
- return token is not None
-
- def get_typecast(self):
- """Returns the typecast or ``None`` of this object as a string."""
- marker = self.token_next_match(0, T.Punctuation, '::')
- if marker is None:
- return None
- next_ = self.token_next(self.token_index(marker), False)
- if next_ is None:
- return None
- return unicode(next_)
-
- def get_ordering(self):
- """Returns the ordering or ``None`` as uppercase string."""
- ordering = self.token_next_by_type(0, T.Keyword.Order)
- if ordering is None:
- return None
- return ordering.value.upper()
-
-
-class IdentifierList(TokenList):
- """A list of :class:`~sqlparse.sql.Identifier`\'s."""
-
- __slots__ = ('value', 'ttype', 'tokens')
-
- def get_identifiers(self):
- """Returns the identifiers.
-
- Whitespaces and punctuations are not included in this generator.
- """
- for x in self.tokens:
- if not x.is_whitespace() and not x.match(T.Punctuation, ','):
- yield x
-
-
-class Parenthesis(TokenList):
- """Tokens between parenthesis."""
- __slots__ = ('value', 'ttype', 'tokens')
-
- @property
- def _groupable_tokens(self):
- return self.tokens[1:-1]
-
-
-class Assignment(TokenList):
- """An assignment like 'var := val;'"""
- __slots__ = ('value', 'ttype', 'tokens')
-
-
-class If(TokenList):
- """An 'if' clause with possible 'else if' or 'else' parts."""
- __slots__ = ('value', 'ttype', 'tokens')
-
-
-class For(TokenList):
- """A 'FOR' loop."""
- __slots__ = ('value', 'ttype', 'tokens')
-
-
-class Comparison(TokenList):
- """A comparison used for example in WHERE clauses."""
- __slots__ = ('value', 'ttype', 'tokens')
-
- @property
- def left(self):
- return self.tokens[0]
-
- @property
- def right(self):
- return self.tokens[-1]
-
-
-class Comment(TokenList):
- """A comment."""
- __slots__ = ('value', 'ttype', 'tokens')
-
-
-class Where(TokenList):
- """A WHERE clause."""
- __slots__ = ('value', 'ttype', 'tokens')
-
-
-class Case(TokenList):
- """A CASE statement with one or more WHEN and possibly an ELSE part."""
-
- __slots__ = ('value', 'ttype', 'tokens')
-
- def get_cases(self):
- """Returns a list of 2-tuples (condition, value).
-
- If an ELSE exists condition is None.
- """
- CONDITION = 1
- VALUE = 2
-
- ret = []
- mode = CONDITION
-
- for token in self.tokens:
- # Set mode from the current statement
- if token.match(T.Keyword, 'CASE'):
- continue
-
- elif token.match(T.Keyword, 'WHEN'):
- ret.append(([], []))
- mode = CONDITION
-
- elif token.match(T.Keyword, 'THEN'):
- mode = VALUE
-
- elif token.match(T.Keyword, 'ELSE'):
- ret.append((None, []))
- mode = VALUE
-
- elif token.match(T.Keyword, 'END'):
- mode = None
-
- # First condition without preceding WHEN
- if mode and not ret:
- ret.append(([], []))
-
- # Append token depending of the current mode
- if mode == CONDITION:
- ret[-1][0].append(token)
-
- elif mode == VALUE:
- ret[-1][1].append(token)
-
- # Return cases list
- return ret
-
-
-class Function(TokenList):
- """A function or procedure call."""
-
- __slots__ = ('value', 'ttype', 'tokens')
-
- def get_parameters(self):
- """Return a list of parameters."""
- parenthesis = self.tokens[-1]
- for t in parenthesis.tokens:
- if isinstance(t, IdentifierList):
- return t.get_identifiers()
- elif isinstance(t, Identifier) or \
- isinstance(t, Function) or \
- t.ttype in T.Literal:
- return [t,]
- return []
-
-
-class Begin(TokenList):
- """A BEGIN/END block."""
-
- __slots__ = ('value', 'ttype', 'tokens')
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/sqlparse/tokens.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/sqlparse/tokens.py b/shell/ext-py/sqlparse-0.1.14/sqlparse/tokens.py
deleted file mode 100644
index 01a9b89..0000000
--- a/shell/ext-py/sqlparse-0.1.14/sqlparse/tokens.py
+++ /dev/null
@@ -1,83 +0,0 @@
-# Copyright (C) 2008 Andi Albrecht, albrecht.andi@gmail.com
-#
-# This module is part of python-sqlparse and is released under
-# the BSD License: http://www.opensource.org/licenses/bsd-license.php.
-
-# The Token implementation is based on pygment's token system written
-# by Georg Brandl.
-# http://pygments.org/
-
-"""Tokens"""
-
-
-class _TokenType(tuple):
- parent = None
-
- def split(self):
- buf = []
- node = self
- while node is not None:
- buf.append(node)
- node = node.parent
- buf.reverse()
- return buf
-
- def __contains__(self, val):
- return val is not None and (self is val or val[:len(self)] == self)
-
- def __getattr__(self, val):
- if not val or not val[0].isupper():
- return tuple.__getattribute__(self, val)
- new = _TokenType(self + (val,))
- setattr(self, val, new)
- new.parent = self
- return new
-
- def __hash__(self):
- return hash(tuple(self))
-
- def __repr__(self):
- return 'Token' + (self and '.' or '') + '.'.join(self)
-
-
-Token = _TokenType()
-
-# Special token types
-Text = Token.Text
-Whitespace = Text.Whitespace
-Newline = Whitespace.Newline
-Error = Token.Error
-# Text that doesn't belong to this lexer (e.g. HTML in PHP)
-Other = Token.Other
-
-# Common token types for source code
-Keyword = Token.Keyword
-Name = Token.Name
-Literal = Token.Literal
-String = Literal.String
-Number = Literal.Number
-Punctuation = Token.Punctuation
-Operator = Token.Operator
-Comparison = Operator.Comparison
-Wildcard = Token.Wildcard
-Comment = Token.Comment
-Assignment = Token.Assignement
-
-# Generic types for non-source code
-Generic = Token.Generic
-
-# String and some others are not direct childs of Token.
-# alias them:
-Token.Token = Token
-Token.String = String
-Token.Number = Number
-
-# SQL specific tokens
-DML = Keyword.DML
-DDL = Keyword.DDL
-Command = Keyword.Command
-
-Group = Token.Group
-Group.Parenthesis = Token.Group.Parenthesis
-Group.Comment = Token.Group.Comment
-Group.Where = Token.Group.Where
[5/8] impala git commit: IMPALA-6999: Upgrade to sqlparse-0.1.19 for
Impala shell
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/tests/test_tokenize.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/tests/test_tokenize.py b/shell/ext-py/sqlparse-0.1.14/tests/test_tokenize.py
deleted file mode 100644
index 0b23fa8..0000000
--- a/shell/ext-py/sqlparse-0.1.14/tests/test_tokenize.py
+++ /dev/null
@@ -1,190 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import sys
-import types
-import unittest
-
-import pytest
-
-import sqlparse
-from sqlparse import lexer
-from sqlparse import sql
-from sqlparse.tokens import *
-
-
-class TestTokenize(unittest.TestCase):
-
- def test_simple(self):
- s = 'select * from foo;'
- stream = lexer.tokenize(s)
- self.assert_(isinstance(stream, types.GeneratorType))
- tokens = list(stream)
- self.assertEqual(len(tokens), 8)
- self.assertEqual(len(tokens[0]), 2)
- self.assertEqual(tokens[0], (Keyword.DML, u'select'))
- self.assertEqual(tokens[-1], (Punctuation, u';'))
-
- def test_backticks(self):
- s = '`foo`.`bar`'
- tokens = list(lexer.tokenize(s))
- self.assertEqual(len(tokens), 3)
- self.assertEqual(tokens[0], (Name, u'`foo`'))
-
- def test_linebreaks(self): # issue1
- s = 'foo\nbar\n'
- tokens = lexer.tokenize(s)
- self.assertEqual(''.join(str(x[1]) for x in tokens), s)
- s = 'foo\rbar\r'
- tokens = lexer.tokenize(s)
- self.assertEqual(''.join(str(x[1]) for x in tokens), s)
- s = 'foo\r\nbar\r\n'
- tokens = lexer.tokenize(s)
- self.assertEqual(''.join(str(x[1]) for x in tokens), s)
- s = 'foo\r\nbar\n'
- tokens = lexer.tokenize(s)
- self.assertEqual(''.join(str(x[1]) for x in tokens), s)
-
- def test_inline_keywords(self): # issue 7
- s = "create created_foo"
- tokens = list(lexer.tokenize(s))
- self.assertEqual(len(tokens), 3)
- self.assertEqual(tokens[0][0], Keyword.DDL)
- self.assertEqual(tokens[2][0], Name)
- self.assertEqual(tokens[2][1], u'created_foo')
- s = "enddate"
- tokens = list(lexer.tokenize(s))
- self.assertEqual(len(tokens), 1)
- self.assertEqual(tokens[0][0], Name)
- s = "join_col"
- tokens = list(lexer.tokenize(s))
- self.assertEqual(len(tokens), 1)
- self.assertEqual(tokens[0][0], Name)
- s = "left join_col"
- tokens = list(lexer.tokenize(s))
- self.assertEqual(len(tokens), 3)
- self.assertEqual(tokens[2][0], Name)
- self.assertEqual(tokens[2][1], 'join_col')
-
- def test_negative_numbers(self):
- s = "values(-1)"
- tokens = list(lexer.tokenize(s))
- self.assertEqual(len(tokens), 4)
- self.assertEqual(tokens[2][0], Number.Integer)
- self.assertEqual(tokens[2][1], '-1')
-
- # Somehow this test fails on Python 3.2
- @pytest.mark.skipif('sys.version_info >= (3,0)')
- def test_tab_expansion(self):
- s = "\t"
- lex = lexer.Lexer()
- lex.tabsize = 5
- tokens = list(lex.get_tokens(s))
- self.assertEqual(tokens[0][1], " " * 5)
-
-
-class TestToken(unittest.TestCase):
-
- def test_str(self):
- token = sql.Token(None, 'FoO')
- self.assertEqual(str(token), 'FoO')
-
- def test_repr(self):
- token = sql.Token(Keyword, 'foo')
- tst = "<Keyword 'foo' at 0x"
- self.assertEqual(repr(token)[:len(tst)], tst)
- token = sql.Token(Keyword, '1234567890')
- tst = "<Keyword '123456...' at 0x"
- self.assertEqual(repr(token)[:len(tst)], tst)
-
- def test_flatten(self):
- token = sql.Token(Keyword, 'foo')
- gen = token.flatten()
- self.assertEqual(type(gen), types.GeneratorType)
- lgen = list(gen)
- self.assertEqual(lgen, [token])
-
-
-class TestTokenList(unittest.TestCase):
-
- def test_repr(self):
- p = sqlparse.parse('foo, bar, baz')[0]
- tst = "<IdentifierList 'foo, b...' at 0x"
- self.assertEqual(repr(p.tokens[0])[:len(tst)], tst)
-
- def test_token_first(self):
- p = sqlparse.parse(' select foo')[0]
- first = p.token_first()
- self.assertEqual(first.value, 'select')
- self.assertEqual(p.token_first(ignore_whitespace=False).value, ' ')
- self.assertEqual(sql.TokenList([]).token_first(), None)
-
- def test_token_matching(self):
- t1 = sql.Token(Keyword, 'foo')
- t2 = sql.Token(Punctuation, ',')
- x = sql.TokenList([t1, t2])
- self.assertEqual(x.token_matching(0, [lambda t: t.ttype is Keyword]),
- t1)
- self.assertEqual(x.token_matching(0,
- [lambda t: t.ttype is Punctuation]),
- t2)
- self.assertEqual(x.token_matching(1, [lambda t: t.ttype is Keyword]),
- None)
-
-
-class TestStream(unittest.TestCase):
- def test_simple(self):
- from cStringIO import StringIO
-
- stream = StringIO("SELECT 1; SELECT 2;")
- lex = lexer.Lexer()
-
- tokens = lex.get_tokens(stream)
- self.assertEqual(len(list(tokens)), 9)
-
- stream.seek(0)
- lex.bufsize = 4
- tokens = list(lex.get_tokens(stream))
- self.assertEqual(len(tokens), 9)
-
- stream.seek(0)
- lex.bufsize = len(stream.getvalue())
- tokens = list(lex.get_tokens(stream))
- self.assertEqual(len(tokens), 9)
-
- def test_error(self):
- from cStringIO import StringIO
-
- stream = StringIO("FOOBAR{")
-
- lex = lexer.Lexer()
- lex.bufsize = 4
- tokens = list(lex.get_tokens(stream))
- self.assertEqual(len(tokens), 2)
- self.assertEqual(tokens[1][0], Error)
-
-
-@pytest.mark.parametrize('expr', ['JOIN', 'LEFT JOIN', 'LEFT OUTER JOIN',
- 'FULL OUTER JOIN', 'NATURAL JOIN',
- 'CROSS JOIN', 'STRAIGHT JOIN',
- 'INNER JOIN', 'LEFT INNER JOIN'])
-def test_parse_join(expr):
- p = sqlparse.parse('%s foo' % expr)[0]
- assert len(p.tokens) == 3
- assert p.tokens[0].ttype is Keyword
-
-
-def test_parse_endifloop():
- p = sqlparse.parse('END IF')[0]
- assert len(p.tokens) == 1
- assert p.tokens[0].ttype is Keyword
- p = sqlparse.parse('END IF')[0]
- assert len(p.tokens) == 1
- p = sqlparse.parse('END\t\nIF')[0]
- assert len(p.tokens) == 1
- assert p.tokens[0].ttype is Keyword
- p = sqlparse.parse('END LOOP')[0]
- assert len(p.tokens) == 1
- assert p.tokens[0].ttype is Keyword
- p = sqlparse.parse('END LOOP')[0]
- assert len(p.tokens) == 1
- assert p.tokens[0].ttype is Keyword
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/tests/utils.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/tests/utils.py b/shell/ext-py/sqlparse-0.1.14/tests/utils.py
deleted file mode 100644
index 9eb46bf..0000000
--- a/shell/ext-py/sqlparse-0.1.14/tests/utils.py
+++ /dev/null
@@ -1,46 +0,0 @@
-# -*- coding: utf-8 -*-
-
-"""Helpers for testing."""
-
-import codecs
-import difflib
-import os
-import unittest
-from StringIO import StringIO
-
-import sqlparse.utils
-
-NL = '\n'
-DIR_PATH = os.path.abspath(os.path.dirname(__file__))
-PARENT_DIR = os.path.dirname(DIR_PATH)
-FILES_DIR = os.path.join(DIR_PATH, 'files')
-
-
-def load_file(filename, encoding='utf-8'):
- """Opens filename with encoding and return it's contents."""
- f = codecs.open(os.path.join(FILES_DIR, filename), 'r', encoding)
- data = f.read()
- f.close()
- return data
-
-
-class TestCaseBase(unittest.TestCase):
- """Base class for test cases with some additional checks."""
-
- # Adopted from Python's tests.
- def ndiffAssertEqual(self, first, second):
- """Like failUnlessEqual except use ndiff for readable output."""
- if first != second:
- sfirst = unicode(first)
- ssecond = unicode(second)
- # Using the built-in .splitlines() method here will cause incorrect
- # results when splitting statements that have quoted CR/CR+LF
- # characters.
- sfirst = sqlparse.utils.split_unquoted_newlines(sfirst)
- ssecond = sqlparse.utils.split_unquoted_newlines(ssecond)
- diff = difflib.ndiff(sfirst, ssecond)
- fp = StringIO()
- fp.write(NL)
- fp.write(NL.join(diff))
- print fp.getvalue()
- raise self.failureException, fp.getvalue()
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.14/tox.ini
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.14/tox.ini b/shell/ext-py/sqlparse-0.1.14/tox.ini
deleted file mode 100644
index e797ca9..0000000
--- a/shell/ext-py/sqlparse-0.1.14/tox.ini
+++ /dev/null
@@ -1,37 +0,0 @@
-[tox]
-envlist=py26,py27,py32,py33,py34,pypy
-
-[testenv]
-deps=
- pytest
- pytest-cov
-commands=
- sqlformat --version # Sanity check.
- py.test --cov=sqlparse/ tests
-
-[testenv:py32]
-changedir={envdir}
-commands=
- sqlformat --version # Sanity check.
- rm -rf tests/
- cp -r {toxinidir}/tests/ tests/
- 2to3 -w --no-diffs -n tests/
- py.test --cov={envdir}/lib/python3.2/site-packages/sqlparse/ tests
-
-[testenv:py33]
-changedir={envdir}
-commands=
- sqlformat --version # Sanity check.
- rm -rf tests/
- cp -r {toxinidir}/tests/ tests/
- 2to3 -w --no-diffs -n tests/
- py.test --cov={envdir}/lib/python3.3/site-packages/sqlparse/ tests
-
-[testenv:py34]
-changedir={envdir}
-commands=
- sqlformat --version # Sanity check.
- rm -rf tests/
- cp -r {toxinidir}/tests/ tests/
- 2to3 -w --no-diffs -n tests/
- py.test --cov={envdir}/lib/python3.4/site-packages/sqlparse/ tests
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/.travis.yml
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/.travis.yml b/shell/ext-py/sqlparse-0.1.19/.travis.yml
new file mode 100644
index 0000000..313afb8
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/.travis.yml
@@ -0,0 +1,14 @@
+language: python
+python: 2.7
+env:
+ - TOX_ENV=py26
+ - TOX_ENV=py27
+ - TOX_ENV=py33
+ - TOX_ENV=py34
+ - TOX_ENV=pypy
+before_install:
+ - sudo apt-get install pypy
+install:
+ - pip install tox
+script:
+ - tox -e $TOX_ENV
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/AUTHORS
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/AUTHORS b/shell/ext-py/sqlparse-0.1.19/AUTHORS
new file mode 100644
index 0000000..78052ff
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/AUTHORS
@@ -0,0 +1,31 @@
+python-sqlparse is written and maintained by Andi Albrecht <al...@gmail.com>.
+
+This module contains code (namely the lexer and filter mechanism) from
+the pygments project that was written by Georg Brandl.
+
+Alphabetical list of contributors:
+* Alexander Beedie <ay...@gmail.com>
+* Alexey Malyshev <no...@gmail.com>
+* casey <ca...@cloudera.com>
+* Cristian Orellana <cr...@groupon.com>
+* Darik Gamble <da...@gmail.com>
+* Florian Bauer <fl...@zmdi.com>
+* Gavin Wahl <gw...@fusionbox.com>
+* JacekPliszka <Ja...@gmail.com>
+* Jesús Leganés Combarro "Piranna" <pi...@gmail.com>
+* Kevin Jing Qiu <ke...@gmail.com>
+* Michael Schuller <ch...@mschuller.net>
+* Mike Amy <co...@googlemail.com>
+* mulos <da...@gmail.com>
+* Piet Delport <pj...@gmail.com>
+* Prudhvi Vatala <pv...@gmail.com>
+* quest <qu...@wonky.windwards.net>
+* Robert Nix <co...@rnix.org>
+* Rocky Meza <rm...@fusionbox.com>
+* Ryan Wooden <ry...@gmail.com>
+* spigwitmer <it...@gmail.com>
+* Tim Graham <ti...@gmail.com>
+* Victor Hahn <in...@victor-hahn.de>
+* vthriller <fa...@yandex.ru>
+* wayne.wuw <wa...@alibaba-inc.com>
+* Yago Riveiro <ya...@gmail.com>
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/CHANGES
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/CHANGES b/shell/ext-py/sqlparse-0.1.19/CHANGES
new file mode 100644
index 0000000..267f2d0
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/CHANGES
@@ -0,0 +1,302 @@
+Release 0.1.19 (Mar 07, 2015)
+-----------------------------
+
+Bug Fixes
+* Fix IndexError when statement contains WITH clauses (issue205).
+
+
+Release 0.1.18 (Oct 25, 2015)
+-----------------------------
+
+Bug Fixes
+* Remove universal wheel support, added in 0.1.17 by mistake.
+
+
+Release 0.1.17 (Oct 24, 2015)
+-----------------------------
+
+Enhancements
+* Speed up parsing of large SQL statements (pull request: issue201, fixes the
+ following issues: issue199, issue135, issue62, issue41, by Ryan Wooden).
+
+Bug Fixes
+* Fix another splitter bug regarding DECLARE (issue194).
+
+Misc
+* Packages on PyPI are signed from now on.
+
+
+Release 0.1.16 (Jul 26, 2015)
+-----------------------------
+
+Bug Fixes
+* Fix a regression in get_alias() introduced in 0.1.15 (issue185).
+* Fix a bug in the splitter regarding DECLARE (issue193).
+* sqlformat command line tool doesn't duplicat newlines anymore (issue191).
+* Don't mix up MySQL comments starting with hash and MSSQL
+ temp tables (issue192).
+* Statement.get_type() now ignores comments at the beginning of
+ a statement (issue186).
+
+
+Release 0.1.15 (Apr 15, 2015)
+-----------------------------
+
+Bug Fixes
+* Fix a regression for identifiers with square bracktes
+ notation (issue153, by darikg).
+* Add missing SQL types (issue154, issue155, issue156, by jukebox).
+* Fix parsing of multi-line comments (issue172, by JacekPliszka).
+* Fix parsing of escaped backslashes (issue174, by caseyching).
+* Fix parsing of identifiers starting with underscore (issue175).
+* Fix misinterpretation of IN keyword (issue183).
+
+Enhancements
+* Improve formatting of HAVING statements.
+* Improve parsing of inline comments (issue163).
+* Group comments to parent object (issue128, issue160).
+* Add double precision builtin (issue169, by darikg).
+* Add support for square bracket array indexing (issue170, issue176,
+ issue177 by darikg).
+* Improve grouping of aliased elements (issue167, by darikg).
+* Support comments starting with '#' character (issue178).
+
+
+Release 0.1.14 (Nov 30, 2014)
+-----------------------------
+
+Bug Fixes
+* Floats in UPDATE statements are now handled correctly (issue145).
+* Properly handle string literals in comparisons (issue148, change proposed
+ by aadis).
+* Fix indentation when using tabs (issue146).
+
+Enhancements
+* Improved formatting in list when newlines precede commas (issue140).
+
+
+Release 0.1.13 (Oct 09, 2014)
+-----------------------------
+
+Bug Fixes
+* Fix a regression in handling of NULL keywords introduced in 0.1.12.
+
+
+Release 0.1.12 (Sep 20, 2014)
+-----------------------------
+
+Bug Fixes
+* Fix handling of NULL keywords in aliased identifiers.
+* Fix SerializerUnicode to split unquoted newlines (issue131, by Michael Schuller).
+* Fix handling of modulo operators without spaces (by gavinwahl).
+
+Enhancements
+* Improve parsing of identifier lists containing placeholders.
+* Speed up query parsing of unquoted lines (by Michael Schuller).
+
+
+Release 0.1.11 (Feb 07, 2014)
+-----------------------------
+
+Bug Fixes
+* Fix incorrect parsing of string literals containing line breaks (issue118).
+* Fix typo in keywords, add MERGE, COLLECT keywords (issue122/124,
+ by Cristian Orellana).
+* Improve parsing of string literals in columns.
+* Fix parsing and formatting of statements containing EXCEPT keyword.
+* Fix Function.get_parameters() (issue126/127, by spigwitmer).
+
+Enhancements
+* Classify DML keywords (issue116, by Victor Hahn).
+* Add missing FOREACH keyword.
+* Grouping of BEGIN/END blocks.
+
+Other
+* Python 2.5 isn't automatically tested anymore, neither Travis nor Tox
+ still support it out of the box.
+
+
+Release 0.1.10 (Nov 02, 2013)
+-----------------------------
+
+Bug Fixes
+* Removed buffered reading again, it obviously causes wrong parsing in some rare
+ cases (issue114).
+* Fix regression in setup.py introduced 10 months ago (issue115).
+
+Enhancements
+* Improved support for JOINs, by Alexander Beedie.
+
+
+Release 0.1.9 (Sep 28, 2013)
+----------------------------
+
+Bug Fixes
+* Fix an regression introduced in 0.1.5 where sqlparse didn't properly
+ distinguished between single and double quoted strings when tagging
+ identifier (issue111).
+
+Enhancements
+* New option to truncate long string literals when formatting.
+* Scientific numbers are pares correctly (issue107).
+* Support for arithmetic expressions (issue109, issue106; by prudhvi).
+
+
+Release 0.1.8 (Jun 29, 2013)
+----------------------------
+
+Bug Fixes
+* Whitespaces within certain keywords are now allowed (issue97, patch proposed
+ by xcombelle).
+
+Enhancements
+* Improve parsing of assignments in UPDATE statements (issue90).
+* Add STRAIGHT_JOIN statement (by Yago Riveiro).
+* Function.get_parameters() now returns the parameter if only one parameter is
+ given (issue94, by wayne.wuw).
+* sqlparse.split() now removes leading and trailing whitespaces from splitted
+ statements.
+* Add USE as keyword token (by mulos).
+* Improve parsing of PEP249-style placeholders (issue103).
+
+
+Release 0.1.7 (Apr 06, 2013)
+----------------------------
+
+Bug Fixes
+ * Fix Python 3 compatibility of sqlformat script (by Piet Delport).
+ * Fix parsing of SQL statements that contain binary data (by Alexey
+ Malyshev).
+ * Fix a bug where keywords were identified as aliased identifiers in
+ invalid SQL statements.
+ * Fix parsing of identifier lists where identifiers are keywords too
+ (issue10).
+
+Enhancements
+ * Top-level API functions now accept encoding keyword to parse
+ statements in certain encodings more reliable (issue20).
+ * Improve parsing speed when SQL contains CLOBs or BLOBs (issue86).
+ * Improve formatting of ORDER BY clauses (issue89).
+ * Formatter now tries to detect runaway indentations caused by
+ parsing errors or invalid SQL statements. When re-indenting such
+ statements the formatter flips back to column 0 before going crazy.
+
+Other
+ * Documentation updates.
+
+
+Release 0.1.6 (Jan 01, 2013)
+----------------------------
+
+sqlparse is now compatible with Python 3 without any patches. The
+Python 3 version is generated during install by 2to3. You'll need
+distribute to install sqlparse for Python 3.
+
+Bug Fixes
+ * Fix parsing error with dollar-quoted procedure bodies (issue83).
+
+Other
+ * Documentation updates.
+ * Test suite now uses tox and py.test.
+ * py3k fixes (by vthriller).
+ * py3k fixes in setup.py (by Florian Bauer).
+ * setup.py now requires distribute (by Florian Bauer).
+
+
+Release 0.1.5 (Nov 13, 2012)
+----------------------------
+
+Bug Fixes
+ * Improve handling of quoted identifiers (issue78).
+ * Improve grouping and formatting of identifiers with operators (issue53).
+ * Improve grouping and formatting of concatenated strings (issue53).
+ * Improve handling of varchar() (by Mike Amy).
+ * Clean up handling of various SQL elements.
+ * Switch to py.test and clean up tests.
+ * Several minor fixes.
+
+Other
+ * Deprecate sqlparse.SQLParseError. Please use
+ sqlparse.exceptions.SQLParseError instead.
+ * Add caching to speed up processing.
+ * Add experimental filters for token processing.
+ * Add sqlformat.parsestream (by quest).
+
+
+Release 0.1.4 (Apr 20, 2012)
+----------------------------
+
+Bug Fixes
+ * Avoid "stair case" effects when identifiers, functions,
+ placeholders or keywords are mixed in identifier lists (issue45,
+ issue49, issue52) and when asterisks are used as operators
+ (issue58).
+ * Make keyword detection more restrict (issue47).
+ * Improve handling of CASE statements (issue46).
+ * Fix statement splitting when parsing recursive statements (issue57,
+ thanks to piranna).
+ * Fix for negative numbers (issue56, thanks to kevinjqiu).
+ * Pretty format comments in identifier lists (issue59).
+ * Several minor bug fixes and improvements.
+
+
+Release 0.1.3 (Jul 29, 2011)
+----------------------------
+
+Bug Fixes
+ * Improve parsing of floats (thanks to Kris).
+ * When formatting a statement a space before LIMIT was removed (issue35).
+ * Fix strip_comments flag (issue38, reported by ooberm...@gmail.com).
+ * Avoid parsing names as keywords (issue39, reported by djo...@taket.org).
+ * Make sure identifier lists in subselects are grouped (issue40,
+ reported by djo...@taket.org).
+ * Split statements with IF as functions correctly (issue33 and
+ issue29, reported by charles....@unige.ch).
+ * Relax detection of keywords, esp. when used as function names
+ (issue36, nyuhu...@gmail.com).
+ * Don't treat single characters as keywords (issue32).
+ * Improve parsing of stand-alone comments (issue26).
+ * Detection of placeholders in paramterized queries (issue22,
+ reported by Glyph Lefkowitz).
+ * Add parsing of MS Access column names with braces (issue27,
+ reported by frankz...@gmail.com).
+
+Other
+ * Replace Django by Flask in App Engine frontend (issue11).
+
+
+Release 0.1.2 (Nov 23, 2010)
+----------------------------
+
+Bug Fixes
+ * Fixed incorrect detection of keyword fragments embed in names (issue7,
+ reported and initial patch by andyboyko).
+ * Stricter detection of identfier aliases (issue8, reported by estama).
+ * WHERE grouping consumed closing parenthesis (issue9, reported by estama).
+ * Fixed an issue with trailing whitespaces (reported by Kris).
+ * Better detection of escaped single quotes (issue13, reported by
+ Martin Brochhaus, patch by bluemaro with test case by Dan Carley).
+ * Ignore identifier in double-quotes when changing cases (issue 21).
+ * Lots of minor fixes targeting encoding, indentation, statement
+ parsing and more (issues 12, 14, 15, 16, 18, 19).
+ * Code cleanup with a pinch of refactoring.
+
+
+Release 0.1.1 (May 6, 2009)
+---------------------------
+
+Bug Fixes
+ * Lexers preserves original line breaks (issue1).
+ * Improved identifier parsing: backtick quotes, wildcards, T-SQL variables
+ prefixed with @.
+ * Improved parsing of identifier lists (issue2).
+ * Recursive recognition of AS (issue4) and CASE.
+ * Improved support for UPDATE statements.
+
+Other
+ * Code cleanup and better test coverage.
+
+
+Release 0.1.0 (Apr 8, 2009)
+---------------------------
+ * Initial release.
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/COPYING
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/COPYING b/shell/ext-py/sqlparse-0.1.19/COPYING
new file mode 100644
index 0000000..7b158da
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/COPYING
@@ -0,0 +1,25 @@
+Copyright (c) 2009, Andi Albrecht <al...@gmail.com>
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+ * Neither the name of the authors nor the names of its contributors may be
+ used to endorse or promote products derived from this software without
+ specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/MANIFEST.in
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/MANIFEST.in b/shell/ext-py/sqlparse-0.1.19/MANIFEST.in
new file mode 100644
index 0000000..e546b1e
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/MANIFEST.in
@@ -0,0 +1,11 @@
+recursive-include docs source/*
+include docs/sqlformat.1
+include docs/Makefile
+recursive-include tests *.py *.sql
+include COPYING
+include TODO
+include AUTHORS
+include CHANGES
+include Makefile
+include pytest.ini
+include tox.ini
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/README.rst
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/README.rst b/shell/ext-py/sqlparse-0.1.19/README.rst
new file mode 100644
index 0000000..dbe2fe4
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/README.rst
@@ -0,0 +1,56 @@
+python-sqlparse - Parse SQL statements
+======================================
+
+sqlparse is a non-validating SQL parser module for Python.
+
+|buildstatus|_
+
+
+Install
+-------
+
+Run::
+
+ python setup.py install
+
+to install python-sqlparse on your system.
+
+python-sqlparse is compatible with Python 2 (>= 2.5) and Python 3 (>= 3.2).
+
+
+Run Tests
+---------
+
+To run the test suite run::
+
+ tox
+
+Note, you'll need tox installed, of course.
+
+
+Links
+-----
+
+Project Page
+ https://github.com/andialbrecht/sqlparse
+
+Documentation
+ http://readthedocs.org/docs/sqlparse/en/latest/
+
+Discussions
+ http://groups.google.com/group/sqlparse
+
+Issues/Bugs
+ https://github.com/andialbrecht/sqlparse/issues
+
+Online Demo
+ http://sqlformat.org
+
+
+python-sqlparse is licensed under the BSD license.
+
+Parts of the code are based on pygments written by Georg Brandl and others.
+pygments-Homepage: http://pygments.org/
+
+.. |buildstatus| image:: https://secure.travis-ci.org/andialbrecht/sqlparse.png?branch=master
+.. _buildstatus: http://travis-ci.org/#!/andialbrecht/sqlparse
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/TODO
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/TODO b/shell/ext-py/sqlparse-0.1.19/TODO
new file mode 100644
index 0000000..166df20
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/TODO
@@ -0,0 +1,7 @@
+* See
+ https://groups.google.com/d/msg/sqlparse/huz9lKXt0Lc/11ybIKPJWbUJ
+ for some interesting hints and suggestions.
+* Provide a function to replace tokens. See this thread: https://groups.google.com/d/msg/sqlparse/5xmBL2UKqX4/ZX9z_peve-AJ
+* Fix bugs on issue tracker.
+* Document filter stack and processing phases.
+* See KnownIssues http://code.google.com/p/python-sqlparse/wiki/KnownIssues
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/bin/sqlformat
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/bin/sqlformat b/shell/ext-py/sqlparse-0.1.19/bin/sqlformat
new file mode 100755
index 0000000..cecbed9
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/bin/sqlformat
@@ -0,0 +1,109 @@
+#!/usr/bin/env python
+
+# Copyright (C) 2008 Andi Albrecht, albrecht.andi@gmail.com
+#
+# This module is part of python-sqlparse and is released under
+# the BSD License: http://www.opensource.org/licenses/bsd-license.php.
+
+import optparse
+import os
+import sys
+
+import sqlparse
+from sqlparse.exceptions import SQLParseError
+
+
+_CASE_CHOICES = ['upper', 'lower', 'capitalize']
+
+
+parser = optparse.OptionParser(usage='%prog [OPTIONS] FILE, ...',
+ version='%%prog %s' % sqlparse.__version__)
+parser.set_description(('Format FILE according to OPTIONS. Use "-" as FILE '
+ 'to read from stdin.'))
+parser.add_option('-v', '--verbose', dest='verbose', action='store_true')
+parser.add_option('-o', '--outfile', dest='outfile', metavar='FILE',
+ help='write output to FILE (defaults to stdout)')
+group = parser.add_option_group('Formatting Options')
+group.add_option('-k', '--keywords', metavar='CHOICE',
+ dest='keyword_case', choices=_CASE_CHOICES,
+ help=('change case of keywords, CHOICE is one of %s'
+ % ', '.join('"%s"' % x for x in _CASE_CHOICES)))
+group.add_option('-i', '--identifiers', metavar='CHOICE',
+ dest='identifier_case', choices=_CASE_CHOICES,
+ help=('change case of identifiers, CHOICE is one of %s'
+ % ', '.join('"%s"' % x for x in _CASE_CHOICES)))
+group.add_option('-l', '--language', metavar='LANG',
+ dest='output_format', choices=['python', 'php'],
+ help=('output a snippet in programming language LANG, '
+ 'choices are "python", "php"'))
+group.add_option('--strip-comments', dest='strip_comments',
+ action='store_true', default=False,
+ help='remove comments')
+group.add_option('-r', '--reindent', dest='reindent',
+ action='store_true', default=False,
+ help='reindent statements')
+group.add_option('--indent_width', dest='indent_width', default=2,
+ help='indentation width (defaults to 2 spaces)')
+
+_FORMATTING_GROUP = group
+
+
+def _error(msg, exit_=None):
+ """Print msg and optionally exit with return code exit_."""
+ sys.stderr.write('[ERROR] %s\n' % msg)
+ if exit_ is not None:
+ sys.exit(exit_)
+
+
+def _build_formatter_opts(options):
+ """Convert command line options to dictionary."""
+ d = {}
+ for option in _FORMATTING_GROUP.option_list:
+ d[option.dest] = getattr(options, option.dest)
+ return d
+
+
+def main():
+ options, args = parser.parse_args()
+ if options.verbose:
+ sys.stderr.write('Verbose mode\n')
+
+ if len(args) != 1:
+ _error('No input data.')
+ parser.print_usage()
+ sys.exit(1)
+
+ if '-' in args: # read from stdin
+ data = sys.stdin.read()
+ else:
+ try:
+ data = ''.join(open(args[0]).readlines())
+ except OSError:
+ err = sys.exc_info()[1] # Python 2.5 compatibility
+ _error('Failed to read %s: %s' % (args[0], err), exit_=1)
+
+ if options.outfile:
+ try:
+ stream = open(options.outfile, 'w')
+ except OSError:
+ err = sys.exc_info()[1] # Python 2.5 compatibility
+ _error('Failed to open %s: %s' % (options.outfile, err), exit_=1)
+ else:
+ stream = sys.stdout
+
+ formatter_opts = _build_formatter_opts(options)
+ try:
+ formatter_opts = sqlparse.formatter.validate_options(formatter_opts)
+ except SQLParseError:
+ err = sys.exc_info()[1] # Python 2.5 compatibility
+ _error('Invalid options: %s' % err, exit_=1)
+
+ s = sqlparse.format(data, **formatter_opts)
+ if sys.version_info < (3,):
+ s = s.encode('utf-8', 'replace')
+ stream.write(s)
+ stream.flush()
+
+
+if __name__ == '__main__':
+ main()
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/docs/source/analyzing.rst
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/docs/source/analyzing.rst b/shell/ext-py/sqlparse-0.1.19/docs/source/analyzing.rst
new file mode 100644
index 0000000..5af5350
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/docs/source/analyzing.rst
@@ -0,0 +1,64 @@
+.. _analyze:
+
+Analyzing the Parsed Statement
+==============================
+
+When the :meth:`~sqlparse.parse` function is called the returned value
+is a tree-ish representation of the analyzed statements. The returned
+objects can be used by applications to retrieve further information about
+the parsed SQL.
+
+
+Base Classes
+------------
+
+All returned objects inherit from these base classes.
+The :class:`~sqlparse.sql.Token` class represents a single token and
+:class:`~sqlparse.sql.TokenList` class is a group of tokens.
+The latter provides methods for inspecting it's child tokens.
+
+.. autoclass:: sqlparse.sql.Token
+ :members:
+
+.. autoclass:: sqlparse.sql.TokenList
+ :members:
+
+
+SQL Representing Classes
+------------------------
+
+The following classes represent distinct parts of a SQL statement.
+
+.. autoclass:: sqlparse.sql.Statement
+ :members:
+
+.. autoclass:: sqlparse.sql.Comment
+ :members:
+
+.. autoclass:: sqlparse.sql.Identifier
+ :members:
+
+.. autoclass:: sqlparse.sql.IdentifierList
+ :members:
+
+.. autoclass:: sqlparse.sql.Where
+ :members:
+
+.. autoclass:: sqlparse.sql.Case
+ :members:
+
+.. autoclass:: sqlparse.sql.Parenthesis
+ :members:
+
+.. autoclass:: sqlparse.sql.If
+ :members:
+
+.. autoclass:: sqlparse.sql.For
+ :members:
+
+.. autoclass:: sqlparse.sql.Assignment
+ :members:
+
+.. autoclass:: sqlparse.sql.Comparison
+ :members:
+
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/docs/source/api.rst
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/docs/source/api.rst b/shell/ext-py/sqlparse-0.1.19/docs/source/api.rst
new file mode 100644
index 0000000..518a428
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/docs/source/api.rst
@@ -0,0 +1,57 @@
+:mod:`sqlparse` -- Parse SQL statements
+=======================================
+
+.. module:: sqlparse
+ :synopsis: Parse SQL statements.
+
+The :mod:`sqlparse` module provides the following functions on module-level.
+
+.. autofunction:: sqlparse.split
+
+.. autofunction:: sqlparse.format
+
+.. autofunction:: sqlparse.parse
+
+In most cases there's no need to set the `encoding` parameter. If
+`encoding` is not set, sqlparse assumes that the given SQL statement
+is encoded either in utf-8 or latin-1.
+
+
+.. _formatting:
+
+Formatting of SQL Statements
+----------------------------
+
+The :meth:`~sqlparse.format` function accepts the following keyword arguments.
+
+``keyword_case``
+ Changes how keywords are formatted. Allowed values are "upper", "lower"
+ and "capitalize".
+
+``identifier_case``
+ Changes how identifiers are formatted. Allowed values are "upper", "lower",
+ and "capitalize".
+
+``strip_comments``
+ If ``True`` comments are removed from the statements.
+
+``truncate_strings``
+ If ``truncate_strings`` is a positive integer, string literals longer than
+ the given value will be truncated.
+
+``truncate_char`` (default: "[...]")
+ If long string literals are truncated (see above) this value will be append
+ to the truncated string.
+
+``reindent``
+ If ``True`` the indentations of the statements are changed.
+
+``indent_tabs``
+ If ``True`` tabs instead of spaces are used for indentation.
+
+``indent_width``
+ The width of the indentation, defaults to 2.
+
+``output_format``
+ If given the output is additionally formatted to be used as a variable
+ in a programming language. Allowed values are "python" and "php".
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/docs/source/changes.rst
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/docs/source/changes.rst b/shell/ext-py/sqlparse-0.1.19/docs/source/changes.rst
new file mode 100644
index 0000000..7c1b861
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/docs/source/changes.rst
@@ -0,0 +1,13 @@
+.. _changes:
+
+Changes in python-sqlparse
+==========================
+
+Upcoming Deprecations
+---------------------
+
+* ``sqlparse.SQLParseError`` is deprecated (version 0.1.5), use
+ ``sqlparse.exceptions.SQLParseError`` instead.
+
+.. include:: ../../CHANGES
+
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/docs/source/conf.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/docs/source/conf.py b/shell/ext-py/sqlparse-0.1.19/docs/source/conf.py
new file mode 100644
index 0000000..5f7d34f
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/docs/source/conf.py
@@ -0,0 +1,200 @@
+# -*- coding: utf-8 -*-
+#
+# python-sqlparse documentation build configuration file, created by
+# sphinx-quickstart on Thu Feb 26 08:19:28 2009.
+#
+# This file is execfile()d with the current directory set to its containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+import datetime
+import sys, os
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#sys.path.append(os.path.abspath('.'))
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../../'))
+
+import sqlparse
+
+# -- General configuration -----------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be extensions
+# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo', 'sphinx.ext.coverage',
+ 'sphinx.ext.autosummary']
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The encoding of source files.
+#source_encoding = 'utf-8'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'python-sqlparse'
+copyright = u'%s, Andi Albrecht' % datetime.date.today().strftime('%Y')
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = sqlparse.__version__
+# The full version, including alpha/beta/rc tags.
+release = sqlparse.__version__
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+#today_fmt = '%B %d, %Y'
+
+# List of documents that shouldn't be included in the build.
+#unused_docs = []
+
+# List of directories, relative to source directory, that shouldn't be searched
+# for source files.
+exclude_trees = []
+
+# The reST default role (used for this markup: `text`) to use for all documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'tango'
+
+# A list of ignored prefixes for module index sorting.
+#modindex_common_prefix = []
+
+
+# -- Options for HTML output ---------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages. Major themes that come with
+# Sphinx are currently 'default' and 'sphinxdoc'.
+#html_theme = 'agogo'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further. For a list of options available for each theme, see the
+# documentation.
+#html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+#html_theme_path = [os.path.abspath('../')]
+
+# The name for this set of Sphinx documents. If None, it defaults to
+# "<project> v<release> documentation".
+#html_title = None
+
+# A shorter title for the navigation bar. Default is the same as html_title.
+#html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+#html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+#html_static_path = ['_static']
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+#html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_use_modindex = True
+
+# If false, no index is generated.
+#html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#html_show_sourcelink = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it. The value of this option must be the
+# base URL from which the finished HTML is served.
+#html_use_opensearch = ''
+
+# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
+#html_file_suffix = ''
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'python-sqlparsedoc'
+
+
+# -- Options for LaTeX output --------------------------------------------------
+
+# The paper size ('letter' or 'a4').
+#latex_paper_size = 'letter'
+
+# The font size ('10pt', '11pt' or '12pt').
+#latex_font_size = '10pt'
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title, author, documentclass [howto/manual]).
+latex_documents = [
+ ('index', 'python-sqlparse.tex', ur'python-sqlparse Documentation',
+ ur'Andi Albrecht', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#latex_use_parts = False
+
+# Additional stuff for the LaTeX preamble.
+#latex_preamble = ''
+
+# Documents to append as an appendix to all manuals.
+#latex_appendices = []
+
+# If false, no module index is generated.
+#latex_use_modindex = True
+todo_include_todos = True
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/docs/source/index.rst
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/docs/source/index.rst b/shell/ext-py/sqlparse-0.1.19/docs/source/index.rst
new file mode 100644
index 0000000..5eb76f7
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/docs/source/index.rst
@@ -0,0 +1,61 @@
+.. python-sqlparse documentation master file, created by
+ sphinx-quickstart on Thu Feb 26 08:19:28 2009.
+ You can adapt this file completely to your liking, but it should at least
+ contain the root `toctree` directive.
+
+python-sqlparse
+===============
+
+:mod:`sqlparse` is a non-validating SQL parser for Python.
+It provides support for parsing, splitting and formatting SQL statements.
+
+The module is compatible with Python 2 (>= 2.5) and Python 3 (>= 3.2)
+and released under the terms of the `New BSD license
+<http://www.opensource.org/licenses/bsd-license.php>`_.
+
+Visit the project page at https://github.com/andialbrecht/sqlparse for
+further information about this project.
+
+
+tl;dr
+-----
+
+.. code-block:: bash
+
+ $ pip install sqlparse
+ $ python
+ >>> import sqlparse
+ >>> print(sqlparse.format('select * from foo', reindent=True))
+ select *
+ from foo
+ >>> parsed = sqlparse.parse('select * from foo')[0]
+ >>> parsed.tokens
+ [<DML 'select' at 0x7f22c5e15368>, <Whitespace ' ' at 0x7f22c5e153b0>, <Wildcard '*' … ]
+ >>>
+
+
+Contents
+--------
+
+.. toctree::
+ :maxdepth: 2
+
+ intro
+ api
+ analyzing
+ ui
+ changes
+ indices
+
+
+Resources
+---------
+
+Project page
+ https://github.com/andialbrecht/sqlparse
+
+Bug tracker
+ https://github.com/andialbrecht/sqlparse/issues
+
+Documentation
+ http://sqlparse.readthedocs.org/
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/docs/source/indices.rst
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/docs/source/indices.rst b/shell/ext-py/sqlparse-0.1.19/docs/source/indices.rst
new file mode 100644
index 0000000..f74c5d8
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/docs/source/indices.rst
@@ -0,0 +1,7 @@
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
+
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/docs/source/intro.rst
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/docs/source/intro.rst b/shell/ext-py/sqlparse-0.1.19/docs/source/intro.rst
new file mode 100644
index 0000000..76d8fba
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/docs/source/intro.rst
@@ -0,0 +1,143 @@
+Introduction
+============
+
+
+Download & Installation
+-----------------------
+
+The latest released version can be obtained from the `Python Package
+Index (PyPI) <http://pypi.python.org/pypi/sqlparse/>`_. To extract the
+install the module system-wide run
+
+.. code-block:: bash
+
+ $ tar cvfz python-sqlparse-VERSION.tar.gz
+ $ cd python-sqlparse/
+ $ sudo python setup.py install
+
+Alternatively you can install :mod:`sqlparse` using :command:`pip`:
+
+.. code-block:: bash
+
+ $ pip install sqlparse
+
+
+Getting Started
+---------------
+
+The :mod:`sqlparse` module provides three simple functions on module level
+to achieve some common tasks when working with SQL statements.
+This section shows some simple usage examples of these functions.
+
+Let's get started with splitting a string containing one or more SQL
+statements into a list of single statements using :meth:`~sqlparse.split`:
+
+.. code-block:: python
+
+ >>> import sqlparse
+ >>> sql = 'select * from foo; select * from bar;'
+ >>> sqlparse.split(sql)
+ [u'select * from foo; ', u'select * from bar;']
+
+The end of a statement is identified by the occurrence of a semicolon.
+Semicolons within certain SQL constructs like ``BEGIN ... END`` blocks
+are handled correctly by the splitting mechanism.
+
+SQL statements can be beautified by using the :meth:`~sqlarse.format` function.
+
+.. code-block:: python
+
+ >>> sql = 'select * from foo where id in (select id from bar);'
+ >>> print sqlparse.format(sql, reindent=True, keyword_case='upper')
+ SELECT *
+ FROM foo
+ WHERE id IN
+ (SELECT id
+ FROM bar);
+
+In this case all keywords in the given SQL are uppercased and the
+indentation is changed to make it more readable. Read :ref:`formatting` for
+a full reference of supported options given as keyword arguments
+to that function.
+
+Before proceeding with a closer look at the internal representation of
+SQL statements, you should be aware that this SQL parser is intentionally
+non-validating. It assumes that the given input is at least some kind
+of SQL and then it tries to analyze as much as possible without making
+too much assumptions about the concrete dialect or the actual statement.
+At least it's up to the user of this API to interpret the results right.
+
+When using the :meth:`~sqlparse.parse` function a tuple of
+:class:`~sqlparse.sql.Statement` instances is returned:
+
+.. code-block:: python
+
+ >>> sql = 'select * from "someschema"."mytable" where id = 1'
+ >>> parsed = sqlparse.parse(sql)
+ >>> parsed
+ (<Statement 'select...' at 0x9ad08ec>,)
+
+Each item of the tuple is a single statement as identified by the above
+mentioned :meth:`~sqlparse.split` function. So let's grab the only element
+from that list and have a look at the ``tokens`` attribute.
+Sub-tokens are stored in this attribute.
+
+.. code-block:: python
+
+ >>> stmt = parsed[0] # grab the Statement object
+ >>> stmt.tokens
+ (<DML 'select' at 0x9b63c34>,
+ <Whitespace ' ' at 0x9b63e8c>,
+ <Operator '*' at 0x9b63e64>,
+ <Whitespace ' ' at 0x9b63c5c>,
+ <Keyword 'from' at 0x9b63c84>,
+ <Whitespace ' ' at 0x9b63cd4>,
+ <Identifier '"somes...' at 0x9b5c62c>,
+ <Whitespace ' ' at 0x9b63f04>,
+ <Where 'where ...' at 0x9b5caac>)
+
+Each object can be converted back to a string at any time:
+
+.. code-block:: python
+
+ >>> unicode(stmt) # str(stmt) for Python 3
+ u'select * from "someschema"."mytable" where id = 1'
+ >>> unicode(stmt.tokens[-1]) # or just the WHERE part
+ u'where id = 1'
+
+Details of the returned objects are described in :ref:`analyze`.
+
+
+Development & Contributing
+--------------------------
+
+To check out the latest sources of this module run
+
+.. code-block:: bash
+
+ $ git clone git://github.com/andialbrecht/sqlparse.git
+
+
+to check out the latest sources from the repository.
+
+:mod:`sqlparse` is currently tested under Python 2.5, 2.6, 2.7, 3.2 and
+pypy. Tests are automatically run on each commit and for each pull
+request on Travis: https://travis-ci.org/andialbrecht/sqlparse
+
+Make sure to run the test suite before sending a pull request by running
+
+.. code-block:: bash
+
+ $ tox
+
+It's ok, if :command:`tox` doesn't find all interpreters listed
+above. Ideally a Python 2 and a Python 3 version should be tested
+locally.
+
+Please file bug reports and feature requests on the project site at
+https://github.com/andialbrecht/sqlparse/issues/new or if you have
+code to contribute upload it to http://codereview.appspot.com and
+add albrecht.andi@googlemail.com as reviewer.
+
+For more information about the review tool and how to use it visit
+it's project page: http://code.google.com/p/rietveld.
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/docs/source/ui.rst
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/docs/source/ui.rst b/shell/ext-py/sqlparse-0.1.19/docs/source/ui.rst
new file mode 100644
index 0000000..264916e
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/docs/source/ui.rst
@@ -0,0 +1,15 @@
+User Interfaces
+===============
+
+``sqlformat``
+ The ``sqlformat`` command line script ist distributed with the module.
+ Run :command:`sqlformat --help` to list available options and for usage
+ hints.
+
+``sqlformat.appspot.com``
+ An example `Google App Engine <http://code.google.com/appengine/>`_
+ application that exposes the formatting features using a web front-end.
+ See http://sqlformat.appspot.com for details.
+ The source for this application is available from a source code check out
+ of the :mod:`sqlparse` module (see :file:`extras/appengine`).
+
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/docs/sqlformat.1
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/docs/sqlformat.1 b/shell/ext-py/sqlparse-0.1.19/docs/sqlformat.1
new file mode 100644
index 0000000..a42ea60
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/docs/sqlformat.1
@@ -0,0 +1,65 @@
+.\" Based on template /usr/share/man-db/examples/manpage.example provided by
+.\" Tom Christiansen <tc...@jhereg.perl.com>.
+.TH SQLFORMAT "1" "December 2010" "python-sqlparse version: 0.1.2" "User Commands"
+.SH NAME
+sqlformat \- reformat SQL
+.SH SYNOPSIS
+.PP
+.B sqlformat
+[
+.I "OPTION"
+] ... [
+.I "FILE"
+] ...
+.SH DESCRIPTION
+.\" Putting a newline after each sentence can generate better output.
+The `sqlformat' command-line tool can be used to reformat SQL file according to
+specified options or prepare a snippet in in some programming language (only
+Python and PHP currently supported).
+Use "-" for
+.I FILE
+to read from stdin.
+.SH OPTIONS
+.TP
+\fB\-i\fR \fICHOICE\fR|\fB\-\-identifiers\fR=\fIFORMAT\fR
+Change case of identifiers.
+.I FORMAT
+is one of "upper", "lower", "capitalize".
+.TP
+\fB\-k\fR \fICHOICE\fR|\fB\-\-keywords\fR=\fIFORMAT\fR
+Change case of keywords.
+.I FORMAT
+is one of "upper", "lower", "capitalize".
+.TP
+\fB\-l\fR \fICHOICE\fR|\fB\-\-language\fR=\fILANG\fR
+Output a snippet in programming language LANG.
+.I LANG
+can be "python", "php".
+.TP
+\fB\-o\fR \fIFILE\fR|\fB\-\-outfile\fR=\fIFILE\fR
+Write output to
+.I FILE
+(defaults to stdout).
+.TP
+.BR \-r | \-\-reindent
+Reindent statements.
+.TP
+\fB\-\-indent_width\fR=\fIINDENT_WIDTH\fR
+Set indent width to
+.IR INDENT_WIDTH .
+Default is 2 spaces.
+.TP
+\fB\-\-strip\-comments
+Remove comments.
+.TP
+.BR \-h | \-\-help
+Print a short help message and exit.
+All subsequent options are ignored.
+.TP
+.BR --verbose
+Verbose output.
+.TP
+.BR \-\-version
+Print program's version number and exit.
+.SH AUTHORS
+This man page was written by Andriy Senkovych <jo...@itblog.org.ua>
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/pytest.ini
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/pytest.ini b/shell/ext-py/sqlparse-0.1.19/pytest.ini
new file mode 100644
index 0000000..a2cbd90
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/pytest.ini
@@ -0,0 +1,6 @@
+[pytest]
+pep8ignore =
+ extras/* ALL
+ examples/* ALL
+ docs/* ALL
+ * E125 E127
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/setup.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/setup.py b/shell/ext-py/sqlparse-0.1.19/setup.py
new file mode 100644
index 0000000..2c6dce8
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/setup.py
@@ -0,0 +1,123 @@
+# Copyright (C) 2008 Andi Albrecht, albrecht.andi@gmail.com
+#
+# This setup script is part of python-sqlparse and is released under
+# the BSD License: http://www.opensource.org/licenses/bsd-license.php.
+
+import re
+import sys
+
+try:
+ from setuptools import setup, find_packages
+ packages = find_packages(exclude=('tests',))
+except ImportError:
+ if sys.version_info[0] == 3:
+ raise RuntimeError('distribute is required to install this package.')
+ from distutils.core import setup
+ packages = ['sqlparse', 'sqlparse.engine']
+
+
+def get_version():
+ """parse __init__.py for version number instead of importing the file
+
+ see http://stackoverflow.com/questions/458550/standard-way-to-embed-version-into-python-package
+ """
+ VERSIONFILE='sqlparse/__init__.py'
+ verstrline = open(VERSIONFILE, "rt").read()
+ VSRE = r'^__version__ = [\'"]([^\'"]*)[\'"]'
+ mo = re.search(VSRE, verstrline, re.M)
+ if mo:
+ return mo.group(1)
+ else:
+ raise RuntimeError('Unable to find version string in %s.'
+ % (VERSIONFILE,))
+
+
+LONG_DESCRIPTION = """
+``sqlparse`` is a non-validating SQL parser module.
+It provides support for parsing, splitting and formatting SQL statements.
+
+Visit the `project page <https://github.com/andialbrecht/sqlparse>`_ for
+additional information and documentation.
+
+**Example Usage**
+
+
+Splitting SQL statements::
+
+ >>> import sqlparse
+ >>> sqlparse.split('select * from foo; select * from bar;')
+ [u'select * from foo; ', u'select * from bar;']
+
+
+Formatting statemtents::
+
+ >>> sql = 'select * from foo where id in (select id from bar);'
+ >>> print sqlparse.format(sql, reindent=True, keyword_case='upper')
+ SELECT *
+ FROM foo
+ WHERE id IN
+ (SELECT id
+ FROM bar);
+
+
+Parsing::
+
+ >>> sql = 'select * from someschema.mytable where id = 1'
+ >>> res = sqlparse.parse(sql)
+ >>> res
+ (<Statement 'select...' at 0x9ad08ec>,)
+ >>> stmt = res[0]
+ >>> unicode(stmt) # converting it back to unicode
+ u'select * from someschema.mytable where id = 1'
+ >>> # This is how the internal representation looks like:
+ >>> stmt.tokens
+ (<DML 'select' at 0x9b63c34>,
+ <Whitespace ' ' at 0x9b63e8c>,
+ <Operator '*' at 0x9b63e64>,
+ <Whitespace ' ' at 0x9b63c5c>,
+ <Keyword 'from' at 0x9b63c84>,
+ <Whitespace ' ' at 0x9b63cd4>,
+ <Identifier 'somes...' at 0x9b5c62c>,
+ <Whitespace ' ' at 0x9b63f04>,
+ <Where 'where ...' at 0x9b5caac>)
+
+"""
+
+VERSION = get_version()
+
+
+kwargs = {}
+if sys.version_info[0] == 3:
+ kwargs['use_2to3'] = True
+
+
+setup(
+ name='sqlparse',
+ version=VERSION,
+ packages=packages,
+ description='Non-validating SQL parser',
+ author='Andi Albrecht',
+ author_email='albrecht.andi@gmail.com',
+ long_description=LONG_DESCRIPTION,
+ license='BSD',
+ url='https://github.com/andialbrecht/sqlparse',
+ classifiers=[
+ 'Development Status :: 4 - Beta',
+ 'Intended Audience :: Developers',
+ 'License :: OSI Approved :: BSD License',
+ 'Operating System :: OS Independent',
+ 'Programming Language :: Python',
+ 'Programming Language :: Python :: 2',
+ 'Programming Language :: Python :: 2.4',
+ 'Programming Language :: Python :: 2.5',
+ 'Programming Language :: Python :: 2.6',
+ 'Programming Language :: Python :: 2.7',
+ 'Programming Language :: Python :: 3',
+ 'Programming Language :: Python :: 3.2',
+ 'Programming Language :: Python :: 3.3',
+ 'Topic :: Database',
+ 'Topic :: Software Development'
+ ],
+ scripts=['bin/sqlformat'],
+ **kwargs
+)
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/sqlparse/__init__.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/sqlparse/__init__.py b/shell/ext-py/sqlparse-0.1.19/sqlparse/__init__.py
new file mode 100644
index 0000000..238bfb6
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/sqlparse/__init__.py
@@ -0,0 +1,78 @@
+# Copyright (C) 2008 Andi Albrecht, albrecht.andi@gmail.com
+#
+# This module is part of python-sqlparse and is released under
+# the BSD License: http://www.opensource.org/licenses/bsd-license.php.
+
+"""Parse SQL statements."""
+
+
+__version__ = '0.1.19'
+
+
+# Setup namespace
+from sqlparse import engine
+from sqlparse import filters
+from sqlparse import formatter
+
+# Deprecated in 0.1.5. Will be removed in 0.2.0
+from sqlparse.exceptions import SQLParseError
+
+
+def parse(sql, encoding=None):
+ """Parse sql and return a list of statements.
+
+ :param sql: A string containting one or more SQL statements.
+ :param encoding: The encoding of the statement (optional).
+ :returns: A tuple of :class:`~sqlparse.sql.Statement` instances.
+ """
+ return tuple(parsestream(sql, encoding))
+
+
+def parsestream(stream, encoding=None):
+ """Parses sql statements from file-like object.
+
+ :param stream: A file-like object.
+ :param encoding: The encoding of the stream contents (optional).
+ :returns: A generator of :class:`~sqlparse.sql.Statement` instances.
+ """
+ stack = engine.FilterStack()
+ stack.full_analyze()
+ return stack.run(stream, encoding)
+
+
+def format(sql, **options):
+ """Format *sql* according to *options*.
+
+ Available options are documented in :ref:`formatting`.
+
+ In addition to the formatting options this function accepts the
+ keyword "encoding" which determines the encoding of the statement.
+
+ :returns: The formatted SQL statement as string.
+ """
+ encoding = options.pop('encoding', None)
+ stack = engine.FilterStack()
+ options = formatter.validate_options(options)
+ stack = formatter.build_filter_stack(stack, options)
+ stack.postprocess.append(filters.SerializerUnicode())
+ return ''.join(stack.run(sql, encoding))
+
+
+def split(sql, encoding=None):
+ """Split *sql* into single statements.
+
+ :param sql: A string containting one or more SQL statements.
+ :param encoding: The encoding of the statement (optional).
+ :returns: A list of strings.
+ """
+ stack = engine.FilterStack()
+ stack.split_statements = True
+ return [unicode(stmt).strip() for stmt in stack.run(sql, encoding)]
+
+
+from sqlparse.engine.filter import StatementFilter
+
+
+def split2(stream):
+ splitter = StatementFilter()
+ return list(splitter.process(None, stream))
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/sqlparse/engine/__init__.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/sqlparse/engine/__init__.py b/shell/ext-py/sqlparse-0.1.19/sqlparse/engine/__init__.py
new file mode 100644
index 0000000..62c82b8
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/sqlparse/engine/__init__.py
@@ -0,0 +1,80 @@
+# Copyright (C) 2008 Andi Albrecht, albrecht.andi@gmail.com
+#
+# This module is part of python-sqlparse and is released under
+# the BSD License: http://www.opensource.org/licenses/bsd-license.php.
+
+"""filter"""
+
+from sqlparse import lexer
+from sqlparse.engine import grouping
+from sqlparse.engine.filter import StatementFilter
+
+# XXX remove this when cleanup is complete
+Filter = object
+
+
+class FilterStack(object):
+
+ def __init__(self):
+ self.preprocess = []
+ self.stmtprocess = []
+ self.postprocess = []
+ self.split_statements = False
+ self._grouping = False
+
+ def _flatten(self, stream):
+ for token in stream:
+ if token.is_group():
+ for t in self._flatten(token.tokens):
+ yield t
+ else:
+ yield token
+
+ def enable_grouping(self):
+ self._grouping = True
+
+ def full_analyze(self):
+ self.enable_grouping()
+
+ def run(self, sql, encoding=None):
+ stream = lexer.tokenize(sql, encoding)
+ # Process token stream
+ if self.preprocess:
+ for filter_ in self.preprocess:
+ stream = filter_.process(self, stream)
+
+ if (self.stmtprocess or self.postprocess or self.split_statements
+ or self._grouping):
+ splitter = StatementFilter()
+ stream = splitter.process(self, stream)
+
+ if self._grouping:
+
+ def _group(stream):
+ for stmt in stream:
+ grouping.group(stmt)
+ yield stmt
+ stream = _group(stream)
+
+ if self.stmtprocess:
+
+ def _run1(stream):
+ ret = []
+ for stmt in stream:
+ for filter_ in self.stmtprocess:
+ filter_.process(self, stmt)
+ ret.append(stmt)
+ return ret
+ stream = _run1(stream)
+
+ if self.postprocess:
+
+ def _run2(stream):
+ for stmt in stream:
+ stmt.tokens = list(self._flatten(stmt.tokens))
+ for filter_ in self.postprocess:
+ stmt = filter_.process(self, stmt)
+ yield stmt
+ stream = _run2(stream)
+
+ return stream
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/sqlparse/engine/filter.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/sqlparse/engine/filter.py b/shell/ext-py/sqlparse-0.1.19/sqlparse/engine/filter.py
new file mode 100644
index 0000000..f7dd264
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/sqlparse/engine/filter.py
@@ -0,0 +1,112 @@
+# -*- coding: utf-8 -*-
+
+from sqlparse.sql import Statement, Token
+from sqlparse import tokens as T
+
+
+class StatementFilter:
+ "Filter that split stream at individual statements"
+
+ def __init__(self):
+ self._in_declare = False
+ self._in_dbldollar = False
+ self._is_create = False
+ self._begin_depth = 0
+
+ def _reset(self):
+ "Set the filter attributes to its default values"
+ self._in_declare = False
+ self._in_dbldollar = False
+ self._is_create = False
+ self._begin_depth = 0
+
+ def _change_splitlevel(self, ttype, value):
+ "Get the new split level (increase, decrease or remain equal)"
+ # PostgreSQL
+ if (ttype == T.Name.Builtin
+ and value.startswith('$') and value.endswith('$')):
+ if self._in_dbldollar:
+ self._in_dbldollar = False
+ return -1
+ else:
+ self._in_dbldollar = True
+ return 1
+ elif self._in_dbldollar:
+ return 0
+
+ # ANSI
+ if ttype not in T.Keyword:
+ return 0
+
+ unified = value.upper()
+
+ if unified == 'DECLARE' and self._is_create and self._begin_depth == 0:
+ self._in_declare = True
+ return 1
+
+ if unified == 'BEGIN':
+ self._begin_depth += 1
+ if self._in_declare or self._is_create:
+ # FIXME(andi): This makes no sense.
+ return 1
+ return 0
+
+ if unified in ('END IF', 'END FOR'):
+ return -1
+
+ if unified == 'END':
+ # Should this respect a preceeding BEGIN?
+ # In CASE ... WHEN ... END this results in a split level -1.
+ self._begin_depth = max(0, self._begin_depth - 1)
+ return -1
+
+ if ttype is T.Keyword.DDL and unified.startswith('CREATE'):
+ self._is_create = True
+ return 0
+
+ if (unified in ('IF', 'FOR')
+ and self._is_create and self._begin_depth > 0):
+ return 1
+
+ # Default
+ return 0
+
+ def process(self, stack, stream):
+ "Process the stream"
+ consume_ws = False
+ splitlevel = 0
+ stmt = None
+ stmt_tokens = []
+
+ # Run over all stream tokens
+ for ttype, value in stream:
+ # Yield token if we finished a statement and there's no whitespaces
+ if consume_ws and ttype not in (T.Whitespace, T.Comment.Single):
+ stmt.tokens = stmt_tokens
+ yield stmt
+
+ # Reset filter and prepare to process next statement
+ self._reset()
+ consume_ws = False
+ splitlevel = 0
+ stmt = None
+
+ # Create a new statement if we are not currently in one of them
+ if stmt is None:
+ stmt = Statement()
+ stmt_tokens = []
+
+ # Change current split level (increase, decrease or remain equal)
+ splitlevel += self._change_splitlevel(ttype, value)
+
+ # Append the token to the current statement
+ stmt_tokens.append(Token(ttype, value))
+
+ # Check if we get the end of a statement
+ if splitlevel <= 0 and ttype is T.Punctuation and value == ';':
+ consume_ws = True
+
+ # Yield pending statement (if any)
+ if stmt is not None:
+ stmt.tokens = stmt_tokens
+ yield stmt
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/sqlparse/engine/grouping.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/sqlparse/engine/grouping.py b/shell/ext-py/sqlparse-0.1.19/sqlparse/engine/grouping.py
new file mode 100644
index 0000000..bb1f0b0
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/sqlparse/engine/grouping.py
@@ -0,0 +1,461 @@
+# -*- coding: utf-8 -*-
+
+import itertools
+
+from sqlparse import sql
+from sqlparse import tokens as T
+
+try:
+ next
+except NameError: # Python < 2.6
+ next = lambda i: i.next()
+
+
+def _group_left_right(tlist, ttype, value, cls,
+ check_right=lambda t: True,
+ check_left=lambda t: True,
+ include_semicolon=False):
+ [_group_left_right(sgroup, ttype, value, cls, check_right, check_left,
+ include_semicolon) for sgroup in tlist.get_sublists()
+ if not isinstance(sgroup, cls)]
+ idx = 0
+ token = tlist.token_next_match(idx, ttype, value)
+ while token:
+ right = tlist.token_next(tlist.token_index(token))
+ left = tlist.token_prev(tlist.token_index(token))
+ if right is None or not check_right(right):
+ token = tlist.token_next_match(tlist.token_index(token) + 1,
+ ttype, value)
+ elif left is None or not check_left(left):
+ token = tlist.token_next_match(tlist.token_index(token) + 1,
+ ttype, value)
+ else:
+ if include_semicolon:
+ sright = tlist.token_next_match(tlist.token_index(right),
+ T.Punctuation, ';')
+ if sright is not None:
+ # only overwrite "right" if a semicolon is actually
+ # present.
+ right = sright
+ tokens = tlist.tokens_between(left, right)[1:]
+ if not isinstance(left, cls):
+ new = cls([left])
+ new_idx = tlist.token_index(left)
+ tlist.tokens.remove(left)
+ tlist.tokens.insert(new_idx, new)
+ left = new
+ left.tokens.extend(tokens)
+ for t in tokens:
+ tlist.tokens.remove(t)
+ token = tlist.token_next_match(tlist.token_index(left) + 1,
+ ttype, value)
+
+
+def _find_matching(idx, tlist, start_ttype, start_value, end_ttype, end_value):
+ depth = 1
+ for tok in tlist.tokens[idx:]:
+ if tok.match(start_ttype, start_value):
+ depth += 1
+ elif tok.match(end_ttype, end_value):
+ depth -= 1
+ if depth == 1:
+ return tok
+ return None
+
+
+def _group_matching(tlist, start_ttype, start_value, end_ttype, end_value,
+ cls, include_semicolon=False, recurse=False):
+
+ [_group_matching(sgroup, start_ttype, start_value, end_ttype, end_value,
+ cls, include_semicolon) for sgroup in tlist.get_sublists()
+ if recurse]
+ if isinstance(tlist, cls):
+ idx = 1
+ else:
+ idx = 0
+ token = tlist.token_next_match(idx, start_ttype, start_value)
+ while token:
+ tidx = tlist.token_index(token)
+ end = _find_matching(tidx, tlist, start_ttype, start_value,
+ end_ttype, end_value)
+ if end is None:
+ idx = tidx + 1
+ else:
+ if include_semicolon:
+ next_ = tlist.token_next(tlist.token_index(end))
+ if next_ and next_.match(T.Punctuation, ';'):
+ end = next_
+ group = tlist.group_tokens(cls, tlist.tokens_between(token, end))
+ _group_matching(group, start_ttype, start_value,
+ end_ttype, end_value, cls, include_semicolon)
+ idx = tlist.token_index(group) + 1
+ token = tlist.token_next_match(idx, start_ttype, start_value)
+
+
+def group_if(tlist):
+ _group_matching(tlist, T.Keyword, 'IF', T.Keyword, 'END IF', sql.If, True)
+
+
+def group_for(tlist):
+ _group_matching(tlist, T.Keyword, 'FOR', T.Keyword, 'END LOOP',
+ sql.For, True)
+
+
+def group_foreach(tlist):
+ _group_matching(tlist, T.Keyword, 'FOREACH', T.Keyword, 'END LOOP',
+ sql.For, True)
+
+
+def group_begin(tlist):
+ _group_matching(tlist, T.Keyword, 'BEGIN', T.Keyword, 'END',
+ sql.Begin, True)
+
+
+def group_as(tlist):
+
+ def _right_valid(token):
+ # Currently limited to DML/DDL. Maybe additional more non SQL reserved
+ # keywords should appear here (see issue8).
+ return not token.ttype in (T.DML, T.DDL)
+
+ def _left_valid(token):
+ if token.ttype is T.Keyword and token.value in ('NULL',):
+ return True
+ return token.ttype is not T.Keyword
+
+ _group_left_right(tlist, T.Keyword, 'AS', sql.Identifier,
+ check_right=_right_valid,
+ check_left=_left_valid)
+
+
+def group_assignment(tlist):
+ _group_left_right(tlist, T.Assignment, ':=', sql.Assignment,
+ include_semicolon=True)
+
+
+def group_comparison(tlist):
+
+ def _parts_valid(token):
+ return (token.ttype in (T.String.Symbol, T.String.Single,
+ T.Name, T.Number, T.Number.Float,
+ T.Number.Integer, T.Literal,
+ T.Literal.Number.Integer, T.Name.Placeholder)
+ or isinstance(token, (sql.Identifier, sql.Parenthesis))
+ or (token.ttype is T.Keyword
+ and token.value.upper() in ['NULL', ]))
+ _group_left_right(tlist, T.Operator.Comparison, None, sql.Comparison,
+ check_left=_parts_valid, check_right=_parts_valid)
+
+
+def group_case(tlist):
+ _group_matching(tlist, T.Keyword, 'CASE', T.Keyword, 'END', sql.Case,
+ include_semicolon=True, recurse=True)
+
+
+def group_identifier(tlist):
+ def _consume_cycle(tl, i):
+ # TODO: Usage of Wildcard token is ambivalent here.
+ x = itertools.cycle((
+ lambda y: (y.match(T.Punctuation, '.')
+ or y.ttype in (T.Operator,
+ T.Wildcard,
+ T.Name)
+ or isinstance(y, sql.SquareBrackets)),
+ lambda y: (y.ttype in (T.String.Symbol,
+ T.Name,
+ T.Wildcard,
+ T.Literal.String.Single,
+ T.Literal.Number.Integer,
+ T.Literal.Number.Float)
+ or isinstance(y, (sql.Parenthesis,
+ sql.SquareBrackets,
+ sql.Function)))))
+ for t in tl.tokens[i:]:
+ # Don't take whitespaces into account.
+ if t.ttype is T.Whitespace:
+ yield t
+ continue
+ if next(x)(t):
+ yield t
+ else:
+ if isinstance(t, sql.Comment) and t.is_multiline():
+ yield t
+ return
+
+ def _next_token(tl, i):
+ # chooses the next token. if two tokens are found then the
+ # first is returned.
+ t1 = tl.token_next_by_type(
+ i, (T.String.Symbol, T.Name, T.Literal.Number.Integer,
+ T.Literal.Number.Float))
+
+ i1 = tl.token_index(t1, start=i) if t1 else None
+ t2_end = None if i1 is None else i1 + 1
+ t2 = tl.token_next_by_instance(i, (sql.Function, sql.Parenthesis), end=t2_end)
+
+ if t1 and t2:
+ i2 = tl.token_index(t2, start=i)
+ if i1 > i2:
+ return t2
+ else:
+ return t1
+ elif t1:
+ return t1
+ else:
+ return t2
+
+ # bottom up approach: group subgroups first
+ [group_identifier(sgroup) for sgroup in tlist.get_sublists()
+ if not isinstance(sgroup, sql.Identifier)]
+
+ # real processing
+ idx = 0
+ token = _next_token(tlist, idx)
+ while token:
+ identifier_tokens = [token] + list(
+ _consume_cycle(tlist,
+ tlist.token_index(token, start=idx) + 1))
+ # remove trailing whitespace
+ if identifier_tokens and identifier_tokens[-1].ttype is T.Whitespace:
+ identifier_tokens = identifier_tokens[:-1]
+ if not (len(identifier_tokens) == 1
+ and (isinstance(identifier_tokens[0], (sql.Function, sql.Parenthesis))
+ or identifier_tokens[0].ttype in (T.Literal.Number.Integer,
+ T.Literal.Number.Float))):
+ group = tlist.group_tokens(sql.Identifier, identifier_tokens)
+ idx = tlist.token_index(group, start=idx) + 1
+ else:
+ idx += 1
+ token = _next_token(tlist, idx)
+
+
+def group_identifier_list(tlist):
+ [group_identifier_list(sgroup) for sgroup in tlist.get_sublists()
+ if not isinstance(sgroup, sql.IdentifierList)]
+ # Allowed list items
+ fend1_funcs = [lambda t: isinstance(t, (sql.Identifier, sql.Function,
+ sql.Case)),
+ lambda t: t.is_whitespace(),
+ lambda t: t.ttype == T.Name,
+ lambda t: t.ttype == T.Wildcard,
+ lambda t: t.match(T.Keyword, 'null'),
+ lambda t: t.match(T.Keyword, 'role'),
+ lambda t: t.ttype == T.Number.Integer,
+ lambda t: t.ttype == T.String.Single,
+ lambda t: t.ttype == T.Name.Placeholder,
+ lambda t: t.ttype == T.Keyword,
+ lambda t: isinstance(t, sql.Comparison),
+ lambda t: isinstance(t, sql.Comment),
+ lambda t: t.ttype == T.Comment.Multiline,
+ ]
+ tcomma = tlist.token_next_match(0, T.Punctuation, ',')
+ start = None
+ while tcomma is not None:
+ # Go back one idx to make sure to find the correct tcomma
+ idx = tlist.token_index(tcomma)
+ before = tlist.token_prev(idx)
+ after = tlist.token_next(idx)
+ # Check if the tokens around tcomma belong to a list
+ bpassed = apassed = False
+ for func in fend1_funcs:
+ if before is not None and func(before):
+ bpassed = True
+ if after is not None and func(after):
+ apassed = True
+ if not bpassed or not apassed:
+ # Something's wrong here, skip ahead to next ","
+ start = None
+ tcomma = tlist.token_next_match(idx + 1,
+ T.Punctuation, ',')
+ else:
+ if start is None:
+ start = before
+ after_idx = tlist.token_index(after, start=idx)
+ next_ = tlist.token_next(after_idx)
+ if next_ is None or not next_.match(T.Punctuation, ','):
+ # Reached the end of the list
+ tokens = tlist.tokens_between(start, after)
+ group = tlist.group_tokens(sql.IdentifierList, tokens)
+ start = None
+ tcomma = tlist.token_next_match(tlist.token_index(group) + 1,
+ T.Punctuation, ',')
+ else:
+ tcomma = next_
+
+
+def group_brackets(tlist):
+ """Group parentheses () or square brackets []
+
+ This is just like _group_matching, but complicated by the fact that
+ round brackets can contain square bracket groups and vice versa
+ """
+
+ if isinstance(tlist, (sql.Parenthesis, sql.SquareBrackets)):
+ idx = 1
+ else:
+ idx = 0
+
+ # Find the first opening bracket
+ token = tlist.token_next_match(idx, T.Punctuation, ['(', '['])
+
+ while token:
+ start_val = token.value # either '(' or '['
+ if start_val == '(':
+ end_val = ')'
+ group_class = sql.Parenthesis
+ else:
+ end_val = ']'
+ group_class = sql.SquareBrackets
+
+ tidx = tlist.token_index(token)
+
+ # Find the corresponding closing bracket
+ end = _find_matching(tidx, tlist, T.Punctuation, start_val,
+ T.Punctuation, end_val)
+
+ if end is None:
+ idx = tidx + 1
+ else:
+ group = tlist.group_tokens(group_class,
+ tlist.tokens_between(token, end))
+
+ # Check for nested bracket groups within this group
+ group_brackets(group)
+ idx = tlist.token_index(group) + 1
+
+ # Find the next opening bracket
+ token = tlist.token_next_match(idx, T.Punctuation, ['(', '['])
+
+
+def group_comments(tlist):
+ [group_comments(sgroup) for sgroup in tlist.get_sublists()
+ if not isinstance(sgroup, sql.Comment)]
+ idx = 0
+ token = tlist.token_next_by_type(idx, T.Comment)
+ while token:
+ tidx = tlist.token_index(token)
+ end = tlist.token_not_matching(tidx + 1,
+ [lambda t: t.ttype in T.Comment,
+ lambda t: t.is_whitespace()])
+ if end is None:
+ idx = tidx + 1
+ else:
+ eidx = tlist.token_index(end)
+ grp_tokens = tlist.tokens_between(token,
+ tlist.token_prev(eidx, False))
+ group = tlist.group_tokens(sql.Comment, grp_tokens)
+ idx = tlist.token_index(group)
+ token = tlist.token_next_by_type(idx, T.Comment)
+
+
+def group_where(tlist):
+ [group_where(sgroup) for sgroup in tlist.get_sublists()
+ if not isinstance(sgroup, sql.Where)]
+ idx = 0
+ token = tlist.token_next_match(idx, T.Keyword, 'WHERE')
+ stopwords = ('ORDER', 'GROUP', 'LIMIT', 'UNION', 'EXCEPT', 'HAVING')
+ while token:
+ tidx = tlist.token_index(token)
+ end = tlist.token_next_match(tidx + 1, T.Keyword, stopwords)
+ if end is None:
+ end = tlist._groupable_tokens[-1]
+ else:
+ end = tlist.tokens[tlist.token_index(end) - 1]
+ group = tlist.group_tokens(sql.Where,
+ tlist.tokens_between(token, end),
+ ignore_ws=True)
+ idx = tlist.token_index(group)
+ token = tlist.token_next_match(idx, T.Keyword, 'WHERE')
+
+
+def group_aliased(tlist):
+ clss = (sql.Identifier, sql.Function, sql.Case)
+ [group_aliased(sgroup) for sgroup in tlist.get_sublists()
+ if not isinstance(sgroup, clss)]
+ idx = 0
+ token = tlist.token_next_by_instance(idx, clss)
+ while token:
+ next_ = tlist.token_next(tlist.token_index(token))
+ if next_ is not None and isinstance(next_, clss):
+ if not next_.value.upper().startswith('VARCHAR'):
+ grp = tlist.tokens_between(token, next_)[1:]
+ token.tokens.extend(grp)
+ for t in grp:
+ tlist.tokens.remove(t)
+ idx = tlist.token_index(token) + 1
+ token = tlist.token_next_by_instance(idx, clss)
+
+
+def group_typecasts(tlist):
+ _group_left_right(tlist, T.Punctuation, '::', sql.Identifier)
+
+
+def group_functions(tlist):
+ [group_functions(sgroup) for sgroup in tlist.get_sublists()
+ if not isinstance(sgroup, sql.Function)]
+ idx = 0
+ token = tlist.token_next_by_type(idx, T.Name)
+ while token:
+ next_ = tlist.token_next(token)
+ if not isinstance(next_, sql.Parenthesis):
+ idx = tlist.token_index(token) + 1
+ else:
+ func = tlist.group_tokens(sql.Function,
+ tlist.tokens_between(token, next_))
+ idx = tlist.token_index(func) + 1
+ token = tlist.token_next_by_type(idx, T.Name)
+
+
+def group_order(tlist):
+ idx = 0
+ token = tlist.token_next_by_type(idx, T.Keyword.Order)
+ while token:
+ prev = tlist.token_prev(token)
+ if isinstance(prev, sql.Identifier):
+ ido = tlist.group_tokens(sql.Identifier,
+ tlist.tokens_between(prev, token))
+ idx = tlist.token_index(ido) + 1
+ else:
+ idx = tlist.token_index(token) + 1
+ token = tlist.token_next_by_type(idx, T.Keyword.Order)
+
+
+def align_comments(tlist):
+ [align_comments(sgroup) for sgroup in tlist.get_sublists()]
+ idx = 0
+ token = tlist.token_next_by_instance(idx, sql.Comment)
+ while token:
+ before = tlist.token_prev(tlist.token_index(token))
+ if isinstance(before, sql.TokenList):
+ grp = tlist.tokens_between(before, token)[1:]
+ before.tokens.extend(grp)
+ for t in grp:
+ tlist.tokens.remove(t)
+ idx = tlist.token_index(before) + 1
+ else:
+ idx = tlist.token_index(token) + 1
+ token = tlist.token_next_by_instance(idx, sql.Comment)
+
+
+def group(tlist):
+ for func in [
+ group_comments,
+ group_brackets,
+ group_functions,
+ group_where,
+ group_case,
+ group_identifier,
+ group_order,
+ group_typecasts,
+ group_as,
+ group_aliased,
+ group_assignment,
+ group_comparison,
+ align_comments,
+ group_identifier_list,
+ group_if,
+ group_for,
+ group_foreach,
+ group_begin,
+ ]:
+ func(tlist)
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/sqlparse/exceptions.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/sqlparse/exceptions.py b/shell/ext-py/sqlparse-0.1.19/sqlparse/exceptions.py
new file mode 100644
index 0000000..ec25afa
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/sqlparse/exceptions.py
@@ -0,0 +1,10 @@
+# Copyright (C) 2012 Andi Albrecht, albrecht.andi@gmail.com
+#
+# This module is part of python-sqlparse and is released under
+# the BSD License: http://www.opensource.org/licenses/bsd-license.php.
+
+"""Exceptions used in this package."""
+
+
+class SQLParseError(Exception):
+ """Base class for exceptions in this module."""
[4/8] impala git commit: IMPALA-6999: Upgrade to sqlparse-0.1.19 for
Impala shell
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/sqlparse/filters.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/sqlparse/filters.py b/shell/ext-py/sqlparse-0.1.19/sqlparse/filters.py
new file mode 100644
index 0000000..676344f
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/sqlparse/filters.py
@@ -0,0 +1,728 @@
+# -*- coding: utf-8 -*-
+
+import re
+
+from os.path import abspath, join
+
+from sqlparse import sql, tokens as T
+from sqlparse.engine import FilterStack
+from sqlparse.lexer import tokenize
+from sqlparse.pipeline import Pipeline
+from sqlparse.tokens import (Comment, Comparison, Keyword, Name, Punctuation,
+ String, Whitespace)
+from sqlparse.utils import memoize_generator
+from sqlparse.utils import split_unquoted_newlines
+
+
+# --------------------------
+# token process
+
+class _CaseFilter:
+
+ ttype = None
+
+ def __init__(self, case=None):
+ if case is None:
+ case = 'upper'
+ assert case in ['lower', 'upper', 'capitalize']
+ self.convert = getattr(unicode, case)
+
+ def process(self, stack, stream):
+ for ttype, value in stream:
+ if ttype in self.ttype:
+ value = self.convert(value)
+ yield ttype, value
+
+
+class KeywordCaseFilter(_CaseFilter):
+ ttype = T.Keyword
+
+
+class IdentifierCaseFilter(_CaseFilter):
+ ttype = (T.Name, T.String.Symbol)
+
+ def process(self, stack, stream):
+ for ttype, value in stream:
+ if ttype in self.ttype and not value.strip()[0] == '"':
+ value = self.convert(value)
+ yield ttype, value
+
+
+class TruncateStringFilter:
+
+ def __init__(self, width, char):
+ self.width = max(width, 1)
+ self.char = unicode(char)
+
+ def process(self, stack, stream):
+ for ttype, value in stream:
+ if ttype is T.Literal.String.Single:
+ if value[:2] == '\'\'':
+ inner = value[2:-2]
+ quote = u'\'\''
+ else:
+ inner = value[1:-1]
+ quote = u'\''
+ if len(inner) > self.width:
+ value = u''.join((quote, inner[:self.width], self.char,
+ quote))
+ yield ttype, value
+
+
+class GetComments:
+ """Get the comments from a stack"""
+ def process(self, stack, stream):
+ for token_type, value in stream:
+ if token_type in Comment:
+ yield token_type, value
+
+
+class StripComments:
+ """Strip the comments from a stack"""
+ def process(self, stack, stream):
+ for token_type, value in stream:
+ if token_type not in Comment:
+ yield token_type, value
+
+
+def StripWhitespace(stream):
+ "Strip the useless whitespaces from a stream leaving only the minimal ones"
+ last_type = None
+ has_space = False
+ ignore_group = frozenset((Comparison, Punctuation))
+
+ for token_type, value in stream:
+ # We got a previous token (not empty first ones)
+ if last_type:
+ if token_type in Whitespace:
+ has_space = True
+ continue
+
+ # Ignore first empty spaces and dot-commas
+ elif token_type in (Whitespace, Whitespace.Newline, ignore_group):
+ continue
+
+ # Yield a whitespace if it can't be ignored
+ if has_space:
+ if not ignore_group.intersection((last_type, token_type)):
+ yield Whitespace, ' '
+ has_space = False
+
+ # Yield the token and set its type for checking with the next one
+ yield token_type, value
+ last_type = token_type
+
+
+class IncludeStatement:
+ """Filter that enable a INCLUDE statement"""
+
+ def __init__(self, dirpath=".", maxrecursive=10, raiseexceptions=False):
+ if maxrecursive <= 0:
+ raise ValueError('Max recursion limit reached')
+
+ self.dirpath = abspath(dirpath)
+ self.maxRecursive = maxrecursive
+ self.raiseexceptions = raiseexceptions
+
+ self.detected = False
+
+ @memoize_generator
+ def process(self, stack, stream):
+ # Run over all tokens in the stream
+ for token_type, value in stream:
+ # INCLUDE statement found, set detected mode
+ if token_type in Name and value.upper() == 'INCLUDE':
+ self.detected = True
+ continue
+
+ # INCLUDE statement was found, parse it
+ elif self.detected:
+ # Omit whitespaces
+ if token_type in Whitespace:
+ continue
+
+ # Found file path to include
+ if token_type in String.Symbol:
+# if token_type in tokens.String.Symbol:
+
+ # Get path of file to include
+ path = join(self.dirpath, value[1:-1])
+
+ try:
+ f = open(path)
+ raw_sql = f.read()
+ f.close()
+
+ # There was a problem loading the include file
+ except IOError, err:
+ # Raise the exception to the interpreter
+ if self.raiseexceptions:
+ raise
+
+ # Put the exception as a comment on the SQL code
+ yield Comment, u'-- IOError: %s\n' % err
+
+ else:
+ # Create new FilterStack to parse readed file
+ # and add all its tokens to the main stack recursively
+ try:
+ filtr = IncludeStatement(self.dirpath,
+ self.maxRecursive - 1,
+ self.raiseexceptions)
+
+ # Max recursion limit reached
+ except ValueError, err:
+ # Raise the exception to the interpreter
+ if self.raiseexceptions:
+ raise
+
+ # Put the exception as a comment on the SQL code
+ yield Comment, u'-- ValueError: %s\n' % err
+
+ stack = FilterStack()
+ stack.preprocess.append(filtr)
+
+ for tv in stack.run(raw_sql):
+ yield tv
+
+ # Set normal mode
+ self.detected = False
+
+ # Don't include any token while in detected mode
+ continue
+
+ # Normal token
+ yield token_type, value
+
+
+# ----------------------
+# statement process
+
+class StripCommentsFilter:
+
+ def _get_next_comment(self, tlist):
+ # TODO(andi) Comment types should be unified, see related issue38
+ token = tlist.token_next_by_instance(0, sql.Comment)
+ if token is None:
+ token = tlist.token_next_by_type(0, T.Comment)
+ return token
+
+ def _process(self, tlist):
+ token = self._get_next_comment(tlist)
+ while token:
+ tidx = tlist.token_index(token)
+ prev = tlist.token_prev(tidx, False)
+ next_ = tlist.token_next(tidx, False)
+ # Replace by whitespace if prev and next exist and if they're not
+ # whitespaces. This doesn't apply if prev or next is a paranthesis.
+ if (prev is not None and next_ is not None
+ and not prev.is_whitespace() and not next_.is_whitespace()
+ and not (prev.match(T.Punctuation, '(')
+ or next_.match(T.Punctuation, ')'))):
+ tlist.tokens[tidx] = sql.Token(T.Whitespace, ' ')
+ else:
+ tlist.tokens.pop(tidx)
+ token = self._get_next_comment(tlist)
+
+ def process(self, stack, stmt):
+ [self.process(stack, sgroup) for sgroup in stmt.get_sublists()]
+ self._process(stmt)
+
+
+class StripWhitespaceFilter:
+
+ def _stripws(self, tlist):
+ func_name = '_stripws_%s' % tlist.__class__.__name__.lower()
+ func = getattr(self, func_name, self._stripws_default)
+ func(tlist)
+
+ def _stripws_default(self, tlist):
+ last_was_ws = False
+ for token in tlist.tokens:
+ if token.is_whitespace():
+ if last_was_ws:
+ token.value = ''
+ else:
+ token.value = ' '
+ last_was_ws = token.is_whitespace()
+
+ def _stripws_identifierlist(self, tlist):
+ # Removes newlines before commas, see issue140
+ last_nl = None
+ for token in tlist.tokens[:]:
+ if (token.ttype is T.Punctuation
+ and token.value == ','
+ and last_nl is not None):
+ tlist.tokens.remove(last_nl)
+ if token.is_whitespace():
+ last_nl = token
+ else:
+ last_nl = None
+ return self._stripws_default(tlist)
+
+ def _stripws_parenthesis(self, tlist):
+ if tlist.tokens[1].is_whitespace():
+ tlist.tokens.pop(1)
+ if tlist.tokens[-2].is_whitespace():
+ tlist.tokens.pop(-2)
+ self._stripws_default(tlist)
+
+ def process(self, stack, stmt, depth=0):
+ [self.process(stack, sgroup, depth + 1)
+ for sgroup in stmt.get_sublists()]
+ self._stripws(stmt)
+ if (
+ depth == 0
+ and stmt.tokens
+ and stmt.tokens[-1].is_whitespace()
+ ):
+ stmt.tokens.pop(-1)
+
+
+class ReindentFilter:
+
+ def __init__(self, width=2, char=' ', line_width=None):
+ self.width = width
+ self.char = char
+ self.indent = 0
+ self.offset = 0
+ self.line_width = line_width
+ self._curr_stmt = None
+ self._last_stmt = None
+
+ def _flatten_up_to_token(self, token):
+ """Yields all tokens up to token plus the next one."""
+ # helper for _get_offset
+ iterator = self._curr_stmt.flatten()
+ for t in iterator:
+ yield t
+ if t == token:
+ raise StopIteration
+
+ def _get_offset(self, token):
+ raw = ''.join(map(unicode, self._flatten_up_to_token(token)))
+ line = raw.splitlines()[-1]
+ # Now take current offset into account and return relative offset.
+ full_offset = len(line) - len(self.char * (self.width * self.indent))
+ return full_offset - self.offset
+
+ def nl(self):
+ # TODO: newline character should be configurable
+ space = (self.char * ((self.indent * self.width) + self.offset))
+ # Detect runaway indenting due to parsing errors
+ if len(space) > 200:
+ # something seems to be wrong, flip back
+ self.indent = self.offset = 0
+ space = (self.char * ((self.indent * self.width) + self.offset))
+ ws = '\n' + space
+ return sql.Token(T.Whitespace, ws)
+
+ def _split_kwds(self, tlist):
+ split_words = ('FROM', 'STRAIGHT_JOIN$', 'JOIN$', 'AND', 'OR',
+ 'GROUP', 'ORDER', 'UNION', 'VALUES',
+ 'SET', 'BETWEEN', 'EXCEPT', 'HAVING')
+
+ def _next_token(i):
+ t = tlist.token_next_match(i, T.Keyword, split_words,
+ regex=True)
+ if t and t.value.upper() == 'BETWEEN':
+ t = _next_token(tlist.token_index(t) + 1)
+ if t and t.value.upper() == 'AND':
+ t = _next_token(tlist.token_index(t) + 1)
+ return t
+
+ idx = 0
+ token = _next_token(idx)
+ added = set()
+ while token:
+ prev = tlist.token_prev(tlist.token_index(token), False)
+ offset = 1
+ if prev and prev.is_whitespace() and prev not in added:
+ tlist.tokens.pop(tlist.token_index(prev))
+ offset += 1
+ uprev = unicode(prev)
+ if (prev and (uprev.endswith('\n') or uprev.endswith('\r'))):
+ nl = tlist.token_next(token)
+ else:
+ nl = self.nl()
+ added.add(nl)
+ tlist.insert_before(token, nl)
+ offset += 1
+ token = _next_token(tlist.token_index(nl) + offset)
+
+ def _split_statements(self, tlist):
+ idx = 0
+ token = tlist.token_next_by_type(idx, (T.Keyword.DDL, T.Keyword.DML))
+ while token:
+ prev = tlist.token_prev(tlist.token_index(token), False)
+ if prev and prev.is_whitespace():
+ tlist.tokens.pop(tlist.token_index(prev))
+ # only break if it's not the first token
+ if prev:
+ nl = self.nl()
+ tlist.insert_before(token, nl)
+ token = tlist.token_next_by_type(tlist.token_index(token) + 1,
+ (T.Keyword.DDL, T.Keyword.DML))
+
+ def _process(self, tlist):
+ func_name = '_process_%s' % tlist.__class__.__name__.lower()
+ func = getattr(self, func_name, self._process_default)
+ func(tlist)
+
+ def _process_where(self, tlist):
+ token = tlist.token_next_match(0, T.Keyword, 'WHERE')
+ try:
+ tlist.insert_before(token, self.nl())
+ except ValueError: # issue121, errors in statement
+ pass
+ self.indent += 1
+ self._process_default(tlist)
+ self.indent -= 1
+
+ def _process_having(self, tlist):
+ token = tlist.token_next_match(0, T.Keyword, 'HAVING')
+ try:
+ tlist.insert_before(token, self.nl())
+ except ValueError: # issue121, errors in statement
+ pass
+ self.indent += 1
+ self._process_default(tlist)
+ self.indent -= 1
+
+ def _process_parenthesis(self, tlist):
+ first = tlist.token_next(0)
+ indented = False
+ if first and first.ttype in (T.Keyword.DML, T.Keyword.DDL):
+ self.indent += 1
+ tlist.tokens.insert(0, self.nl())
+ indented = True
+ num_offset = self._get_offset(
+ tlist.token_next_match(0, T.Punctuation, '('))
+ self.offset += num_offset
+ self._process_default(tlist, stmts=not indented)
+ if indented:
+ self.indent -= 1
+ self.offset -= num_offset
+
+ def _process_identifierlist(self, tlist):
+ identifiers = list(tlist.get_identifiers())
+ if len(identifiers) > 1 and not tlist.within(sql.Function):
+ first = list(identifiers[0].flatten())[0]
+ if self.char == '\t':
+ # when using tabs we don't count the actual word length
+ # in spaces.
+ num_offset = 1
+ else:
+ num_offset = self._get_offset(first) - len(first.value)
+ self.offset += num_offset
+ for token in identifiers[1:]:
+ tlist.insert_before(token, self.nl())
+ self.offset -= num_offset
+ self._process_default(tlist)
+
+ def _process_case(self, tlist):
+ is_first = True
+ num_offset = None
+ case = tlist.tokens[0]
+ outer_offset = self._get_offset(case) - len(case.value)
+ self.offset += outer_offset
+ for cond, value in tlist.get_cases():
+ if is_first:
+ tcond = list(cond[0].flatten())[0]
+ is_first = False
+ num_offset = self._get_offset(tcond) - len(tcond.value)
+ self.offset += num_offset
+ continue
+ if cond is None:
+ token = value[0]
+ else:
+ token = cond[0]
+ tlist.insert_before(token, self.nl())
+ # Line breaks on group level are done. Now let's add an offset of
+ # 5 (=length of "when", "then", "else") and process subgroups.
+ self.offset += 5
+ self._process_default(tlist)
+ self.offset -= 5
+ if num_offset is not None:
+ self.offset -= num_offset
+ end = tlist.token_next_match(0, T.Keyword, 'END')
+ tlist.insert_before(end, self.nl())
+ self.offset -= outer_offset
+
+ def _process_default(self, tlist, stmts=True, kwds=True):
+ if stmts:
+ self._split_statements(tlist)
+ if kwds:
+ self._split_kwds(tlist)
+ [self._process(sgroup) for sgroup in tlist.get_sublists()]
+
+ def process(self, stack, stmt):
+ if isinstance(stmt, sql.Statement):
+ self._curr_stmt = stmt
+ self._process(stmt)
+ if isinstance(stmt, sql.Statement):
+ if self._last_stmt is not None:
+ if unicode(self._last_stmt).endswith('\n'):
+ nl = '\n'
+ else:
+ nl = '\n\n'
+ stmt.tokens.insert(
+ 0, sql.Token(T.Whitespace, nl))
+ if self._last_stmt != stmt:
+ self._last_stmt = stmt
+
+
+# FIXME: Doesn't work ;)
+class RightMarginFilter:
+
+ keep_together = (
+ # sql.TypeCast, sql.Identifier, sql.Alias,
+ )
+
+ def __init__(self, width=79):
+ self.width = width
+ self.line = ''
+
+ def _process(self, stack, group, stream):
+ for token in stream:
+ if token.is_whitespace() and '\n' in token.value:
+ if token.value.endswith('\n'):
+ self.line = ''
+ else:
+ self.line = token.value.splitlines()[-1]
+ elif (token.is_group()
+ and not token.__class__ in self.keep_together):
+ token.tokens = self._process(stack, token, token.tokens)
+ else:
+ val = unicode(token)
+ if len(self.line) + len(val) > self.width:
+ match = re.search('^ +', self.line)
+ if match is not None:
+ indent = match.group()
+ else:
+ indent = ''
+ yield sql.Token(T.Whitespace, '\n%s' % indent)
+ self.line = indent
+ self.line += val
+ yield token
+
+ def process(self, stack, group):
+ return
+ group.tokens = self._process(stack, group, group.tokens)
+
+
+class ColumnsSelect:
+ """Get the columns names of a SELECT query"""
+ def process(self, stack, stream):
+ mode = 0
+ oldValue = ""
+ parenthesis = 0
+
+ for token_type, value in stream:
+ # Ignore comments
+ if token_type in Comment:
+ continue
+
+ # We have not detected a SELECT statement
+ if mode == 0:
+ if token_type in Keyword and value == 'SELECT':
+ mode = 1
+
+ # We have detected a SELECT statement
+ elif mode == 1:
+ if value == 'FROM':
+ if oldValue:
+ yield oldValue
+
+ mode = 3 # Columns have been checked
+
+ elif value == 'AS':
+ oldValue = ""
+ mode = 2
+
+ elif (token_type == Punctuation
+ and value == ',' and not parenthesis):
+ if oldValue:
+ yield oldValue
+ oldValue = ""
+
+ elif token_type not in Whitespace:
+ if value == '(':
+ parenthesis += 1
+ elif value == ')':
+ parenthesis -= 1
+
+ oldValue += value
+
+ # We are processing an AS keyword
+ elif mode == 2:
+ # We check also for Keywords because a bug in SQLParse
+ if token_type == Name or token_type == Keyword:
+ yield value
+ mode = 1
+
+
+# ---------------------------
+# postprocess
+
+class SerializerUnicode:
+
+ def process(self, stack, stmt):
+ raw = unicode(stmt)
+ lines = split_unquoted_newlines(raw)
+ res = '\n'.join(line.rstrip() for line in lines)
+ return res
+
+
+def Tokens2Unicode(stream):
+ result = ""
+
+ for _, value in stream:
+ result += unicode(value)
+
+ return result
+
+
+class OutputFilter:
+ varname_prefix = ''
+
+ def __init__(self, varname='sql'):
+ self.varname = self.varname_prefix + varname
+ self.count = 0
+
+ def _process(self, stream, varname, has_nl):
+ raise NotImplementedError
+
+ def process(self, stack, stmt):
+ self.count += 1
+ if self.count > 1:
+ varname = '%s%d' % (self.varname, self.count)
+ else:
+ varname = self.varname
+
+ has_nl = len(unicode(stmt).strip().splitlines()) > 1
+ stmt.tokens = self._process(stmt.tokens, varname, has_nl)
+ return stmt
+
+
+class OutputPythonFilter(OutputFilter):
+ def _process(self, stream, varname, has_nl):
+ # SQL query asignation to varname
+ if self.count > 1:
+ yield sql.Token(T.Whitespace, '\n')
+ yield sql.Token(T.Name, varname)
+ yield sql.Token(T.Whitespace, ' ')
+ yield sql.Token(T.Operator, '=')
+ yield sql.Token(T.Whitespace, ' ')
+ if has_nl:
+ yield sql.Token(T.Operator, '(')
+ yield sql.Token(T.Text, "'")
+
+ # Print the tokens on the quote
+ for token in stream:
+ # Token is a new line separator
+ if token.is_whitespace() and '\n' in token.value:
+ # Close quote and add a new line
+ yield sql.Token(T.Text, " '")
+ yield sql.Token(T.Whitespace, '\n')
+
+ # Quote header on secondary lines
+ yield sql.Token(T.Whitespace, ' ' * (len(varname) + 4))
+ yield sql.Token(T.Text, "'")
+
+ # Indentation
+ after_lb = token.value.split('\n', 1)[1]
+ if after_lb:
+ yield sql.Token(T.Whitespace, after_lb)
+ continue
+
+ # Token has escape chars
+ elif "'" in token.value:
+ token.value = token.value.replace("'", "\\'")
+
+ # Put the token
+ yield sql.Token(T.Text, token.value)
+
+ # Close quote
+ yield sql.Token(T.Text, "'")
+ if has_nl:
+ yield sql.Token(T.Operator, ')')
+
+
+class OutputPHPFilter(OutputFilter):
+ varname_prefix = '$'
+
+ def _process(self, stream, varname, has_nl):
+ # SQL query asignation to varname (quote header)
+ if self.count > 1:
+ yield sql.Token(T.Whitespace, '\n')
+ yield sql.Token(T.Name, varname)
+ yield sql.Token(T.Whitespace, ' ')
+ if has_nl:
+ yield sql.Token(T.Whitespace, ' ')
+ yield sql.Token(T.Operator, '=')
+ yield sql.Token(T.Whitespace, ' ')
+ yield sql.Token(T.Text, '"')
+
+ # Print the tokens on the quote
+ for token in stream:
+ # Token is a new line separator
+ if token.is_whitespace() and '\n' in token.value:
+ # Close quote and add a new line
+ yield sql.Token(T.Text, ' ";')
+ yield sql.Token(T.Whitespace, '\n')
+
+ # Quote header on secondary lines
+ yield sql.Token(T.Name, varname)
+ yield sql.Token(T.Whitespace, ' ')
+ yield sql.Token(T.Operator, '.=')
+ yield sql.Token(T.Whitespace, ' ')
+ yield sql.Token(T.Text, '"')
+
+ # Indentation
+ after_lb = token.value.split('\n', 1)[1]
+ if after_lb:
+ yield sql.Token(T.Whitespace, after_lb)
+ continue
+
+ # Token has escape chars
+ elif '"' in token.value:
+ token.value = token.value.replace('"', '\\"')
+
+ # Put the token
+ yield sql.Token(T.Text, token.value)
+
+ # Close quote
+ yield sql.Token(T.Text, '"')
+ yield sql.Token(T.Punctuation, ';')
+
+
+class Limit:
+ """Get the LIMIT of a query.
+
+ If not defined, return -1 (SQL specification for no LIMIT query)
+ """
+ def process(self, stack, stream):
+ index = 7
+ stream = list(stream)
+ stream.reverse()
+
+ # Run over all tokens in the stream from the end
+ for token_type, value in stream:
+ index -= 1
+
+# if index and token_type in Keyword:
+ if index and token_type in Keyword and value == 'LIMIT':
+ return stream[4 - index][1]
+
+ return -1
+
+
+def compact(stream):
+ """Function that return a compacted version of the stream"""
+ pipe = Pipeline()
+
+ pipe.append(StripComments())
+ pipe.append(StripWhitespace)
+
+ return pipe(stream)
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/sqlparse/formatter.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/sqlparse/formatter.py b/shell/ext-py/sqlparse-0.1.19/sqlparse/formatter.py
new file mode 100644
index 0000000..811f5af
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/sqlparse/formatter.py
@@ -0,0 +1,137 @@
+# Copyright (C) 2008 Andi Albrecht, albrecht.andi@gmail.com
+#
+# This module is part of python-sqlparse and is released under
+# the BSD License: http://www.opensource.org/licenses/bsd-license.php.
+
+"""SQL formatter"""
+
+from sqlparse import filters
+from sqlparse.exceptions import SQLParseError
+
+
+def validate_options(options):
+ """Validates options."""
+ kwcase = options.get('keyword_case', None)
+ if kwcase not in [None, 'upper', 'lower', 'capitalize']:
+ raise SQLParseError('Invalid value for keyword_case: %r' % kwcase)
+
+ idcase = options.get('identifier_case', None)
+ if idcase not in [None, 'upper', 'lower', 'capitalize']:
+ raise SQLParseError('Invalid value for identifier_case: %r' % idcase)
+
+ ofrmt = options.get('output_format', None)
+ if ofrmt not in [None, 'sql', 'python', 'php']:
+ raise SQLParseError('Unknown output format: %r' % ofrmt)
+
+ strip_comments = options.get('strip_comments', False)
+ if strip_comments not in [True, False]:
+ raise SQLParseError('Invalid value for strip_comments: %r'
+ % strip_comments)
+
+ strip_ws = options.get('strip_whitespace', False)
+ if strip_ws not in [True, False]:
+ raise SQLParseError('Invalid value for strip_whitespace: %r'
+ % strip_ws)
+
+ truncate_strings = options.get('truncate_strings', None)
+ if truncate_strings is not None:
+ try:
+ truncate_strings = int(truncate_strings)
+ except (ValueError, TypeError):
+ raise SQLParseError('Invalid value for truncate_strings: %r'
+ % truncate_strings)
+ if truncate_strings <= 1:
+ raise SQLParseError('Invalid value for truncate_strings: %r'
+ % truncate_strings)
+ options['truncate_strings'] = truncate_strings
+ options['truncate_char'] = options.get('truncate_char', '[...]')
+
+ reindent = options.get('reindent', False)
+ if reindent not in [True, False]:
+ raise SQLParseError('Invalid value for reindent: %r'
+ % reindent)
+ elif reindent:
+ options['strip_whitespace'] = True
+ indent_tabs = options.get('indent_tabs', False)
+ if indent_tabs not in [True, False]:
+ raise SQLParseError('Invalid value for indent_tabs: %r' % indent_tabs)
+ elif indent_tabs:
+ options['indent_char'] = '\t'
+ else:
+ options['indent_char'] = ' '
+ indent_width = options.get('indent_width', 2)
+ try:
+ indent_width = int(indent_width)
+ except (TypeError, ValueError):
+ raise SQLParseError('indent_width requires an integer')
+ if indent_width < 1:
+ raise SQLParseError('indent_width requires an positive integer')
+ options['indent_width'] = indent_width
+
+ right_margin = options.get('right_margin', None)
+ if right_margin is not None:
+ try:
+ right_margin = int(right_margin)
+ except (TypeError, ValueError):
+ raise SQLParseError('right_margin requires an integer')
+ if right_margin < 10:
+ raise SQLParseError('right_margin requires an integer > 10')
+ options['right_margin'] = right_margin
+
+ return options
+
+
+def build_filter_stack(stack, options):
+ """Setup and return a filter stack.
+
+ Args:
+ stack: :class:`~sqlparse.filters.FilterStack` instance
+ options: Dictionary with options validated by validate_options.
+ """
+ # Token filter
+ if options.get('keyword_case', None):
+ stack.preprocess.append(
+ filters.KeywordCaseFilter(options['keyword_case']))
+
+ if options.get('identifier_case', None):
+ stack.preprocess.append(
+ filters.IdentifierCaseFilter(options['identifier_case']))
+
+ if options.get('truncate_strings', None) is not None:
+ stack.preprocess.append(filters.TruncateStringFilter(
+ width=options['truncate_strings'], char=options['truncate_char']))
+
+ # After grouping
+ if options.get('strip_comments', False):
+ stack.enable_grouping()
+ stack.stmtprocess.append(filters.StripCommentsFilter())
+
+ if (options.get('strip_whitespace', False)
+ or options.get('reindent', False)):
+ stack.enable_grouping()
+ stack.stmtprocess.append(filters.StripWhitespaceFilter())
+
+ if options.get('reindent', False):
+ stack.enable_grouping()
+ stack.stmtprocess.append(
+ filters.ReindentFilter(char=options['indent_char'],
+ width=options['indent_width']))
+
+ if options.get('right_margin', False):
+ stack.enable_grouping()
+ stack.stmtprocess.append(
+ filters.RightMarginFilter(width=options['right_margin']))
+
+ # Serializer
+ if options.get('output_format'):
+ frmt = options['output_format']
+ if frmt.lower() == 'php':
+ fltr = filters.OutputPHPFilter()
+ elif frmt.lower() == 'python':
+ fltr = filters.OutputPythonFilter()
+ else:
+ fltr = None
+ if fltr is not None:
+ stack.postprocess.append(fltr)
+
+ return stack
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/sqlparse/functions.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/sqlparse/functions.py b/shell/ext-py/sqlparse-0.1.19/sqlparse/functions.py
new file mode 100644
index 0000000..e54457e
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/sqlparse/functions.py
@@ -0,0 +1,44 @@
+'''
+Created on 17/05/2012
+
+@author: piranna
+
+Several utility functions to extract info from the SQL sentences
+'''
+
+from sqlparse.filters import ColumnsSelect, Limit
+from sqlparse.pipeline import Pipeline
+from sqlparse.tokens import Keyword, Whitespace
+
+
+def getlimit(stream):
+ """Function that return the LIMIT of a input SQL """
+ pipe = Pipeline()
+
+ pipe.append(Limit())
+
+ result = pipe(stream)
+ try:
+ return int(result)
+ except ValueError:
+ return result
+
+
+def getcolumns(stream):
+ """Function that return the colums of a SELECT query"""
+ pipe = Pipeline()
+
+ pipe.append(ColumnsSelect())
+
+ return pipe(stream)
+
+
+class IsType(object):
+ """Functor that return is the statement is of a specific type"""
+ def __init__(self, type):
+ self.type = type
+
+ def __call__(self, stream):
+ for token_type, value in stream:
+ if token_type not in Whitespace:
+ return token_type in Keyword and value == self.type
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/sqlparse/keywords.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/sqlparse/keywords.py b/shell/ext-py/sqlparse-0.1.19/sqlparse/keywords.py
new file mode 100644
index 0000000..1595aa8
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/sqlparse/keywords.py
@@ -0,0 +1,574 @@
+from sqlparse import tokens
+
+KEYWORDS = {
+ 'ABORT': tokens.Keyword,
+ 'ABS': tokens.Keyword,
+ 'ABSOLUTE': tokens.Keyword,
+ 'ACCESS': tokens.Keyword,
+ 'ADA': tokens.Keyword,
+ 'ADD': tokens.Keyword,
+ 'ADMIN': tokens.Keyword,
+ 'AFTER': tokens.Keyword,
+ 'AGGREGATE': tokens.Keyword,
+ 'ALIAS': tokens.Keyword,
+ 'ALL': tokens.Keyword,
+ 'ALLOCATE': tokens.Keyword,
+ 'ANALYSE': tokens.Keyword,
+ 'ANALYZE': tokens.Keyword,
+ 'ANY': tokens.Keyword,
+ 'ARE': tokens.Keyword,
+ 'ASC': tokens.Keyword.Order,
+ 'ASENSITIVE': tokens.Keyword,
+ 'ASSERTION': tokens.Keyword,
+ 'ASSIGNMENT': tokens.Keyword,
+ 'ASYMMETRIC': tokens.Keyword,
+ 'AT': tokens.Keyword,
+ 'ATOMIC': tokens.Keyword,
+ 'AUTHORIZATION': tokens.Keyword,
+ 'AVG': tokens.Keyword,
+
+ 'BACKWARD': tokens.Keyword,
+ 'BEFORE': tokens.Keyword,
+ 'BEGIN': tokens.Keyword,
+ 'BETWEEN': tokens.Keyword,
+ 'BITVAR': tokens.Keyword,
+ 'BIT_LENGTH': tokens.Keyword,
+ 'BOTH': tokens.Keyword,
+ 'BREADTH': tokens.Keyword,
+
+ # 'C': tokens.Keyword, # most likely this is an alias
+ 'CACHE': tokens.Keyword,
+ 'CALL': tokens.Keyword,
+ 'CALLED': tokens.Keyword,
+ 'CARDINALITY': tokens.Keyword,
+ 'CASCADE': tokens.Keyword,
+ 'CASCADED': tokens.Keyword,
+ 'CAST': tokens.Keyword,
+ 'CATALOG': tokens.Keyword,
+ 'CATALOG_NAME': tokens.Keyword,
+ 'CHAIN': tokens.Keyword,
+ 'CHARACTERISTICS': tokens.Keyword,
+ 'CHARACTER_LENGTH': tokens.Keyword,
+ 'CHARACTER_SET_CATALOG': tokens.Keyword,
+ 'CHARACTER_SET_NAME': tokens.Keyword,
+ 'CHARACTER_SET_SCHEMA': tokens.Keyword,
+ 'CHAR_LENGTH': tokens.Keyword,
+ 'CHECK': tokens.Keyword,
+ 'CHECKED': tokens.Keyword,
+ 'CHECKPOINT': tokens.Keyword,
+ 'CLASS': tokens.Keyword,
+ 'CLASS_ORIGIN': tokens.Keyword,
+ 'CLOB': tokens.Keyword,
+ 'CLOSE': tokens.Keyword,
+ 'CLUSTER': tokens.Keyword,
+ 'COALESCE': tokens.Keyword,
+ 'COBOL': tokens.Keyword,
+ 'COLLATE': tokens.Keyword,
+ 'COLLATION': tokens.Keyword,
+ 'COLLATION_CATALOG': tokens.Keyword,
+ 'COLLATION_NAME': tokens.Keyword,
+ 'COLLATION_SCHEMA': tokens.Keyword,
+ 'COLLECT': tokens.Keyword,
+ 'COLUMN': tokens.Keyword,
+ 'COLUMN_NAME': tokens.Keyword,
+ 'COMMAND_FUNCTION': tokens.Keyword,
+ 'COMMAND_FUNCTION_CODE': tokens.Keyword,
+ 'COMMENT': tokens.Keyword,
+ 'COMMIT': tokens.Keyword.DML,
+ 'COMMITTED': tokens.Keyword,
+ 'COMPLETION': tokens.Keyword,
+ 'CONDITION_NUMBER': tokens.Keyword,
+ 'CONNECT': tokens.Keyword,
+ 'CONNECTION': tokens.Keyword,
+ 'CONNECTION_NAME': tokens.Keyword,
+ 'CONSTRAINT': tokens.Keyword,
+ 'CONSTRAINTS': tokens.Keyword,
+ 'CONSTRAINT_CATALOG': tokens.Keyword,
+ 'CONSTRAINT_NAME': tokens.Keyword,
+ 'CONSTRAINT_SCHEMA': tokens.Keyword,
+ 'CONSTRUCTOR': tokens.Keyword,
+ 'CONTAINS': tokens.Keyword,
+ 'CONTINUE': tokens.Keyword,
+ 'CONVERSION': tokens.Keyword,
+ 'CONVERT': tokens.Keyword,
+ 'COPY': tokens.Keyword,
+ 'CORRESPONTING': tokens.Keyword,
+ 'COUNT': tokens.Keyword,
+ 'CREATEDB': tokens.Keyword,
+ 'CREATEUSER': tokens.Keyword,
+ 'CROSS': tokens.Keyword,
+ 'CUBE': tokens.Keyword,
+ 'CURRENT': tokens.Keyword,
+ 'CURRENT_DATE': tokens.Keyword,
+ 'CURRENT_PATH': tokens.Keyword,
+ 'CURRENT_ROLE': tokens.Keyword,
+ 'CURRENT_TIME': tokens.Keyword,
+ 'CURRENT_TIMESTAMP': tokens.Keyword,
+ 'CURRENT_USER': tokens.Keyword,
+ 'CURSOR': tokens.Keyword,
+ 'CURSOR_NAME': tokens.Keyword,
+ 'CYCLE': tokens.Keyword,
+
+ 'DATA': tokens.Keyword,
+ 'DATABASE': tokens.Keyword,
+ 'DATETIME_INTERVAL_CODE': tokens.Keyword,
+ 'DATETIME_INTERVAL_PRECISION': tokens.Keyword,
+ 'DAY': tokens.Keyword,
+ 'DEALLOCATE': tokens.Keyword,
+ 'DECLARE': tokens.Keyword,
+ 'DEFAULT': tokens.Keyword,
+ 'DEFAULTS': tokens.Keyword,
+ 'DEFERRABLE': tokens.Keyword,
+ 'DEFERRED': tokens.Keyword,
+ 'DEFINED': tokens.Keyword,
+ 'DEFINER': tokens.Keyword,
+ 'DELIMITER': tokens.Keyword,
+ 'DELIMITERS': tokens.Keyword,
+ 'DEREF': tokens.Keyword,
+ 'DESC': tokens.Keyword.Order,
+ 'DESCRIBE': tokens.Keyword,
+ 'DESCRIPTOR': tokens.Keyword,
+ 'DESTROY': tokens.Keyword,
+ 'DESTRUCTOR': tokens.Keyword,
+ 'DETERMINISTIC': tokens.Keyword,
+ 'DIAGNOSTICS': tokens.Keyword,
+ 'DICTIONARY': tokens.Keyword,
+ 'DISCONNECT': tokens.Keyword,
+ 'DISPATCH': tokens.Keyword,
+ 'DO': tokens.Keyword,
+ 'DOMAIN': tokens.Keyword,
+ 'DYNAMIC': tokens.Keyword,
+ 'DYNAMIC_FUNCTION': tokens.Keyword,
+ 'DYNAMIC_FUNCTION_CODE': tokens.Keyword,
+
+ 'EACH': tokens.Keyword,
+ 'ENCODING': tokens.Keyword,
+ 'ENCRYPTED': tokens.Keyword,
+ 'END-EXEC': tokens.Keyword,
+ 'EQUALS': tokens.Keyword,
+ 'ESCAPE': tokens.Keyword,
+ 'EVERY': tokens.Keyword,
+ 'EXCEPT': tokens.Keyword,
+ 'ESCEPTION': tokens.Keyword,
+ 'EXCLUDING': tokens.Keyword,
+ 'EXCLUSIVE': tokens.Keyword,
+ 'EXEC': tokens.Keyword,
+ 'EXECUTE': tokens.Keyword,
+ 'EXISTING': tokens.Keyword,
+ 'EXISTS': tokens.Keyword,
+ 'EXTERNAL': tokens.Keyword,
+ 'EXTRACT': tokens.Keyword,
+
+ 'FALSE': tokens.Keyword,
+ 'FETCH': tokens.Keyword,
+ 'FINAL': tokens.Keyword,
+ 'FIRST': tokens.Keyword,
+ 'FORCE': tokens.Keyword,
+ 'FOREACH': tokens.Keyword,
+ 'FOREIGN': tokens.Keyword,
+ 'FORTRAN': tokens.Keyword,
+ 'FORWARD': tokens.Keyword,
+ 'FOUND': tokens.Keyword,
+ 'FREE': tokens.Keyword,
+ 'FREEZE': tokens.Keyword,
+ 'FULL': tokens.Keyword,
+ 'FUNCTION': tokens.Keyword,
+
+ # 'G': tokens.Keyword,
+ 'GENERAL': tokens.Keyword,
+ 'GENERATED': tokens.Keyword,
+ 'GET': tokens.Keyword,
+ 'GLOBAL': tokens.Keyword,
+ 'GO': tokens.Keyword,
+ 'GOTO': tokens.Keyword,
+ 'GRANT': tokens.Keyword,
+ 'GRANTED': tokens.Keyword,
+ 'GROUPING': tokens.Keyword,
+
+ 'HANDLER': tokens.Keyword,
+ 'HAVING': tokens.Keyword,
+ 'HIERARCHY': tokens.Keyword,
+ 'HOLD': tokens.Keyword,
+ 'HOST': tokens.Keyword,
+
+ 'IDENTITY': tokens.Keyword,
+ 'IGNORE': tokens.Keyword,
+ 'ILIKE': tokens.Keyword,
+ 'IMMEDIATE': tokens.Keyword,
+ 'IMMUTABLE': tokens.Keyword,
+
+ 'IMPLEMENTATION': tokens.Keyword,
+ 'IMPLICIT': tokens.Keyword,
+ 'INCLUDING': tokens.Keyword,
+ 'INCREMENT': tokens.Keyword,
+ 'INDEX': tokens.Keyword,
+
+ 'INDITCATOR': tokens.Keyword,
+ 'INFIX': tokens.Keyword,
+ 'INHERITS': tokens.Keyword,
+ 'INITIALIZE': tokens.Keyword,
+ 'INITIALLY': tokens.Keyword,
+ 'INOUT': tokens.Keyword,
+ 'INPUT': tokens.Keyword,
+ 'INSENSITIVE': tokens.Keyword,
+ 'INSTANTIABLE': tokens.Keyword,
+ 'INSTEAD': tokens.Keyword,
+ 'INTERSECT': tokens.Keyword,
+ 'INTO': tokens.Keyword,
+ 'INVOKER': tokens.Keyword,
+ 'IS': tokens.Keyword,
+ 'ISNULL': tokens.Keyword,
+ 'ISOLATION': tokens.Keyword,
+ 'ITERATE': tokens.Keyword,
+
+ # 'K': tokens.Keyword,
+ 'KEY': tokens.Keyword,
+ 'KEY_MEMBER': tokens.Keyword,
+ 'KEY_TYPE': tokens.Keyword,
+
+ 'LANCOMPILER': tokens.Keyword,
+ 'LANGUAGE': tokens.Keyword,
+ 'LARGE': tokens.Keyword,
+ 'LAST': tokens.Keyword,
+ 'LATERAL': tokens.Keyword,
+ 'LEADING': tokens.Keyword,
+ 'LENGTH': tokens.Keyword,
+ 'LESS': tokens.Keyword,
+ 'LEVEL': tokens.Keyword,
+ 'LIMIT': tokens.Keyword,
+ 'LISTEN': tokens.Keyword,
+ 'LOAD': tokens.Keyword,
+ 'LOCAL': tokens.Keyword,
+ 'LOCALTIME': tokens.Keyword,
+ 'LOCALTIMESTAMP': tokens.Keyword,
+ 'LOCATION': tokens.Keyword,
+ 'LOCATOR': tokens.Keyword,
+ 'LOCK': tokens.Keyword,
+ 'LOWER': tokens.Keyword,
+
+ # 'M': tokens.Keyword,
+ 'MAP': tokens.Keyword,
+ 'MATCH': tokens.Keyword,
+ 'MAXVALUE': tokens.Keyword,
+ 'MESSAGE_LENGTH': tokens.Keyword,
+ 'MESSAGE_OCTET_LENGTH': tokens.Keyword,
+ 'MESSAGE_TEXT': tokens.Keyword,
+ 'METHOD': tokens.Keyword,
+ 'MINUTE': tokens.Keyword,
+ 'MINVALUE': tokens.Keyword,
+ 'MOD': tokens.Keyword,
+ 'MODE': tokens.Keyword,
+ 'MODIFIES': tokens.Keyword,
+ 'MODIFY': tokens.Keyword,
+ 'MONTH': tokens.Keyword,
+ 'MORE': tokens.Keyword,
+ 'MOVE': tokens.Keyword,
+ 'MUMPS': tokens.Keyword,
+
+ 'NAMES': tokens.Keyword,
+ 'NATIONAL': tokens.Keyword,
+ 'NATURAL': tokens.Keyword,
+ 'NCHAR': tokens.Keyword,
+ 'NCLOB': tokens.Keyword,
+ 'NEW': tokens.Keyword,
+ 'NEXT': tokens.Keyword,
+ 'NO': tokens.Keyword,
+ 'NOCREATEDB': tokens.Keyword,
+ 'NOCREATEUSER': tokens.Keyword,
+ 'NONE': tokens.Keyword,
+ 'NOT': tokens.Keyword,
+ 'NOTHING': tokens.Keyword,
+ 'NOTIFY': tokens.Keyword,
+ 'NOTNULL': tokens.Keyword,
+ 'NULL': tokens.Keyword,
+ 'NULLABLE': tokens.Keyword,
+ 'NULLIF': tokens.Keyword,
+
+ 'OBJECT': tokens.Keyword,
+ 'OCTET_LENGTH': tokens.Keyword,
+ 'OF': tokens.Keyword,
+ 'OFF': tokens.Keyword,
+ 'OFFSET': tokens.Keyword,
+ 'OIDS': tokens.Keyword,
+ 'OLD': tokens.Keyword,
+ 'ONLY': tokens.Keyword,
+ 'OPEN': tokens.Keyword,
+ 'OPERATION': tokens.Keyword,
+ 'OPERATOR': tokens.Keyword,
+ 'OPTION': tokens.Keyword,
+ 'OPTIONS': tokens.Keyword,
+ 'ORDINALITY': tokens.Keyword,
+ 'OUT': tokens.Keyword,
+ 'OUTPUT': tokens.Keyword,
+ 'OVERLAPS': tokens.Keyword,
+ 'OVERLAY': tokens.Keyword,
+ 'OVERRIDING': tokens.Keyword,
+ 'OWNER': tokens.Keyword,
+
+ 'PAD': tokens.Keyword,
+ 'PARAMETER': tokens.Keyword,
+ 'PARAMETERS': tokens.Keyword,
+ 'PARAMETER_MODE': tokens.Keyword,
+ 'PARAMATER_NAME': tokens.Keyword,
+ 'PARAMATER_ORDINAL_POSITION': tokens.Keyword,
+ 'PARAMETER_SPECIFIC_CATALOG': tokens.Keyword,
+ 'PARAMETER_SPECIFIC_NAME': tokens.Keyword,
+ 'PARAMATER_SPECIFIC_SCHEMA': tokens.Keyword,
+ 'PARTIAL': tokens.Keyword,
+ 'PASCAL': tokens.Keyword,
+ 'PENDANT': tokens.Keyword,
+ 'PLACING': tokens.Keyword,
+ 'PLI': tokens.Keyword,
+ 'POSITION': tokens.Keyword,
+ 'POSTFIX': tokens.Keyword,
+ 'PRECISION': tokens.Keyword,
+ 'PREFIX': tokens.Keyword,
+ 'PREORDER': tokens.Keyword,
+ 'PREPARE': tokens.Keyword,
+ 'PRESERVE': tokens.Keyword,
+ 'PRIMARY': tokens.Keyword,
+ 'PRIOR': tokens.Keyword,
+ 'PRIVILEGES': tokens.Keyword,
+ 'PROCEDURAL': tokens.Keyword,
+ 'PROCEDURE': tokens.Keyword,
+ 'PUBLIC': tokens.Keyword,
+
+ 'RAISE': tokens.Keyword,
+ 'READ': tokens.Keyword,
+ 'READS': tokens.Keyword,
+ 'RECHECK': tokens.Keyword,
+ 'RECURSIVE': tokens.Keyword,
+ 'REF': tokens.Keyword,
+ 'REFERENCES': tokens.Keyword,
+ 'REFERENCING': tokens.Keyword,
+ 'REINDEX': tokens.Keyword,
+ 'RELATIVE': tokens.Keyword,
+ 'RENAME': tokens.Keyword,
+ 'REPEATABLE': tokens.Keyword,
+ 'RESET': tokens.Keyword,
+ 'RESTART': tokens.Keyword,
+ 'RESTRICT': tokens.Keyword,
+ 'RESULT': tokens.Keyword,
+ 'RETURN': tokens.Keyword,
+ 'RETURNED_LENGTH': tokens.Keyword,
+ 'RETURNED_OCTET_LENGTH': tokens.Keyword,
+ 'RETURNED_SQLSTATE': tokens.Keyword,
+ 'RETURNS': tokens.Keyword,
+ 'REVOKE': tokens.Keyword,
+ 'RIGHT': tokens.Keyword,
+ 'ROLE': tokens.Keyword,
+ 'ROLLBACK': tokens.Keyword.DML,
+ 'ROLLUP': tokens.Keyword,
+ 'ROUTINE': tokens.Keyword,
+ 'ROUTINE_CATALOG': tokens.Keyword,
+ 'ROUTINE_NAME': tokens.Keyword,
+ 'ROUTINE_SCHEMA': tokens.Keyword,
+ 'ROW': tokens.Keyword,
+ 'ROWS': tokens.Keyword,
+ 'ROW_COUNT': tokens.Keyword,
+ 'RULE': tokens.Keyword,
+
+ 'SAVE_POINT': tokens.Keyword,
+ 'SCALE': tokens.Keyword,
+ 'SCHEMA': tokens.Keyword,
+ 'SCHEMA_NAME': tokens.Keyword,
+ 'SCOPE': tokens.Keyword,
+ 'SCROLL': tokens.Keyword,
+ 'SEARCH': tokens.Keyword,
+ 'SECOND': tokens.Keyword,
+ 'SECURITY': tokens.Keyword,
+ 'SELF': tokens.Keyword,
+ 'SENSITIVE': tokens.Keyword,
+ 'SERIALIZABLE': tokens.Keyword,
+ 'SERVER_NAME': tokens.Keyword,
+ 'SESSION': tokens.Keyword,
+ 'SESSION_USER': tokens.Keyword,
+ 'SETOF': tokens.Keyword,
+ 'SETS': tokens.Keyword,
+ 'SHARE': tokens.Keyword,
+ 'SHOW': tokens.Keyword,
+ 'SIMILAR': tokens.Keyword,
+ 'SIMPLE': tokens.Keyword,
+ 'SIZE': tokens.Keyword,
+ 'SOME': tokens.Keyword,
+ 'SOURCE': tokens.Keyword,
+ 'SPACE': tokens.Keyword,
+ 'SPECIFIC': tokens.Keyword,
+ 'SPECIFICTYPE': tokens.Keyword,
+ 'SPECIFIC_NAME': tokens.Keyword,
+ 'SQL': tokens.Keyword,
+ 'SQLCODE': tokens.Keyword,
+ 'SQLERROR': tokens.Keyword,
+ 'SQLEXCEPTION': tokens.Keyword,
+ 'SQLSTATE': tokens.Keyword,
+ 'SQLWARNING': tokens.Keyword,
+ 'STABLE': tokens.Keyword,
+ 'START': tokens.Keyword.DML,
+ 'STATE': tokens.Keyword,
+ 'STATEMENT': tokens.Keyword,
+ 'STATIC': tokens.Keyword,
+ 'STATISTICS': tokens.Keyword,
+ 'STDIN': tokens.Keyword,
+ 'STDOUT': tokens.Keyword,
+ 'STORAGE': tokens.Keyword,
+ 'STRICT': tokens.Keyword,
+ 'STRUCTURE': tokens.Keyword,
+ 'STYPE': tokens.Keyword,
+ 'SUBCLASS_ORIGIN': tokens.Keyword,
+ 'SUBLIST': tokens.Keyword,
+ 'SUBSTRING': tokens.Keyword,
+ 'SUM': tokens.Keyword,
+ 'SYMMETRIC': tokens.Keyword,
+ 'SYSID': tokens.Keyword,
+ 'SYSTEM': tokens.Keyword,
+ 'SYSTEM_USER': tokens.Keyword,
+
+ 'TABLE': tokens.Keyword,
+ 'TABLE_NAME': tokens.Keyword,
+ 'TEMP': tokens.Keyword,
+ 'TEMPLATE': tokens.Keyword,
+ 'TEMPORARY': tokens.Keyword,
+ 'TERMINATE': tokens.Keyword,
+ 'THAN': tokens.Keyword,
+ 'TIMESTAMP': tokens.Keyword,
+ 'TIMEZONE_HOUR': tokens.Keyword,
+ 'TIMEZONE_MINUTE': tokens.Keyword,
+ 'TO': tokens.Keyword,
+ 'TOAST': tokens.Keyword,
+ 'TRAILING': tokens.Keyword,
+ 'TRANSATION': tokens.Keyword,
+ 'TRANSACTIONS_COMMITTED': tokens.Keyword,
+ 'TRANSACTIONS_ROLLED_BACK': tokens.Keyword,
+ 'TRANSATION_ACTIVE': tokens.Keyword,
+ 'TRANSFORM': tokens.Keyword,
+ 'TRANSFORMS': tokens.Keyword,
+ 'TRANSLATE': tokens.Keyword,
+ 'TRANSLATION': tokens.Keyword,
+ 'TREAT': tokens.Keyword,
+ 'TRIGGER': tokens.Keyword,
+ 'TRIGGER_CATALOG': tokens.Keyword,
+ 'TRIGGER_NAME': tokens.Keyword,
+ 'TRIGGER_SCHEMA': tokens.Keyword,
+ 'TRIM': tokens.Keyword,
+ 'TRUE': tokens.Keyword,
+ 'TRUNCATE': tokens.Keyword,
+ 'TRUSTED': tokens.Keyword,
+ 'TYPE': tokens.Keyword,
+
+ 'UNCOMMITTED': tokens.Keyword,
+ 'UNDER': tokens.Keyword,
+ 'UNENCRYPTED': tokens.Keyword,
+ 'UNION': tokens.Keyword,
+ 'UNIQUE': tokens.Keyword,
+ 'UNKNOWN': tokens.Keyword,
+ 'UNLISTEN': tokens.Keyword,
+ 'UNNAMED': tokens.Keyword,
+ 'UNNEST': tokens.Keyword,
+ 'UNTIL': tokens.Keyword,
+ 'UPPER': tokens.Keyword,
+ 'USAGE': tokens.Keyword,
+ 'USE': tokens.Keyword,
+ 'USER': tokens.Keyword,
+ 'USER_DEFINED_TYPE_CATALOG': tokens.Keyword,
+ 'USER_DEFINED_TYPE_NAME': tokens.Keyword,
+ 'USER_DEFINED_TYPE_SCHEMA': tokens.Keyword,
+ 'USING': tokens.Keyword,
+
+ 'VACUUM': tokens.Keyword,
+ 'VALID': tokens.Keyword,
+ 'VALIDATOR': tokens.Keyword,
+ 'VALUES': tokens.Keyword,
+ 'VARIABLE': tokens.Keyword,
+ 'VERBOSE': tokens.Keyword,
+ 'VERSION': tokens.Keyword,
+ 'VIEW': tokens.Keyword,
+ 'VOLATILE': tokens.Keyword,
+
+ 'WHENEVER': tokens.Keyword,
+ 'WITH': tokens.Keyword,
+ 'WITHOUT': tokens.Keyword,
+ 'WORK': tokens.Keyword,
+ 'WRITE': tokens.Keyword,
+
+ 'YEAR': tokens.Keyword,
+
+ 'ZONE': tokens.Keyword,
+
+ # Name.Builtin
+ 'ARRAY': tokens.Name.Builtin,
+ 'BIGINT': tokens.Name.Builtin,
+ 'BINARY': tokens.Name.Builtin,
+ 'BIT': tokens.Name.Builtin,
+ 'BLOB': tokens.Name.Builtin,
+ 'BOOLEAN': tokens.Name.Builtin,
+ 'CHAR': tokens.Name.Builtin,
+ 'CHARACTER': tokens.Name.Builtin,
+ 'DATE': tokens.Name.Builtin,
+ 'DEC': tokens.Name.Builtin,
+ 'DECIMAL': tokens.Name.Builtin,
+ 'FLOAT': tokens.Name.Builtin,
+ 'INT': tokens.Name.Builtin,
+ 'INT8': tokens.Name.Builtin,
+ 'INTEGER': tokens.Name.Builtin,
+ 'INTERVAL': tokens.Name.Builtin,
+ 'LONG': tokens.Name.Builtin,
+ 'NUMBER': tokens.Name.Builtin,
+ 'NUMERIC': tokens.Name.Builtin,
+ 'REAL': tokens.Name.Builtin,
+ 'SERIAL': tokens.Name.Builtin,
+ 'SERIAL8': tokens.Name.Builtin,
+ 'SIGNED': tokens.Name.Builtin,
+ 'SMALLINT': tokens.Name.Builtin,
+ 'TEXT': tokens.Name.Builtin,
+ 'TINYINT': tokens.Name.Builtin,
+ 'UNSIGNED': tokens.Name.Builtin,
+ 'VARCHAR': tokens.Name.Builtin,
+ 'VARCHAR2': tokens.Name.Builtin,
+ 'VARYING': tokens.Name.Builtin,
+}
+
+
+KEYWORDS_COMMON = {
+ 'SELECT': tokens.Keyword.DML,
+ 'INSERT': tokens.Keyword.DML,
+ 'DELETE': tokens.Keyword.DML,
+ 'UPDATE': tokens.Keyword.DML,
+ 'REPLACE': tokens.Keyword.DML,
+ 'MERGE': tokens.Keyword.DML,
+ 'DROP': tokens.Keyword.DDL,
+ 'CREATE': tokens.Keyword.DDL,
+ 'ALTER': tokens.Keyword.DDL,
+
+ 'WHERE': tokens.Keyword,
+ 'FROM': tokens.Keyword,
+ 'INNER': tokens.Keyword,
+ 'JOIN': tokens.Keyword,
+ 'STRAIGHT_JOIN': tokens.Keyword,
+ 'AND': tokens.Keyword,
+ 'OR': tokens.Keyword,
+ 'LIKE': tokens.Keyword,
+ 'ON': tokens.Keyword,
+ 'IN': tokens.Keyword,
+ 'SET': tokens.Keyword,
+
+ 'BY': tokens.Keyword,
+ 'GROUP': tokens.Keyword,
+ 'ORDER': tokens.Keyword,
+ 'LEFT': tokens.Keyword,
+ 'OUTER': tokens.Keyword,
+ 'FULL': tokens.Keyword,
+
+ 'IF': tokens.Keyword,
+ 'END': tokens.Keyword,
+ 'THEN': tokens.Keyword,
+ 'LOOP': tokens.Keyword,
+ 'AS': tokens.Keyword,
+ 'ELSE': tokens.Keyword,
+ 'FOR': tokens.Keyword,
+
+ 'CASE': tokens.Keyword,
+ 'WHEN': tokens.Keyword,
+ 'MIN': tokens.Keyword,
+ 'MAX': tokens.Keyword,
+ 'DISTINCT': tokens.Keyword,
+}
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/sqlparse/lexer.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/sqlparse/lexer.py b/shell/ext-py/sqlparse-0.1.19/sqlparse/lexer.py
new file mode 100644
index 0000000..fd29f5c
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/sqlparse/lexer.py
@@ -0,0 +1,362 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2008 Andi Albrecht, albrecht.andi@gmail.com
+#
+# This module is part of python-sqlparse and is released under
+# the BSD License: http://www.opensource.org/licenses/bsd-license.php.
+
+"""SQL Lexer"""
+
+# This code is based on the SqlLexer in pygments.
+# http://pygments.org/
+# It's separated from the rest of pygments to increase performance
+# and to allow some customizations.
+
+import re
+import sys
+
+from sqlparse import tokens
+from sqlparse.keywords import KEYWORDS, KEYWORDS_COMMON
+from cStringIO import StringIO
+
+
+class include(str):
+ pass
+
+
+class combined(tuple):
+ """Indicates a state combined from multiple states."""
+
+ def __new__(cls, *args):
+ return tuple.__new__(cls, args)
+
+ def __init__(self, *args):
+ # tuple.__init__ doesn't do anything
+ pass
+
+
+def is_keyword(value):
+ test = value.upper()
+ return KEYWORDS_COMMON.get(test, KEYWORDS.get(test, tokens.Name)), value
+
+
+def apply_filters(stream, filters, lexer=None):
+ """
+ Use this method to apply an iterable of filters to
+ a stream. If lexer is given it's forwarded to the
+ filter, otherwise the filter receives `None`.
+ """
+
+ def _apply(filter_, stream):
+ for token in filter_.filter(lexer, stream):
+ yield token
+
+ for filter_ in filters:
+ stream = _apply(filter_, stream)
+ return stream
+
+
+class LexerMeta(type):
+ """
+ Metaclass for Lexer, creates the self._tokens attribute from
+ self.tokens on the first instantiation.
+ """
+
+ def _process_state(cls, unprocessed, processed, state):
+ assert type(state) is str, "wrong state name %r" % state
+ assert state[0] != '#', "invalid state name %r" % state
+ if state in processed:
+ return processed[state]
+ tokenlist = processed[state] = []
+ rflags = cls.flags
+ for tdef in unprocessed[state]:
+ if isinstance(tdef, include):
+ # it's a state reference
+ assert tdef != state, "circular state reference %r" % state
+ tokenlist.extend(cls._process_state(
+ unprocessed, processed, str(tdef)))
+ continue
+
+ assert type(tdef) is tuple, "wrong rule def %r" % tdef
+
+ try:
+ rex = re.compile(tdef[0], rflags).match
+ except Exception, err:
+ raise ValueError(("uncompilable regex %r in state"
+ " %r of %r: %s"
+ % (tdef[0], state, cls, err)))
+
+ assert type(tdef[1]) is tokens._TokenType or callable(tdef[1]), \
+ ('token type must be simple type or callable, not %r'
+ % (tdef[1],))
+
+ if len(tdef) == 2:
+ new_state = None
+ else:
+ tdef2 = tdef[2]
+ if isinstance(tdef2, str):
+ # an existing state
+ if tdef2 == '#pop':
+ new_state = -1
+ elif tdef2 in unprocessed:
+ new_state = (tdef2,)
+ elif tdef2 == '#push':
+ new_state = tdef2
+ elif tdef2[:5] == '#pop:':
+ new_state = -int(tdef2[5:])
+ else:
+ assert False, 'unknown new state %r' % tdef2
+ elif isinstance(tdef2, combined):
+ # combine a new state from existing ones
+ new_state = '_tmp_%d' % cls._tmpname
+ cls._tmpname += 1
+ itokens = []
+ for istate in tdef2:
+ assert istate != state, \
+ 'circular state ref %r' % istate
+ itokens.extend(cls._process_state(unprocessed,
+ processed, istate))
+ processed[new_state] = itokens
+ new_state = (new_state,)
+ elif isinstance(tdef2, tuple):
+ # push more than one state
+ for state in tdef2:
+ assert (state in unprocessed or
+ state in ('#pop', '#push')), \
+ 'unknown new state ' + state
+ new_state = tdef2
+ else:
+ assert False, 'unknown new state def %r' % tdef2
+ tokenlist.append((rex, tdef[1], new_state))
+ return tokenlist
+
+ def process_tokendef(cls):
+ cls._all_tokens = {}
+ cls._tmpname = 0
+ processed = cls._all_tokens[cls.__name__] = {}
+ #tokendefs = tokendefs or cls.tokens[name]
+ for state in cls.tokens.keys():
+ cls._process_state(cls.tokens, processed, state)
+ return processed
+
+ def __call__(cls, *args, **kwds):
+ if not hasattr(cls, '_tokens'):
+ cls._all_tokens = {}
+ cls._tmpname = 0
+ if hasattr(cls, 'token_variants') and cls.token_variants:
+ # don't process yet
+ pass
+ else:
+ cls._tokens = cls.process_tokendef()
+
+ return type.__call__(cls, *args, **kwds)
+
+
+class Lexer(object):
+
+ __metaclass__ = LexerMeta
+
+ encoding = 'utf-8'
+ stripall = False
+ stripnl = False
+ tabsize = 0
+ flags = re.IGNORECASE | re.UNICODE
+
+ tokens = {
+ 'root': [
+ (r'(--|# ).*?(\r\n|\r|\n)', tokens.Comment.Single),
+ # $ matches *before* newline, therefore we have two patterns
+ # to match Comment.Single
+ (r'(--|# ).*?$', tokens.Comment.Single),
+ (r'(\r\n|\r|\n)', tokens.Newline),
+ (r'\s+', tokens.Whitespace),
+ (r'/\*', tokens.Comment.Multiline, 'multiline-comments'),
+ (r':=', tokens.Assignment),
+ (r'::', tokens.Punctuation),
+ (r'[*]', tokens.Wildcard),
+ (r'CASE\b', tokens.Keyword), # extended CASE(foo)
+ (r"`(``|[^`])*`", tokens.Name),
+ (r"´(´´|[^´])*´", tokens.Name),
+ (r'\$([^\W\d]\w*)?\$', tokens.Name.Builtin),
+ (r'\?{1}', tokens.Name.Placeholder),
+ (r'%\(\w+\)s', tokens.Name.Placeholder),
+ (r'%s', tokens.Name.Placeholder),
+ (r'[$:?]\w+', tokens.Name.Placeholder),
+ # FIXME(andi): VALUES shouldn't be listed here
+ # see https://github.com/andialbrecht/sqlparse/pull/64
+ (r'VALUES', tokens.Keyword),
+ (r'(@|##|#)[^\W\d_]\w+', tokens.Name),
+ # IN is special, it may be followed by a parenthesis, but
+ # is never a functino, see issue183
+ (r'in\b(?=[ (])?', tokens.Keyword),
+ (r'[^\W\d_]\w*(?=[.(])', tokens.Name), # see issue39
+ (r'[-]?0x[0-9a-fA-F]+', tokens.Number.Hexadecimal),
+ (r'[-]?[0-9]*(\.[0-9]+)?[eE][-]?[0-9]+', tokens.Number.Float),
+ (r'[-]?[0-9]*\.[0-9]+', tokens.Number.Float),
+ (r'[-]?[0-9]+', tokens.Number.Integer),
+ (r"'(''|\\\\|\\'|[^'])*'", tokens.String.Single),
+ # not a real string literal in ANSI SQL:
+ (r'(""|".*?[^\\]")', tokens.String.Symbol),
+ # sqlite names can be escaped with [square brackets]. left bracket
+ # cannot be preceded by word character or a right bracket --
+ # otherwise it's probably an array index
+ (r'(?<![\w\])])(\[[^\]]+\])', tokens.Name),
+ (r'((LEFT\s+|RIGHT\s+|FULL\s+)?(INNER\s+|OUTER\s+|STRAIGHT\s+)?|(CROSS\s+|NATURAL\s+)?)?JOIN\b', tokens.Keyword),
+ (r'END(\s+IF|\s+LOOP)?\b', tokens.Keyword),
+ (r'NOT NULL\b', tokens.Keyword),
+ (r'CREATE(\s+OR\s+REPLACE)?\b', tokens.Keyword.DDL),
+ (r'DOUBLE\s+PRECISION\b', tokens.Name.Builtin),
+ (r'(?<=\.)[^\W\d_]\w*', tokens.Name),
+ (r'[^\W\d]\w*', is_keyword),
+ (r'[;:()\[\],\.]', tokens.Punctuation),
+ (r'[<>=~!]+', tokens.Operator.Comparison),
+ (r'[+/@#%^&|`?^-]+', tokens.Operator),
+ ],
+ 'multiline-comments': [
+ (r'/\*', tokens.Comment.Multiline, 'multiline-comments'),
+ (r'\*/', tokens.Comment.Multiline, '#pop'),
+ (r'[^/\*]+', tokens.Comment.Multiline),
+ (r'[/*]', tokens.Comment.Multiline),
+ ]}
+
+ def __init__(self):
+ self.filters = []
+
+ def add_filter(self, filter_, **options):
+ from sqlparse.filters import Filter
+ if not isinstance(filter_, Filter):
+ filter_ = filter_(**options)
+ self.filters.append(filter_)
+
+ def _decode(self, text):
+ if sys.version_info[0] == 3:
+ if isinstance(text, str):
+ return text
+ if self.encoding == 'guess':
+ try:
+ text = text.decode('utf-8')
+ if text.startswith(u'\ufeff'):
+ text = text[len(u'\ufeff'):]
+ except UnicodeDecodeError:
+ text = text.decode('latin1')
+ else:
+ try:
+ text = text.decode(self.encoding)
+ except UnicodeDecodeError:
+ text = text.decode('unicode-escape')
+
+ if self.tabsize > 0:
+ text = text.expandtabs(self.tabsize)
+ return text
+
+ def get_tokens(self, text, unfiltered=False):
+ """
+ Return an iterable of (tokentype, value) pairs generated from
+ `text`. If `unfiltered` is set to `True`, the filtering mechanism
+ is bypassed even if filters are defined.
+
+ Also preprocess the text, i.e. expand tabs and strip it if
+ wanted and applies registered filters.
+ """
+ if isinstance(text, basestring):
+ if self.stripall:
+ text = text.strip()
+ elif self.stripnl:
+ text = text.strip('\n')
+
+ if sys.version_info[0] < 3 and isinstance(text, unicode):
+ text = StringIO(text.encode('utf-8'))
+ self.encoding = 'utf-8'
+ else:
+ text = StringIO(text)
+
+ def streamer():
+ for i, t, v in self.get_tokens_unprocessed(text):
+ yield t, v
+ stream = streamer()
+ if not unfiltered:
+ stream = apply_filters(stream, self.filters, self)
+ return stream
+
+ def get_tokens_unprocessed(self, stream, stack=('root',)):
+ """
+ Split ``text`` into (tokentype, text) pairs.
+
+ ``stack`` is the inital stack (default: ``['root']``)
+ """
+ pos = 0
+ tokendefs = self._tokens # see __call__, pylint:disable=E1101
+ statestack = list(stack)
+ statetokens = tokendefs[statestack[-1]]
+ known_names = {}
+
+ text = stream.read()
+ text = self._decode(text)
+
+ while 1:
+ for rexmatch, action, new_state in statetokens:
+ m = rexmatch(text, pos)
+ if m:
+ value = m.group()
+ if value in known_names:
+ yield pos, known_names[value], value
+ elif type(action) is tokens._TokenType:
+ yield pos, action, value
+ elif hasattr(action, '__call__'):
+ ttype, value = action(value)
+ known_names[value] = ttype
+ yield pos, ttype, value
+ else:
+ for item in action(self, m):
+ yield item
+ pos = m.end()
+ if new_state is not None:
+ # state transition
+ if isinstance(new_state, tuple):
+ for state in new_state:
+ if state == '#pop':
+ statestack.pop()
+ elif state == '#push':
+ statestack.append(statestack[-1])
+ elif (
+ # Ugly hack - multiline-comments
+ # are not stackable
+ state != 'multiline-comments'
+ or not statestack
+ or statestack[-1] != 'multiline-comments'
+ ):
+ statestack.append(state)
+ elif isinstance(new_state, int):
+ # pop
+ del statestack[new_state:]
+ elif new_state == '#push':
+ statestack.append(statestack[-1])
+ else:
+ assert False, "wrong state def: %r" % new_state
+ statetokens = tokendefs[statestack[-1]]
+ break
+ else:
+ try:
+ if text[pos] == '\n':
+ # at EOL, reset state to "root"
+ pos += 1
+ statestack = ['root']
+ statetokens = tokendefs['root']
+ yield pos, tokens.Text, u'\n'
+ continue
+ yield pos, tokens.Error, text[pos]
+ pos += 1
+ except IndexError:
+ break
+
+
+def tokenize(sql, encoding=None):
+ """Tokenize sql.
+
+ Tokenize *sql* using the :class:`Lexer` and return a 2-tuple stream
+ of ``(token type, value)`` items.
+ """
+ lexer = Lexer()
+ if encoding is not None:
+ lexer.encoding = encoding
+ return lexer.get_tokens(sql)
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/sqlparse/pipeline.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/sqlparse/pipeline.py b/shell/ext-py/sqlparse-0.1.19/sqlparse/pipeline.py
new file mode 100644
index 0000000..34dad19
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/sqlparse/pipeline.py
@@ -0,0 +1,31 @@
+# Copyright (C) 2011 Jesus Leganes "piranna", piranna@gmail.com
+#
+# This module is part of python-sqlparse and is released under
+# the BSD License: http://www.opensource.org/licenses/bsd-license.php.
+
+from types import GeneratorType
+
+
+class Pipeline(list):
+ """Pipeline to process filters sequentially"""
+
+ def __call__(self, stream):
+ """Run the pipeline
+
+ Return a static (non generator) version of the result
+ """
+
+ # Run the stream over all the filters on the pipeline
+ for filter in self:
+ # Functions and callable objects (objects with '__call__' method)
+ if callable(filter):
+ stream = filter(stream)
+
+ # Normal filters (objects with 'process' method)
+ else:
+ stream = filter.process(None, stream)
+
+ # If last filter return a generator, staticalize it inside a list
+ if isinstance(stream, GeneratorType):
+ return list(stream)
+ return stream
http://git-wip-us.apache.org/repos/asf/impala/blob/49413d9c/shell/ext-py/sqlparse-0.1.19/sqlparse/sql.py
----------------------------------------------------------------------
diff --git a/shell/ext-py/sqlparse-0.1.19/sqlparse/sql.py b/shell/ext-py/sqlparse-0.1.19/sqlparse/sql.py
new file mode 100644
index 0000000..7325712
--- /dev/null
+++ b/shell/ext-py/sqlparse-0.1.19/sqlparse/sql.py
@@ -0,0 +1,684 @@
+# -*- coding: utf-8 -*-
+
+"""This module contains classes representing syntactical elements of SQL."""
+
+import re
+import sys
+
+from sqlparse import tokens as T
+
+
+class Token(object):
+ """Base class for all other classes in this module.
+
+ It represents a single token and has two instance attributes:
+ ``value`` is the unchange value of the token and ``ttype`` is
+ the type of the token.
+ """
+
+ __slots__ = ('value', 'ttype', 'parent', 'normalized', 'is_keyword')
+
+ def __init__(self, ttype, value):
+ self.value = value
+ if ttype in T.Keyword:
+ self.normalized = value.upper()
+ else:
+ self.normalized = value
+ self.ttype = ttype
+ self.is_keyword = ttype in T.Keyword
+ self.parent = None
+
+ def __str__(self):
+ if sys.version_info[0] == 3:
+ return self.value
+ else:
+ return unicode(self).encode('utf-8')
+
+ def __repr__(self):
+ short = self._get_repr_value()
+ if sys.version_info[0] < 3:
+ short = short.encode('utf-8')
+ return '<%s \'%s\' at 0x%07x>' % (self._get_repr_name(),
+ short, id(self))
+
+ def __unicode__(self):
+ """Returns a unicode representation of this object."""
+ return self.value or ''
+
+ def to_unicode(self):
+ """Returns a unicode representation of this object.
+
+ .. deprecated:: 0.1.5
+ Use ``unicode(token)`` (for Python 3: ``str(token)``) instead.
+ """
+ return unicode(self)
+
+ def _get_repr_name(self):
+ return str(self.ttype).split('.')[-1]
+
+ def _get_repr_value(self):
+ raw = unicode(self)
+ if len(raw) > 7:
+ raw = raw[:6] + u'...'
+ return re.sub('\s+', ' ', raw)
+
+ def flatten(self):
+ """Resolve subgroups."""
+ yield self
+
+ def match(self, ttype, values, regex=False):
+ """Checks whether the token matches the given arguments.
+
+ *ttype* is a token type. If this token doesn't match the given token
+ type.
+ *values* is a list of possible values for this token. The values
+ are OR'ed together so if only one of the values matches ``True``
+ is returned. Except for keyword tokens the comparison is
+ case-sensitive. For convenience it's ok to pass in a single string.
+ If *regex* is ``True`` (default is ``False``) the given values are
+ treated as regular expressions.
+ """
+ type_matched = self.ttype is ttype
+ if not type_matched or values is None:
+ return type_matched
+
+ if regex:
+ if isinstance(values, basestring):
+ values = set([values])
+
+ if self.ttype is T.Keyword:
+ values = set(re.compile(v, re.IGNORECASE) for v in values)
+ else:
+ values = set(re.compile(v) for v in values)
+
+ for pattern in values:
+ if pattern.search(self.value):
+ return True
+ return False
+
+ if isinstance(values, basestring):
+ if self.is_keyword:
+ return values.upper() == self.normalized
+ return values == self.value
+
+ if self.is_keyword:
+ for v in values:
+ if v.upper() == self.normalized:
+ return True
+ return False
+
+ return self.value in values
+
+ def is_group(self):
+ """Returns ``True`` if this object has children."""
+ return False
+
+ def is_whitespace(self):
+ """Return ``True`` if this token is a whitespace token."""
+ return self.ttype and self.ttype in T.Whitespace
+
+ def within(self, group_cls):
+ """Returns ``True`` if this token is within *group_cls*.
+
+ Use this method for example to check if an identifier is within
+ a function: ``t.within(sql.Function)``.
+ """
+ parent = self.parent
+ while parent:
+ if isinstance(parent, group_cls):
+ return True
+ parent = parent.parent
+ return False
+
+ def is_child_of(self, other):
+ """Returns ``True`` if this token is a direct child of *other*."""
+ return self.parent == other
+
+ def has_ancestor(self, other):
+ """Returns ``True`` if *other* is in this tokens ancestry."""
+ parent = self.parent
+ while parent:
+ if parent == other:
+ return True
+ parent = parent.parent
+ return False
+
+
+class TokenList(Token):
+ """A group of tokens.
+
+ It has an additional instance attribute ``tokens`` which holds a
+ list of child-tokens.
+ """
+
+ __slots__ = ('value', 'ttype', 'tokens')
+
+ def __init__(self, tokens=None):
+ if tokens is None:
+ tokens = []
+ self.tokens = tokens
+ Token.__init__(self, None, self._to_string())
+
+ def __unicode__(self):
+ return self._to_string()
+
+ def __str__(self):
+ str_ = self._to_string()
+ if sys.version_info[0] < 2:
+ str_ = str_.encode('utf-8')
+ return str_
+
+ def _to_string(self):
+ if sys.version_info[0] == 3:
+ return ''.join(x.value for x in self.flatten())
+ else:
+ return ''.join(unicode(x) for x in self.flatten())
+
+ def _get_repr_name(self):
+ return self.__class__.__name__
+
+ def _pprint_tree(self, max_depth=None, depth=0):
+ """Pretty-print the object tree."""
+ indent = ' ' * (depth * 2)
+ for idx, token in enumerate(self.tokens):
+ if token.is_group():
+ pre = ' +-'
+ else:
+ pre = ' | '
+ print '%s%s%d %s \'%s\'' % (indent, pre, idx,
+ token._get_repr_name(),
+ token._get_repr_value())
+ if (token.is_group() and (max_depth is None or depth < max_depth)):
+ token._pprint_tree(max_depth, depth + 1)
+
+ def _remove_quotes(self, val):
+ """Helper that removes surrounding quotes from strings."""
+ if not val:
+ return val
+ if val[0] in ('"', '\'') and val[-1] == val[0]:
+ val = val[1:-1]
+ return val
+
+ def get_token_at_offset(self, offset):
+ """Returns the token that is on position offset."""
+ idx = 0
+ for token in self.flatten():
+ end = idx + len(token.value)
+ if idx <= offset <= end:
+ return token
+ idx = end
+
+ def flatten(self):
+ """Generator yielding ungrouped tokens.
+
+ This method is recursively called for all child tokens.
+ """
+ for token in self.tokens:
+ if isinstance(token, TokenList):
+ for item in token.flatten():
+ yield item
+ else:
+ yield token
+
+# def __iter__(self):
+# return self
+#
+# def next(self):
+# for token in self.tokens:
+# yield token
+
+ def is_group(self):
+ return True
+
+ def get_sublists(self):
+# return [x for x in self.tokens if isinstance(x, TokenList)]
+ for x in self.tokens:
+ if isinstance(x, TokenList):
+ yield x
+
+ @property
+ def _groupable_tokens(self):
+ return self.tokens
+
+ def token_first(self, ignore_whitespace=True, ignore_comments=False):
+ """Returns the first child token.
+
+ If *ignore_whitespace* is ``True`` (the default), whitespace
+ tokens are ignored.
+
+ if *ignore_comments* is ``True`` (default: ``False``), comments are
+ ignored too.
+ """
+ for token in self.tokens:
+ if ignore_whitespace and token.is_whitespace():
+ continue
+ if ignore_comments and isinstance(token, Comment):
+ continue
+ return token
+
+ def token_next_by_instance(self, idx, clss, end=None):
+ """Returns the next token matching a class.
+
+ *idx* is where to start searching in the list of child tokens.
+ *clss* is a list of classes the token should be an instance of.
+
+ If no matching token can be found ``None`` is returned.
+ """
+ if not isinstance(clss, (list, tuple)):
+ clss = (clss,)
+
+ for token in self.tokens[idx:end]:
+ if isinstance(token, clss):
+ return token
+
+ def token_next_by_type(self, idx, ttypes):
+ """Returns next matching token by it's token type."""
+ if not isinstance(ttypes, (list, tuple)):
+ ttypes = [ttypes]
+
+ for token in self.tokens[idx:]:
+ if token.ttype in ttypes:
+ return token
+
+ def token_next_match(self, idx, ttype, value, regex=False):
+ """Returns next token where it's ``match`` method returns ``True``."""
+ if not isinstance(idx, int):
+ idx = self.token_index(idx)
+
+ for n in xrange(idx, len(self.tokens)):
+ token = self.tokens[n]
+ if token.match(ttype, value, regex):
+ return token
+
+ def token_not_matching(self, idx, funcs):
+ for token in self.tokens[idx:]:
+ passed = False
+ for func in funcs:
+ if func(token):
+ passed = True
+ break
+
+ if not passed:
+ return token
+
+ def token_matching(self, idx, funcs):
+ for token in self.tokens[idx:]:
+ for func in funcs:
+ if func(token):
+ return token
+
+ def token_prev(self, idx, skip_ws=True):
+ """Returns the previous token relative to *idx*.
+
+ If *skip_ws* is ``True`` (the default) whitespace tokens are ignored.
+ ``None`` is returned if there's no previous token.
+ """
+ if idx is None:
+ return None
+
+ if not isinstance(idx, int):
+ idx = self.token_index(idx)
+
+ while idx:
+ idx -= 1
+ if self.tokens[idx].is_whitespace() and skip_ws:
+ continue
+ return self.tokens[idx]
+
+ def token_next(self, idx, skip_ws=True):
+ """Returns the next token relative to *idx*.
+
+ If *skip_ws* is ``True`` (the default) whitespace tokens are ignored.
+ ``None`` is returned if there's no next token.
+ """
+ if idx is None:
+ return None
+
+ if not isinstance(idx, int):
+ idx = self.token_index(idx)
+
+ while idx < len(self.tokens) - 1:
+ idx += 1
+ if self.tokens[idx].is_whitespace() and skip_ws:
+ continue
+ return self.tokens[idx]
+
+ def token_index(self, token, start=0):
+ """Return list index of token."""
+ if start > 0:
+ # Performing `index` manually is much faster when starting in the middle
+ # of the list of tokens and expecting to find the token near to the starting
+ # index.
+ for i in xrange(start, len(self.tokens)):
+ if self.tokens[i] == token:
+ return i
+ return -1
+ return self.tokens.index(token)
+
+ def tokens_between(self, start, end, exclude_end=False):
+ """Return all tokens between (and including) start and end.
+
+ If *exclude_end* is ``True`` (default is ``False``) the end token
+ is included too.
+ """
+ # FIXME(andi): rename exclude_end to inlcude_end
+ if exclude_end:
+ offset = 0
+ else:
+ offset = 1
+ end_idx = self.token_index(end) + offset
+ start_idx = self.token_index(start)
+ return self.tokens[start_idx:end_idx]
+
+ def group_tokens(self, grp_cls, tokens, ignore_ws=False):
+ """Replace tokens by an instance of *grp_cls*."""
+ idx = self.token_index(tokens[0])
+ if ignore_ws:
+ while tokens and tokens[-1].is_whitespace():
+ tokens = tokens[:-1]
+ for t in tokens:
+ self.tokens.remove(t)
+ grp = grp_cls(tokens)
+ for token in tokens:
+ token.parent = grp
+ grp.parent = self
+ self.tokens.insert(idx, grp)
+ return grp
+
+ def insert_before(self, where, token):
+ """Inserts *token* before *where*."""
+ self.tokens.insert(self.token_index(where), token)
+
+ def insert_after(self, where, token, skip_ws=True):
+ """Inserts *token* after *where*."""
+ next_token = self.token_next(where, skip_ws=skip_ws)
+ if next_token is None:
+ self.tokens.append(token)
+ else:
+ self.tokens.insert(self.token_index(next_token), token)
+
+ def has_alias(self):
+ """Returns ``True`` if an alias is present."""
+ return self.get_alias() is not None
+
+ def get_alias(self):
+ """Returns the alias for this identifier or ``None``."""
+
+ # "name AS alias"
+ kw = self.token_next_match(0, T.Keyword, 'AS')
+ if kw is not None:
+ return self._get_first_name(kw, keywords=True)
+
+ # "name alias" or "complicated column expression alias"
+ if len(self.tokens) > 2 \
+ and self.token_next_by_type(0, T.Whitespace) is not None:
+ return self._get_first_name(reverse=True)
+
+ return None
+
+ def get_name(self):
+ """Returns the name of this identifier.
+
+ This is either it's alias or it's real name. The returned valued can
+ be considered as the name under which the object corresponding to
+ this identifier is known within the current statement.
+ """
+ alias = self.get_alias()
+ if alias is not None:
+ return alias
+ return self.get_real_name()
+
+ def get_real_name(self):
+ """Returns the real name (object name) of this identifier."""
+ # a.b
+ dot = self.token_next_match(0, T.Punctuation, '.')
+ if dot is not None:
+ return self._get_first_name(self.token_index(dot))
+
+ return self._get_first_name()
+
+ def get_parent_name(self):
+ """Return name of the parent object if any.
+
+ A parent object is identified by the first occuring dot.
+ """
+ dot = self.token_next_match(0, T.Punctuation, '.')
+ if dot is None:
+ return None
+ prev_ = self.token_prev(self.token_index(dot))
+ if prev_ is None: # something must be verry wrong here..
+ return None
+ return self._remove_quotes(prev_.value)
+
+ def _get_first_name(self, idx=None, reverse=False, keywords=False):
+ """Returns the name of the first token with a name"""
+
+ if idx and not isinstance(idx, int):
+ idx = self.token_index(idx) + 1
+
+ tokens = self.tokens[idx:] if idx else self.tokens
+ tokens = reversed(tokens) if reverse else tokens
+ types = [T.Name, T.Wildcard, T.String.Symbol]
+
+ if keywords:
+ types.append(T.Keyword)
+
+ for tok in tokens:
+ if tok.ttype in types:
+ return self._remove_quotes(tok.value)
+ elif isinstance(tok, Identifier) or isinstance(tok, Function):
+ return tok.get_name()
+ return None
+
+class Statement(TokenList):
+ """Represents a SQL statement."""
+
+ __slots__ = ('value', 'ttype', 'tokens')
+
+ def get_type(self):
+ """Returns the type of a statement.
+
+ The returned value is a string holding an upper-cased reprint of
+ the first DML or DDL keyword. If the first token in this group
+ isn't a DML or DDL keyword "UNKNOWN" is returned.
+
+ Whitespaces and comments at the beginning of the statement
+ are ignored.
+ """
+ first_token = self.token_first(ignore_comments=True)
+ if first_token is None:
+ # An "empty" statement that either has not tokens at all
+ # or only whitespace tokens.
+ return 'UNKNOWN'
+
+ elif first_token.ttype in (T.Keyword.DML, T.Keyword.DDL):
+ return first_token.normalized
+
+ return 'UNKNOWN'
+
+
+class Identifier(TokenList):
+ """Represents an identifier.
+
+ Identifiers may have aliases or typecasts.
+ """
+
+ __slots__ = ('value', 'ttype', 'tokens')
+
+ def is_wildcard(self):
+ """Return ``True`` if this identifier contains a wildcard."""
+ token = self.token_next_by_type(0, T.Wildcard)
+ return token is not None
+
+ def get_typecast(self):
+ """Returns the typecast or ``None`` of this object as a string."""
+ marker = self.token_next_match(0, T.Punctuation, '::')
+ if marker is None:
+ return None
+ next_ = self.token_next(self.token_index(marker), False)
+ if next_ is None:
+ return None
+ return unicode(next_)
+
+ def get_ordering(self):
+ """Returns the ordering or ``None`` as uppercase string."""
+ ordering = self.token_next_by_type(0, T.Keyword.Order)
+ if ordering is None:
+ return None
+ return ordering.value.upper()
+
+ def get_array_indices(self):
+ """Returns an iterator of index token lists"""
+
+ for tok in self.tokens:
+ if isinstance(tok, SquareBrackets):
+ # Use [1:-1] index to discard the square brackets
+ yield tok.tokens[1:-1]
+
+
+class IdentifierList(TokenList):
+ """A list of :class:`~sqlparse.sql.Identifier`\'s."""
+
+ __slots__ = ('value', 'ttype', 'tokens')
+
+ def get_identifiers(self):
+ """Returns the identifiers.
+
+ Whitespaces and punctuations are not included in this generator.
+ """
+ for x in self.tokens:
+ if not x.is_whitespace() and not x.match(T.Punctuation, ','):
+ yield x
+
+
+class Parenthesis(TokenList):
+ """Tokens between parenthesis."""
+ __slots__ = ('value', 'ttype', 'tokens')
+
+ @property
+ def _groupable_tokens(self):
+ return self.tokens[1:-1]
+
+
+class SquareBrackets(TokenList):
+ """Tokens between square brackets"""
+
+ __slots__ = ('value', 'ttype', 'tokens')
+
+ @property
+ def _groupable_tokens(self):
+ return self.tokens[1:-1]
+
+class Assignment(TokenList):
+ """An assignment like 'var := val;'"""
+ __slots__ = ('value', 'ttype', 'tokens')
+
+
+class If(TokenList):
+ """An 'if' clause with possible 'else if' or 'else' parts."""
+ __slots__ = ('value', 'ttype', 'tokens')
+
+
+class For(TokenList):
+ """A 'FOR' loop."""
+ __slots__ = ('value', 'ttype', 'tokens')
+
+
+class Comparison(TokenList):
+ """A comparison used for example in WHERE clauses."""
+ __slots__ = ('value', 'ttype', 'tokens')
+
+ @property
+ def left(self):
+ return self.tokens[0]
+
+ @property
+ def right(self):
+ return self.tokens[-1]
+
+
+class Comment(TokenList):
+ """A comment."""
+ __slots__ = ('value', 'ttype', 'tokens')
+
+ def is_multiline(self):
+ return self.tokens and self.tokens[0].ttype == T.Comment.Multiline
+
+
+class Where(TokenList):
+ """A WHERE clause."""
+ __slots__ = ('value', 'ttype', 'tokens')
+
+
+class Case(TokenList):
+ """A CASE statement with one or more WHEN and possibly an ELSE part."""
+
+ __slots__ = ('value', 'ttype', 'tokens')
+
+ def get_cases(self):
+ """Returns a list of 2-tuples (condition, value).
+
+ If an ELSE exists condition is None.
+ """
+ CONDITION = 1
+ VALUE = 2
+
+ ret = []
+ mode = CONDITION
+
+ for token in self.tokens:
+ # Set mode from the current statement
+ if token.match(T.Keyword, 'CASE'):
+ continue
+
+ elif token.match(T.Keyword, 'WHEN'):
+ ret.append(([], []))
+ mode = CONDITION
+
+ elif token.match(T.Keyword, 'THEN'):
+ mode = VALUE
+
+ elif token.match(T.Keyword, 'ELSE'):
+ ret.append((None, []))
+ mode = VALUE
+
+ elif token.match(T.Keyword, 'END'):
+ mode = None
+
+ # First condition without preceding WHEN
+ if mode and not ret:
+ ret.append(([], []))
+
+ # Append token depending of the current mode
+ if mode == CONDITION:
+ ret[-1][0].append(token)
+
+ elif mode == VALUE:
+ ret[-1][1].append(token)
+
+ # Return cases list
+ return ret
+
+
+class Function(TokenList):
+ """A function or procedure call."""
+
+ __slots__ = ('value', 'ttype', 'tokens')
+
+ def get_parameters(self):
+ """Return a list of parameters."""
+ parenthesis = self.tokens[-1]
+ for t in parenthesis.tokens:
+ if isinstance(t, IdentifierList):
+ return t.get_identifiers()
+ elif isinstance(t, Identifier) or \
+ isinstance(t, Function) or \
+ t.ttype in T.Literal:
+ return [t,]
+ return []
+
+
+class Begin(TokenList):
+ """A BEGIN/END block."""
+
+ __slots__ = ('value', 'ttype', 'tokens')