You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@kylin.apache.org by GitBox <gi...@apache.org> on 2018/07/08 09:49:19 UTC
[GitHub] shaofengshi closed pull request #23: Kylin es
shaofengshi closed pull request #23: Kylin es
URL: https://github.com/apache/kylin/pull/23
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000000..929054cd6f
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,105 @@
+### Eclipse
+*.pydevproject
+.metadata
+.gradle
+tmp/
+*.tmp
+*.bak
+*.swp
+*~.nib
+*.versionsBackup
+local.properties
+release.properties
+.loadpath
+
+# maven release files
+*.versionsBackup
+**/pom.xml.tag
+
+#IDEA
+*.iml
+#.settings
+
+# External tool builders
+.externalToolBuilders/
+
+# Locally stored "Eclipse launch configurations"
+*.launch
+
+# CDT-specific
+.cproject
+
+# PDT-specific
+.buildpath
+
+# sbteclipse plugin
+.target
+
+# TeXlipse plugin
+.texlipse
+
+### Java
+*.class
+
+# Mobile Tools for Java (J2ME)
+.mtj.tmp/
+
+# Package Files #
+*.jar
+*.war
+*.ear
+*.tgz
+*.tar.gz
+
+
+
+# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
+hs_err_pid*
+
+
+### Origin
+# /
+.classpath
+.project
+.idea
+**/*.DS_Store
+*.releaseBackup
+*.log
+jdbc/kylin_jdbc.log.*
+*.DS_Store
+
+
+
+# .settings
+org.eclipse.m2e.core.prefs
+org.eclipse.wst.common.component
+org.eclipse.wst.common.project.facet.core.xml
+org.eclipse.wst.validation.prefs
+server/.settings/.jsdtscope
+server/.settings/org.eclipse.wst.common.project.facet.core.prefs.xml
+server/.settings/org.eclipse.wst.jsdt.ui.superType.container
+server/.settings/org.eclipse.wst.jsdt.ui.superType.name
+
+#web
+webapp/app/WEB-INF/*
+webapp/dist
+webapp/node_modules:
+webapp/CHANGELOG.md
+
+# generated files
+target
+logs
+
+# package file and folder
+dist
+tomcat
+lib
+webapp/app/components/*
+commit.sha1
+
+# website
+website/_site
+website/.sass-cache
+
+
+
diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs
new file mode 100644
index 0000000000..99f26c0203
--- /dev/null
+++ b/.settings/org.eclipse.core.resources.prefs
@@ -0,0 +1,2 @@
+eclipse.preferences.version=1
+encoding/<project>=UTF-8
diff --git a/KEYS b/KEYS
new file mode 100644
index 0000000000..06e5b4e4f1
--- /dev/null
+++ b/KEYS
@@ -0,0 +1,131 @@
+This file contains the PGP keys of various developers.
+Please don't use them for email unless you have to. Their main
+purpose is code signing.
+
+Examples of importing this file in your keystore:
+ gpg --import KEYS.txt
+ (need pgp and other examples here)
+
+Examples of adding your key to this file:
+ pgp -kxa <your name> and append it to this file.
+ (pgpk -ll <your name> && pgpk -xa <your name>) >> this file.
+ (gpg --list-sigs <your name>
+ && gpg --armor --export <your name>) >> this file.
+
+-----------------------------------------------------------------------------------
+
+pub 4096R/027DC364 2015-04-10
+uid Luke Han <lu...@apache.org>
+sig 3 027DC364 2015-04-10 Luke Han <lu...@apache.org>
+sub 4096R/19BCFD87 2015-04-10
+sig 027DC364 2015-04-10 Luke Han <lu...@apache.org>
+
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1
+
+mQINBFUno+8BEACxRnKkHFkA/qbYKyidAsMgvgsZZIwONlAoQE+01MKsGPbJU/DR
+zVyzD5nFskaoZ3dLTeQfKP/BJ9qfXpm0zlS4PPHkrbKOeV+SawiiOvSLYCijz6vU
+T1lbgW3yIpMkFp0zpIaZo7BpA2Q9s0qvPeZrDenu1X4IeNrUHs2QInYFDrUpyghw
+BOrt4Yrx7KmtWDPia/Hvbd5nqy9coHZO+HYzBJhXPXK5arbrCDF+NEJFcUI3VlP1
+mPF0LG/SfeVtsjPVo+Ub9HIMlwd6NnzIveSXc5FWIaweQ5ko38TGgKLJ9UXkH5C+
+c/sPOaKVAYIcX5i94BLigjxAQAg15kDr1dX7gPnjQNZdLjMRWiTf4eSN5+rWK+j+
+IeA8QIcVGz7fmhyZ8J/9Gr9TdERvMuhoV/ll8aCGAQU5ApLfgVMIK8kW1vr8dFDh
+x0tnrFZIPcNRg4K9ISR/DEFxYvSkXlxaBudIJiM5itgJ2dgltIIUX5uD8h2gi94A
+4wJfI9m1F2v84lDjv6ve+O3O/P3OPErAKaKjn8w/PHFmGfVAGLkRIV/GQx39gDOf
+QFMoGhqOdLeSIWLodg73HtpZAlVEVZ0+50cHtKqWfQ357Lw3aRJdO/WWrcDZQxut
+lTiULde+vxgwWExpZf98HbpcwFNckA6+m0bwCCc9GEXkDUbPbuvpPvjJOQARAQAB
+tB1MdWtlIEhhbiA8bHVrZWhhbkBhcGFjaGUub3JnPokCOAQTAQIAIgUCVSej7wIb
+AwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQoRMvEgJ9w2RoDhAArSo9kHlQ
+e84hP//+4VSccubHudPTX/+GdDcrAa0+QVS+lhnqe47sLsrvY2Hslm77AIZnYDOk
+U76FpKkK4AcMjP+ayLT6KI/ds4k9ca3OvIgj9TYtFczOg2w1dq0sTKx5MXPXJpDL
+o/Ee7Vuk10rKK9796B8keUHfFvexgfIFW39LaIOG0US5sx7IYNMIbpzEknl9dPi/
+/Mxa0+TTbeh+Jj+SkeGOV9SlcT3HWX+oXK3+saV5ChQHR4Db+wh0vhfgixi0YMbJ
+iik0aPeATiLWvccpS3t5mBE5jP+Ss1xeptXaAmXJl9QC3J5lnIMZPhI7ie83agO8
+ZRcffUMfWw/4HJbB6kj9Cs82zIT8Qfz+w+3PyoLvU65IHmY8FxknQX2jg9XFMZ+C
+c9NgJQyAOl2watglEJytjZ9UMfg36Zg0NYUmK+TMJzuMjtPbXgS7EyboMju9cdbS
+1U4AStGFHUUcnw9r7jP9O9w7DtIjo9fdIDsfHkuNf/RMbZ4xAXWF/tbk++NpIjSY
+qBXD2ZRUg+E6QfDvXOabpJ/14cVoYUlcFYeN9MpgDAKh94xmsyyxpnnyCWI3WTBj
+aMT/p7xCCvYJ5XmyyV4zRHpQ/tHuZ/XnNfkeJavCxMXuslf5AyBP64uRB00EksDf
+eMShtxE026rc9XyGf/pb7Ofg1zpbGl+Fhca5Ag0EVSej7wEQAKykheo7Hpq+m5Pl
+NFGPp3Y8kN5hV5XoJLvKgbF4UnSB8/4IXRyLfTO+5F1ePNRiRBGtWAseWKtsN/qj
+dgkc103Ckhl3NX/bH7o7mXsyOUThxAFCz/hzR1ca20H3lv63fB1L6wUD24aZ+m+9
+xf3RqiPNC/OYEvIN0NKeFNzpWWGNTaYcHReKiwwT8VpL0HzI8iu7oJI3Z/8Cmugk
+N39FiR8lJTQspmRzUJerQHqB2E6M0B252DTD7syZZYHOgP56A9V9KsMApQ4mC/qz
+ntdc0wTwLdAejrNs+qaSuIxSXhW9kPPJbazsDnvqEI3oAIU/4Zo+xt/QZpENgZTO
+xEnzIoxby75Qq54vCZSxnVsjJfJv4ZB3DIq0bwEeOK75gcyGAShnxDCzdD0rkvwj
+oyqiGu15bRKW+AiJAll2qs7PCV7sEUFx9pvz9QOx2JYyNKY246koe0j5nUDPY5uU
+NnFdq2x+LpmuT3hZ/ptEhXzgQUEyw0LFGz8sMtkZ8Vk0gbJnv1nVqYVDjfcv9Ggy
+DePBFkIpp/jWEVPWhlgQwYouNHF+W5Rs/r7eKpIwAQMw+aOcmn7BCxLkznZCvodG
+373kIjy5to+hg0tIWiVcirGdCh1ktPxeRrvTkoFd001x8GU+z+E6JGYrIIuTi4JE
+/cEqaV+wuKKqr2cFxxql9RXGYUWhABEBAAGJAh8EGAECAAkFAlUno+8CGwwACgkQ
+oRMvEgJ9w2R2VQ/9FGp/O1F8TYITJWAyOUQONdk8pZriAk+9Oon9DkxhsYdDD1oB
+td43wvFLr5dC7j9wvbnmYDWTlAwbeZYfZSS38YlrAH8/jwsuEtz6ez2faqb4qAz6
+OSGuJv3sp1T4fFUpd/O5KyGej90fmoFygSD7PdY9OEPKCHZpJuVqYMA+IaU2QXyL
+giX4jspR5SJVaJJ+kyB7+IVIVwMqxhDQ0PU4v+7hx+kVyoL7f26P+SvMkPYzM2/w
+HQrWMvuVT1NhPclk9wxdOJL1WJLNoR5U/wtZ/DBIV2gu457AyzuopHB8ye55J3q3
+mbrf24LrCUZxiofh+5GTjlC8EAcs79mAKRXUxVRah4rZovRg5jwCHgMsU0VFBzNH
+fUq0euFIhyW14c8KOhVVCUcsDwONK0EEKJnG+cPhyjrzhdtfEy6StDCvg43PI7/w
+Dodm8LJi7H1yG4LHDKalZ22st2K5DjYJWRBWXUYpCFWxQiOzGq8LdyYL3lQ/lBxt
+ElBcVmy0Qvqf5/r9jVOpZWWvoAhJT0bj5B0Ca3CuGtkiVPJIMc2QuTAItWyRDN7T
+EBG4pxGI2XsApbts0AkkINEYAKN+Z/o9BtrpEvodFPn2TBR3D9zPM+XiSZXTZ/gG
+zJ5Mba6u6w6DZa238CtPdRTMh5ZAGhgkWYCrpQJ+CD9qflqNFzpTQ4FtPg8=
+=YeBe
+-----END PGP PUBLIC KEY BLOCK-----
+pub 4096R/6FCF4720 2015-09-29
+uid [ultimate] Shaofeng Shi <sh...@apache.org>
+sig 3 6FCF4720 2015-09-29 Shaofeng Shi <sh...@apache.org>
+sub 4096R/90EFBBD8 2015-09-29
+sig 6FCF4720 2015-09-29 Shaofeng Shi <sh...@apache.org>
+
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Comment: GPGTools - https://gpgtools.org
+
+mQINBFYKTLQBEACwI2aNrCpaYAHI4nMjKSVOlBBCVA5mPaLTLeRXPWm9LY1HSjtl
+HAS3GZYijhwhJ9ZiDtHTvZUyZdEAJZf16jNg7p5dhdSfjp/b91a8NfNuoLuSXx7R
+NdBd4xB8IeSjZDGeMw5wqUXrta0w7dNJjEiYs9/4lh3BkA/x/NPRfjkYmHk6wixj
+YGs6rpolACqRBnZ612qoRI7eOrSy9WC6gU3sajHlWiOPHECBF7jNHU6aeQH4YIig
+UufHALU1gdb4o8qH6h9QibvCKoqdI7LwnD4Pn53SXAfpnNpkgH0SLzdpG++argd+
+33IQeo6DCNazkQ//qdRlrXFsriGNEq8Aip+614hyDkS3N20MZzHrX2sfOz19oZN4
+5UNvkz7gbys/L4MJL1RFapjgofXFKr7wNPAdWE+ZvJ4Irw88pM1LmtDSS/m1tyLn
+hz51vHcw/sBo90DisgNjPZUvO9kHXb1LTVVVmL6EdO0PFp9b4qlKGWUk+IGKejI6
+5ooSlMSjvPAYlmQvy+dcl5uVu+n9M/S5gPIWGiPxrJc73SzOZaHedTSVd2g9yJCz
+smuKKni7og19wgPrAEPvmVS/uObp+rmae2R5/EqgBL1PAIxnPCJbqUJp/865ti2K
+atQeua1yntn3cTFCpO6CkSQXk2LjMtLglZl4AOmf+9UmKi/AsefvSD8t+wARAQAB
+tCVTaGFvZmVuZyBTaGkgPHNoYW9mZW5nc2hpQGFwYWNoZS5vcmc+iQI3BBMBCgAh
+BQJWCky0AhsDBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEKvJ+IVvz0cgVE4P
+/RucXCEPWyG3GY7pGtgBuouBfhHhrWKAkVd58818NwLfOkvTWbGuGNWBqQ1CRPGk
+8ZhaNoVx3T8RqIOOyLx+ynOB6iBhPrVOyMsIxb7jh1BnMhe2aGTZGrrQCChGZk2H
+cdxVbCipO4Pw+YdUtJ4Mc3Xm14TkwAygGqrfJEKXgiAWBlvzrKp6twc69jgSd+To
+/UIrSFdPSlpbEpPiMuGKfhrz8OxPYy6N+/AgwIqGub/8NX9cteLCXRQmL5h7Z+6f
+7Wa4XGlEDfxCJFnWKsUYHwP5QHkJPfESGdbYgI+nrN8rv25rtINfWaiBgAsFSg4D
+KYZBXAelumDmLD/Rpn+hhjpeC5f6lPCqap1atHoqLx/s8YI8149on049pzA5t/zK
++G+c8bYf8xm6Ky/dBQP15ExoeeIpjC5A5vNlxenaEXXCya50J4+JLC5DTi9xRU7R
+Il3InFjeRmx1hPeyuYSY2irzBFvDhdt7U0wvwERS0lHMkwvB/MPrEwuB/II2tsFQ
+eZvHFhROYh4H8TqE9qICRtcdqNR8BXsZdEJtno4YH//b58130qJSNwcBF/mt+IcZ
+ryVQe3Y87evcF8RWTvWd4KQxe8BGoGQ5o8U230bxJ2bNYl+6O8Xzg+Q4HGpX4DGw
+EUVf4fE3n9O6kH0p9P6RAgnvlsKtsyubxlWz2zNXFuteuQINBFYKTLQBEADbKpjV
+NxaIuUg+wYSLP5erFvHwqDT5OJCj2y4wbwTzVb/1wZL2QwIPzb8lmlJuRsHo1E+j
+bjQE0I1RWiCAdekIpljBHc7fLsYhV0ONMgQZxlh3lpZCIftKmIPX4Eh93bhDBtRi
+CnYcWa/MDEH8rccwZaz0j+2OstIAv7x5/mUC55eutFtH4m5wRkbAjgcZbu6h0MNf
+KbBB4sOfHyx9L1bDIph42Hoh8rp+sMDYNf5Hnxr3Qt4ojANtQgQJzXudI6GzDZHy
+laIle5nJudyPy/TdF/8QXsUT62fbO8W8e4Q8Ljx192RULGoVYlz8WPkXF1vbL8Yb
+mMJyQ9YCgc5jIg/NusMFT0OXHV3YNQ/tZNh57i27pGO4RolRRbrXPrE/npIGBrsR
+8myPu3ussAMSWF2ejhWupKYS943ujNDOhKqC88R5mwHSzDpfytMUMsElwzrioAPV
+EiCCcJ8/Q0u8RAlF9A629iquIA5pp4Ga+9CLHdeGUohmnpIePz71+sXLW+VUOUlM
++euttiH/ltnRV0LKvS0w6sSnfOFOcOGZKxJ1FJhD5rx86dy7vjrWveOfoCNKjm3a
+jeTOfptvuXzJy0rEgVsKBRLCosjJIVgyWR3UkyXFCq3HfS0IutmdKMzxO+/J6t0c
+pnUsIHEiMlV/2TjwM/icFfUSDzAM/D+OoYe/IQARAQABiQIfBBgBCgAJBQJWCky0
+AhsMAAoJEKvJ+IVvz0cgJ0QQAJt5YtBkMKzPc/QGQo9T86ArCijnt2Z0P6d3S0Qr
+dye87F8ZmM3MRl/Bd7EpZ/+l3157FTX3jBKBrWsKTFcrG5IXrbMTS81pi555AwBm
+HcwCDBdFaLBKlqmQemgNKgnOGt+/G3w1oetPlbkKQEgFjCFfMF8CS8HdNqJU8pmd
+RE3IHMEvwrqx1dbGntRvUrSxs+rRgjJHympIAwIPdLHO6YSIrsbZEghUeVzd/5xi
+cEN06IieGY5t7PImpxeTmYAAdbQsN2GvYOG8NdOY7GrQ2Pu83mreb5Ma/Ui5UtPd
+3KWanXTP4lliY1DYkphp8USjdTul+srOK6X4dQ4y3EYyOzOytvMNV+G4bcYoTnoS
+gUkT1p0oHArxrQej3Py+no4Y5FuYEPJdZ3oHOhkqgn9j/UZBRxfc6VMRAD3ymRXC
+I7eDwdwOryyLBvA16L1+gs2kc5cbbfeFgfFiEFlMfT3eUN9mu5cQ3dNCQ119fcJp
+cu3E67ykSeibobuwNFj+JCilCUGrV5KN3FPVet3TX8F09O9z0CJwjmiK6+JkvVV8
+GB8WCCu0wEwSHYlTZkzOmXMNYZjk8fNYkWplWbF6xNPs1A00XqDwsa/V0F9EzJV+
+nYr5b8EWaM9qsjRh2aFPFtV+E/cSc7sdJxIY/TSdMfqhB13l7LTU/NwmgjE2TTUH
+WYIE
+=6yhP
+-----END PGP PUBLIC KEY BLOCK-----
diff --git a/LICENSE b/LICENSE
index 29dc2956c2..e2d4f7bc7c 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,54 +1,247 @@
-Apache License
-Version 2.0, January 2004
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+
+
+
+==============================================================================
+Apache Kylin Subcomponents:
+
+The Apache Kylin project contains subcomponents with separate copyright
+notices and license terms. Your use of the source code for the these
+subcomponents is subject to the terms and conditions of the following
+licenses.
+
+
+==============================================================================
+For AdminLTE:
+==============================================================================
+This product bundles AdminLTE, which is available under a
+"MIT" license. For details, see
+https://github.com/almasaeed2010/AdminLTE/blob/master/LICENSE.
+
+
+==============================================================================
+For english-words.80 (scowl-2015.05.18).txt:
+==============================================================================
+This product bundles SCOWL, which is available under a
+MIT-like license. For details, see
+https://raw.githubusercontent.com/kevina/wordlist/master/scowl/Copyright
+
+==============================================================================
+For Google Fonts and Source Sans Pro
+==============================================================================
+This product bundles Google Fonts:
+
+Q: Can I use the Fonts API on any page?
+A: Yes. All the fonts on Google Fonts are licensed under open source licenses
+that let you use them on any website, from a private blog to a big commercial site.
+https://developers.google.com/fonts/faq?hl=en.
+https://www.google.com/fonts/attribution
+
+Source Sans Pro is under SIL Open Font License 1.1,
+For detail, see
+http://scripts.sil.org/cms/scripts/page.php?site_id=nrsi&id=OFL
-http://www.apache.org/licenses/
-TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-1. Definitions.
-
-"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document.
-
-"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License.
-
-"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity.
-
-"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License.
-
-"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files.
-
-"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types.
-
-"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below).
-
-"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof.
-
-"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution."
-
-"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work.
-
-2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form.
-
-3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed.
-
-4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions:
-
-You must give any other recipients of the Work or Derivative Works a copy of this License; and
-You must cause any modified files to carry prominent notices stating that You changed the files; and
-You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and
-If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License.
-
-You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License.
-5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions.
-
-6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file.
-
-7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License.
-
-8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages.
-
-9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability.
-
-END OF TERMS AND CONDITIONS
diff --git a/NOTICE b/NOTICE
new file mode 100644
index 0000000000..cd0c1ba4db
--- /dev/null
+++ b/NOTICE
@@ -0,0 +1,6 @@
+Apache Kylin
+Copyright 2014-2015 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
diff --git a/README.md b/README.md
index 44f6859d47..176d89c79f 100644
--- a/README.md
+++ b/README.md
@@ -1,94 +1,26 @@
-Kylin OLAP
-===========
+Apache Kylin
+============
-
-Overview
-------------
> Extreme OLAP Engine for Big Data
-An open source distributed Analytical Engine from eBay to provide SQL interface and multi-dimensional analysis (OLAP) on Hadoop to support TB to PB size analysis.
-
-
-Getting Started
----------------
-
-### Prerequisites ###
-
-#### Hadoop ####
-- Hadoop: 2.2.0.2.0.6.0-61 or above
-- Hive: 0.12.0.2.0.6.0-61 or above
-- HBase: 0.96.0.2.0.6.0-61-hadoop2
-
-
-_Tested with Hortonworks HDP 2.1.3 and Cloudera Quickstart VM 5.1._
-
-#### Misc ####
-- Command hadoop, hive, hbase is workable on your hadoop cluster
-- JDK Runtime: JDK7 (OpenJDK or Oracle JDK)
-- Maven
-- Git
-- Tomcat (CATALINA_HOME being set)
-- Npm
-
-#### Before Install ####
-- For the quick tutorial, we assume that your hadoop has priviledges disabled which means any user could run hadoop and hive command
-
-
-Installation
-------------
-It is very easy to install Kylin for exploration/development. There are 3 supported options:
-
-
-1. Sandbox (HDP or CDH sandbox)
-2. Dev Environment (IDE + Sandbox)
-3. Docker container (Docker)
-
-#### Sandbox (On-Hadoop-CLI installation) ####
-
-If you are free to install Kylin on your hadoop CLI machine or Hadoop sandbox, this is the most convenient scenario, for it puts everything in a single machine.
-
-
-
-
-
-For normal users, the application in the above picture means `Kylin Web`, which contains a web interface for cube building, querying and all sorts of management. Kylin Web launches a query engine for querying and a cube build engine for building cubes. These two engines interact with the components in Hadoop CLI, like hive and hbase.
-
-For a hands-on tutorial please visit [On-Hadoop-CLI installation](https://github.com/KylinOLAP/Kylin/wiki/On-Hadoop-CLI-installation).
-
-#### Dev Environment (Off-Hadoop-CLI Installation) ####
-
-This is typically for development environment setup.
-
-Applications does not necessarily mean Kylin Web, it can also be any test cases or java applications running at your local IDE(query engine and cube build engine can be launched programmatically in your code). In this case, it is no longer reasonable to assume that all the Kylin components reside in the same machine as your Hadoop CLI. Fortunately, Kylin still works under such condition with proper settings on your CLI machine.
-
-
-
-For a hands-on tutorial please visit [Off-Hadoop-CLI installation](https://github.com/KylinOLAP/Kylin/wiki/Off-Hadoop-CLI-Installation-(Dev-Env-Setup))
-
-#### Docker Container ####
-With help from [SequenceIQ](http://sequenceiq.com/), there's docker container for Kylin (along with Hadoop, HBase and Hive) available now:[sequenceiq/docker-kylin](https://github.com/sequenceiq/docker-kylin). The only thing you will need to do is to pull the container from the official Docker repository to be up and running in few minutes.
-
-Features:
-
-- Size - Light weight compared to downloading and setting up HDP or CDH sandbox.
-- Cluster support - Supports multi node installation.
-- Fully Automated - No manual steps. One command does it all
-
-For a hands-on tutorial please visit [Kylin Docker installation](https://github.com/KylinOLAP/Kylin/wiki/On-Hadoop-Kylin-installation-using-Docker).
-
+Apache Kylin is an open source Distributed Analytics Engine, contributed by eBay Inc., provides SQL interface and multi-dimensional analysis (OLAP) on Hadoop supporting extremely large datasets.
+For more details, see the website [http://kylin.apache.org](http://kylin.apache.org).
-### Resources ###
+Documentation
+=============
+Please refer to [http://kylin.apache.org/docs/](http://kylin.apache.org/docs/).
-* Web Site: <http://kylin.io>
+Get Help
+============
+The fastest way to get response from our developers is to send email to our mail list <de...@kylin.apache.org>,
+and remember to subscribe our mail list via <de...@kylin.apache.org>
-* Google Group: [Kylin OLAP Group](https://groups.google.com/forum/#!forum/kylin-olap)
+Licnese
+============
+Please refer to [LICENSE](https://github.com/apache/kylin/blob/master/LICENSE) file.
-* Developer Mail: <ky...@googlegroups.com>
-* How To Contribute: See [wiki](https://github.com/KylinOLAP/Kylin/wiki/How-to-Contribute)
-* Presentation: [Kylin Hadoop OLAP Engine v1.0](https://github.com/KylinOLAP/Kylin/blob/master/docs/Kylin_Hadoop_OLAP_Engine_v1.0.pdf?raw=true)
-* Apache Proposal: [Apache Kylin](https://wiki.apache.org/incubator/KylinProposal)
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index e36f434e51..2833b71c45 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -1,3 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
@@ -6,9 +25,9 @@
<name>Kylin:AtopCalcite</name>
<parent>
- <groupId>com.kylinolap</groupId>
+ <groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>0.6.3-SNAPSHOT</version>
+ <version>1.2</version>
</parent>
<properties>
diff --git a/atopcalcite/src/main/java/net/hydromatic/optiq/runtime/SqlFunctions.java b/atopcalcite/src/main/java/net/hydromatic/optiq/runtime/SqlFunctions.java
deleted file mode 100644
index f9ba797d76..0000000000
--- a/atopcalcite/src/main/java/net/hydromatic/optiq/runtime/SqlFunctions.java
+++ /dev/null
@@ -1,1705 +0,0 @@
-/*
- * OVERRIDE POINTS:
- * - divide(BigDecimal,BigDecimal), was `b0.divide(b1)`, now `b0.divide(b1, MathContext.DECIMAL64);`
- */
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package net.hydromatic.optiq.runtime;
-
-import net.hydromatic.avatica.ByteString;
-
-import net.hydromatic.linq4j.Enumerable;
-import net.hydromatic.linq4j.Linq4j;
-import net.hydromatic.linq4j.expressions.Primitive;
-import net.hydromatic.linq4j.function.Deterministic;
-import net.hydromatic.linq4j.function.Function1;
-import net.hydromatic.linq4j.function.NonDeterministic;
-
-import net.hydromatic.optiq.DataContext;
-
-import org.eigenbase.util14.DateTimeUtil;
-
-import java.math.*;
-import java.sql.SQLException;
-import java.sql.Timestamp;
-import java.text.DecimalFormat;
-import java.util.*;
-import java.util.regex.Pattern;
-
-/**
- * Helper methods to implement SQL functions in generated code.
- *
- * <p>Not present: and, or, not (builtin operators are better, because they
- * use lazy evaluation. Implementations do not check for null values; the
- * calling code must do that.</p>
- *
- * <p>Many of the functions do not check for null values. This is intentional.
- * If null arguments are possible, the code-generation framework checks for
- * nulls before calling the functions.</p>
- */
-@SuppressWarnings({"unused", "rawtypes", "unchecked"})
-@Deterministic
-public class SqlFunctions {
- private static final DecimalFormat DOUBLE_FORMAT =
- new DecimalFormat("0.0E0");
-
- /** The julian date of the epoch, 1970-01-01. */
- public static final int EPOCH_JULIAN = 2440588;
-
- private static final TimeZone LOCAL_TZ = TimeZone.getDefault();
-
- private static final Function1<List<Object>, Enumerable<Object>>
- LIST_AS_ENUMERABLE =
- new Function1<List<Object>, Enumerable<Object>>() {
- public Enumerable<Object> apply(List<Object> list) {
- return Linq4j.asEnumerable(list);
- }
- };
-
- private SqlFunctions() {
- }
-
- /** SQL SUBSTRING(string FROM ... FOR ...) function. */
- public static String substring(String s, int from, int for_) {
- return s.substring(from - 1, Math.min(from - 1 + for_, s.length()));
- }
-
- /** SQL SUBSTRING(string FROM ...) function. */
- public static String substring(String s, int from) {
- return s.substring(from - 1);
- }
-
- /** SQL UPPER(string) function. */
- public static String upper(String s) {
- return s.toUpperCase();
- }
-
- /** SQL LOWER(string) function. */
- public static String lower(String s) {
- return s.toLowerCase();
- }
-
- /** SQL INITCAP(string) function. */
- public static String initcap(String s) {
- // Assumes Alpha as [A-Za-z0-9]
- // white space is treated as everything else.
- final int len = s.length();
- boolean start = true;
- final StringBuilder newS = new StringBuilder();
-
- for (int i = 0; i < len; i++) {
- char curCh = s.charAt(i);
- final int c = (int) curCh;
- if (start) { // curCh is whitespace or first character of word.
- if (c > 47 && c < 58) { // 0-9
- start = false;
- } else if (c > 64 && c < 91) { // A-Z
- start = false;
- } else if (c > 96 && c < 123) { // a-z
- start = false;
- curCh = (char) (c - 32); // Uppercase this character
- }
- // else {} whitespace
- } else { // Inside of a word or white space after end of word.
- if (c > 47 && c < 58) { // 0-9
- // noop
- } else if (c > 64 && c < 91) { // A-Z
- curCh = (char) (c + 32); // Lowercase this character
- } else if (c > 96 && c < 123) { // a-z
- // noop
- } else { // whitespace
- start = true;
- }
- }
- newS.append(curCh);
- } // for each character in s
- return newS.toString();
- }
-
- /** SQL CHARACTER_LENGTH(string) function. */
- public static int charLength(String s) {
- return s.length();
- }
-
- /** SQL {@code string || string} operator. */
- public static String concat(String s0, String s1) {
- return s0 + s1;
- }
-
- /** SQL {@code binary || binary} operator. */
- public static ByteString concat(ByteString s0, ByteString s1) {
- return s0.concat(s1);
- }
-
- /** SQL {@code RTRIM} function applied to string. */
- public static String rtrim(String s) {
- return trim_(s, false, true, ' ');
- }
-
- /** SQL {@code LTRIM} function. */
- public static String ltrim(String s) {
- return trim_(s, true, false, ' ');
- }
-
- /** SQL {@code TRIM(... seek FROM s)} function. */
- public static String trim(boolean leading, boolean trailing, String seek,
- String s) {
- return trim_(s, leading, trailing, seek.charAt(0));
- }
-
- /** SQL {@code TRIM} function. */
- private static String trim_(String s, boolean left, boolean right, char c) {
- int j = s.length();
- if (right) {
- for (;;) {
- if (j == 0) {
- return "";
- }
- if (s.charAt(j - 1) != c) {
- break;
- }
- --j;
- }
- }
- int i = 0;
- if (left) {
- for (;;) {
- if (i == j) {
- return "";
- }
- if (s.charAt(i) != c) {
- break;
- }
- ++i;
- }
- }
- return s.substring(i, j);
- }
-
- /** SQL {@code TRIM} function applied to binary string. */
- public static ByteString trim(ByteString s) {
- return trim_(s, true, true);
- }
-
- /** Helper for CAST. */
- public static ByteString rtrim(ByteString s) {
- return trim_(s, false, true);
- }
-
- /** SQL {@code TRIM} function applied to binary string. */
- private static ByteString trim_(ByteString s, boolean left, boolean right) {
- int j = s.length();
- if (right) {
- for (;;) {
- if (j == 0) {
- return ByteString.EMPTY;
- }
- if (s.byteAt(j - 1) != 0) {
- break;
- }
- --j;
- }
- }
- int i = 0;
- if (left) {
- for (;;) {
- if (i == j) {
- return ByteString.EMPTY;
- }
- if (s.byteAt(i) != 0) {
- break;
- }
- ++i;
- }
- }
- return s.substring(i, j);
- }
-
- /** SQL {@code OVERLAY} function. */
- public static String overlay(String s, String r, int start) {
- if (s == null || r == null) {
- return null;
- }
- return s.substring(0, start - 1)
- + r
- + s.substring(start - 1 + r.length());
- }
-
- /** SQL {@code OVERLAY} function. */
- public static String overlay(String s, String r, int start, int length) {
- if (s == null || r == null) {
- return null;
- }
- return s.substring(0, start - 1)
- + r
- + s.substring(start - 1 + length);
- }
-
- /** SQL {@code OVERLAY} function applied to binary strings. */
- public static ByteString overlay(ByteString s, ByteString r, int start) {
- if (s == null || r == null) {
- return null;
- }
- return s.substring(0, start - 1)
- .concat(r)
- .concat(s.substring(start - 1 + r.length()));
- }
-
- /** SQL {@code OVERLAY} function applied to binary strings. */
- public static ByteString overlay(ByteString s, ByteString r, int start,
- int length) {
- if (s == null || r == null) {
- return null;
- }
- return s.substring(0, start - 1)
- .concat(r)
- .concat(s.substring(start - 1 + length));
- }
-
- /** SQL {@code LIKE} function. */
- public static boolean like(String s, String pattern) {
- final String regex = Like.sqlToRegexLike(pattern, null);
- return Pattern.matches(regex, s);
- }
-
- /** SQL {@code LIKE} function with escape. */
- public static boolean like(String s, String pattern, String escape) {
- final String regex = Like.sqlToRegexLike(pattern, escape);
- return Pattern.matches(regex, s);
- }
-
- /** SQL {@code SIMILAR} function. */
- public static boolean similar(String s, String pattern) {
- final String regex = Like.sqlToRegexSimilar(pattern, null);
- return Pattern.matches(regex, s);
- }
-
- /** SQL {@code SIMILAR} function with escape. */
- public static boolean similar(String s, String pattern, String escape) {
- final String regex = Like.sqlToRegexSimilar(pattern, escape);
- return Pattern.matches(regex, s);
- }
-
- // =
-
- /** SQL = operator applied to Object values (including String; neither
- * side may be null). */
- public static boolean eq(Object b0, Object b1) {
- return b0.equals(b1);
- }
-
- /** SQL = operator applied to BigDecimal values (neither may be null). */
- public static boolean eq(BigDecimal b0, BigDecimal b1) {
- return b0.stripTrailingZeros().equals(b1.stripTrailingZeros());
- }
-
- // <>
-
- /** SQL <> operator applied to Object values (including String;
- * neither side may be null). */
- public static boolean ne(Object b0, Object b1) {
- return !b0.equals(b1);
- }
-
- /** SQL <> operator applied to BigDecimal values. */
- public static boolean ne(BigDecimal b0, BigDecimal b1) {
- return b0.compareTo(b1) != 0;
- }
-
- // <
-
- /** SQL < operator applied to boolean values. */
- public static boolean lt(boolean b0, boolean b1) {
- return compare(b0, b1) < 0;
- }
-
- /** SQL < operator applied to String values. */
- public static boolean lt(String b0, String b1) {
- return b0.compareTo(b1) < 0;
- }
-
- /** SQL < operator applied to ByteString values. */
- public static boolean lt(ByteString b0, ByteString b1) {
- return b0.compareTo(b1) < 0;
- }
-
- /** SQL < operator applied to BigDecimal values. */
- public static boolean lt(BigDecimal b0, BigDecimal b1) {
- return b0.compareTo(b1) < 0;
- }
-
- // <=
-
- /** SQL ≤ operator applied to boolean values. */
- public static boolean le(boolean b0, boolean b1) {
- return compare(b0, b1) <= 0;
- }
-
- /** SQL ≤ operator applied to String values. */
- public static boolean le(String b0, String b1) {
- return b0.compareTo(b1) <= 0;
- }
-
- /** SQL ≤ operator applied to ByteString values. */
- public static boolean le(ByteString b0, ByteString b1) {
- return b0.compareTo(b1) <= 0;
- }
-
- /** SQL ≤ operator applied to BigDecimal values. */
- public static boolean le(BigDecimal b0, BigDecimal b1) {
- return b0.compareTo(b1) <= 0;
- }
-
- // >
-
- /** SQL > operator applied to boolean values. */
- public static boolean gt(boolean b0, boolean b1) {
- return compare(b0, b1) > 0;
- }
-
- /** SQL > operator applied to String values. */
- public static boolean gt(String b0, String b1) {
- return b0.compareTo(b1) > 0;
- }
-
- /** SQL > operator applied to ByteString values. */
- public static boolean gt(ByteString b0, ByteString b1) {
- return b0.compareTo(b1) > 0;
- }
-
- /** SQL > operator applied to BigDecimal values. */
- public static boolean gt(BigDecimal b0, BigDecimal b1) {
- return b0.compareTo(b1) > 0;
- }
-
- // >=
-
- /** SQL ≥ operator applied to boolean values. */
- public static boolean ge(boolean b0, boolean b1) {
- return compare(b0, b1) >= 0;
- }
-
- /** SQL ≥ operator applied to String values. */
- public static boolean ge(String b0, String b1) {
- return b0.compareTo(b1) >= 0;
- }
-
- /** SQL ≥ operator applied to ByteString values. */
- public static boolean ge(ByteString b0, ByteString b1) {
- return b0.compareTo(b1) >= 0;
- }
-
- /** SQL ≥ operator applied to BigDecimal values. */
- public static boolean ge(BigDecimal b0, BigDecimal b1) {
- return b0.compareTo(b1) >= 0;
- }
-
- // +
-
- /** SQL <code>+</code> operator applied to int values. */
- public static int plus(int b0, int b1) {
- return b0 + b1;
- }
-
- /** SQL <code>+</code> operator applied to int values; left side may be
- * null. */
- public static Integer plus(Integer b0, int b1) {
- return b0 == null ? null : (b0 + b1);
- }
-
- /** SQL <code>+</code> operator applied to int values; right side may be
- * null. */
- public static Integer plus(int b0, Integer b1) {
- return b1 == null ? null : (b0 + b1);
- }
-
- /** SQL <code>+</code> operator applied to nullable int values. */
- public static Integer plus(Integer b0, Integer b1) {
- return (b0 == null || b1 == null) ? null : (b0 + b1);
- }
-
- /** SQL <code>+</code> operator applied to nullable long and int values. */
- public static Long plus(Long b0, Integer b1) {
- return (b0 == null || b1 == null)
- ? null
- : (b0.longValue() + b1.longValue());
- }
-
- /** SQL <code>+</code> operator applied to nullable int and long values. */
- public static Long plus(Integer b0, Long b1) {
- return (b0 == null || b1 == null)
- ? null
- : (b0.longValue() + b1.longValue());
- }
-
- /** SQL <code>+</code> operator applied to BigDecimal values. */
- public static BigDecimal plus(BigDecimal b0, BigDecimal b1) {
- return (b0 == null || b1 == null) ? null : b0.add(b1);
- }
-
- // -
-
- /** SQL <code>-</code> operator applied to int values. */
- public static int minus(int b0, int b1) {
- return b0 - b1;
- }
-
- /** SQL <code>-</code> operator applied to int values; left side may be
- * null. */
- public static Integer minus(Integer b0, int b1) {
- return b0 == null ? null : (b0 - b1);
- }
-
- /** SQL <code>-</code> operator applied to int values; right side may be
- * null. */
- public static Integer minus(int b0, Integer b1) {
- return b1 == null ? null : (b0 - b1);
- }
-
- /** SQL <code>-</code> operator applied to nullable int values. */
- public static Integer minus(Integer b0, Integer b1) {
- return (b0 == null || b1 == null) ? null : (b0 - b1);
- }
-
- /** SQL <code>-</code> operator applied to nullable long and int values. */
- public static Long minus(Long b0, Integer b1) {
- return (b0 == null || b1 == null)
- ? null
- : (b0.longValue() - b1.longValue());
- }
-
- /** SQL <code>-</code> operator applied to nullable int and long values. */
- public static Long minus(Integer b0, Long b1) {
- return (b0 == null || b1 == null)
- ? null
- : (b0.longValue() - b1.longValue());
- }
-
- /** SQL <code>-</code> operator applied to BigDecimal values. */
- public static BigDecimal minus(BigDecimal b0, BigDecimal b1) {
- return (b0 == null || b1 == null) ? null : b0.subtract(b1);
- }
-
- // /
-
- /** SQL <code>/</code> operator applied to int values. */
- public static int divide(int b0, int b1) {
- return b0 / b1;
- }
-
- /** SQL <code>/</code> operator applied to int values; left side may be
- * null. */
- public static Integer divide(Integer b0, int b1) {
- return b0 == null ? null : (b0 / b1);
- }
-
- /** SQL <code>/</code> operator applied to int values; right side may be
- * null. */
- public static Integer divide(int b0, Integer b1) {
- return b1 == null ? null : (b0 / b1);
- }
-
- /** SQL <code>/</code> operator applied to nullable int values. */
- public static Integer divide(Integer b0, Integer b1) {
- return (b0 == null || b1 == null) ? null : (b0 / b1);
- }
-
- /** SQL <code>/</code> operator applied to nullable long and int values. */
- public static Long divide(Long b0, Integer b1) {
- return (b0 == null || b1 == null)
- ? null
- : (b0.longValue() / b1.longValue());
- }
-
- /** SQL <code>/</code> operator applied to nullable int and long values. */
- public static Long divide(Integer b0, Long b1) {
- return (b0 == null || b1 == null)
- ? null
- : (b0.longValue() / b1.longValue());
- }
-
- /** SQL <code>/</code> operator applied to BigDecimal values. */
- public static BigDecimal divide(BigDecimal b0, BigDecimal b1) {
- // OVERRIDE POINT
- return (b0 == null || b1 == null) ? null : b0.divide(b1, MathContext.DECIMAL64);
- }
-
- // *
-
- /** SQL <code>*</code> operator applied to int values. */
- public static int multiply(int b0, int b1) {
- return b0 * b1;
- }
-
- /** SQL <code>*</code> operator applied to int values; left side may be
- * null. */
- public static Integer multiply(Integer b0, int b1) {
- return b0 == null ? null : (b0 * b1);
- }
-
- /** SQL <code>*</code> operator applied to int values; right side may be
- * null. */
- public static Integer multiply(int b0, Integer b1) {
- return b1 == null ? null : (b0 * b1);
- }
-
- /** SQL <code>*</code> operator applied to nullable int values. */
- public static Integer multiply(Integer b0, Integer b1) {
- return (b0 == null || b1 == null) ? null : (b0 * b1);
- }
-
- /** SQL <code>*</code> operator applied to nullable long and int values. */
- public static Long multiply(Long b0, Integer b1) {
- return (b0 == null || b1 == null)
- ? null
- : (b0.longValue() * b1.longValue());
- }
-
- /** SQL <code>*</code> operator applied to nullable int and long values. */
- public static Long multiply(Integer b0, Long b1) {
- return (b0 == null || b1 == null)
- ? null
- : (b0.longValue() * b1.longValue());
- }
-
- /** SQL <code>*</code> operator applied to BigDecimal values. */
- public static BigDecimal multiply(BigDecimal b0, BigDecimal b1) {
- return (b0 == null || b1 == null) ? null : b0.multiply(b1);
- }
-
- // EXP
-
- /** SQL <code>EXP</code> operator applied to double values. */
- public static double exp(double b0) {
- return Math.exp(b0);
- }
-
- public static double exp(long b0) {
- return Math.exp(b0);
- }
-
- // POWER
-
- /** SQL <code>POWER</code> operator applied to double values. */
- public static double power(double b0, double b1) {
- return Math.pow(b0, b1);
- }
-
- public static double power(long b0, long b1) {
- return Math.pow(b0, b1);
- }
-
- public static double power(long b0, BigDecimal b1) {
- return Math.pow(b0, b1.doubleValue());
- }
-
- // LN
-
- /** SQL {@code LN(number)} function applied to double values. */
- public static double ln(double d) {
- return Math.log(d);
- }
-
- /** SQL {@code LN(number)} function applied to long values. */
- public static double ln(long b0) {
- return Math.log(b0);
- }
-
- /** SQL {@code LN(number)} function applied to BigDecimal values. */
- public static double ln(BigDecimal d) {
- return Math.log(d.doubleValue());
- }
-
- // LOG10
-
- /** SQL <code>LOG10(numeric)</code> operator applied to double values. */
- public static double log10(double b0) {
- return Math.log10(b0);
- }
-
- /** SQL {@code LOG10(number)} function applied to long values. */
- public static double log10(long b0) {
- return Math.log10(b0);
- }
-
- /** SQL {@code LOG10(number)} function applied to BigDecimal values. */
- public static double log10(BigDecimal d) {
- return Math.log10(d.doubleValue());
- }
-
- // MOD
-
- /** SQL <code>MOD</code> operator applied to byte values. */
- public static byte mod(byte b0, byte b1) {
- return (byte) (b0 % b1);
- }
-
- /** SQL <code>MOD</code> operator applied to short values. */
- public static short mod(short b0, short b1) {
- return (short) (b0 % b1);
- }
-
- /** SQL <code>MOD</code> operator applied to int values. */
- public static int mod(int b0, int b1) {
- return b0 % b1;
- }
-
- /** SQL <code>MOD</code> operator applied to long values. */
- public static long mod(long b0, long b1) {
- return b0 % b1;
- }
-
- // temporary
- public static BigDecimal mod(BigDecimal b0, int b1) {
- return mod(b0, BigDecimal.valueOf(b1));
- }
-
- // temporary
- public static int mod(int b0, BigDecimal b1) {
- return mod(b0, b1.intValue());
- }
-
- public static BigDecimal mod(BigDecimal b0, BigDecimal b1) {
- final BigDecimal[] bigDecimals = b0.divideAndRemainder(b1);
- return bigDecimals[1];
- }
-
- // ABS
-
- /** SQL <code>ABS</code> operator applied to byte values. */
- public static byte abs(byte b0) {
- return (byte) Math.abs(b0);
- }
-
- /** SQL <code>ABS</code> operator applied to short values. */
- public static short abs(short b0) {
- return (short) Math.abs(b0);
- }
-
- /** SQL <code>ABS</code> operator applied to int values. */
- public static int abs(int b0) {
- return Math.abs(b0);
- }
-
- /** SQL <code>ABS</code> operator applied to long values. */
- public static long abs(long b0) {
- return Math.abs(b0);
- }
-
- /** SQL <code>ABS</code> operator applied to float values. */
- public static float abs(float b0) {
- return Math.abs(b0);
- }
-
- /** SQL <code>ABS</code> operator applied to double values. */
- public static double abs(double b0) {
- return Math.abs(b0);
- }
-
- /** SQL <code>ABS</code> operator applied to BigDecimal values. */
- public static BigDecimal abs(BigDecimal b0) {
- return b0.abs();
- }
-
- // Helpers
-
- /** Helper for implementing MIN. Somewhat similar to LEAST operator. */
- public static <T extends Comparable<T>> T lesser(T b0, T b1) {
- return b0 == null || b0.compareTo(b1) > 0 ? b1 : b0;
- }
-
- /** LEAST operator. */
- public static <T extends Comparable<T>> T least(T b0, T b1) {
- return b0 == null || b1 != null && b0.compareTo(b1) > 0 ? b1 : b0;
- }
-
- public static boolean greater(boolean b0, boolean b1) {
- return b0 || b1;
- }
-
- public static boolean lesser(boolean b0, boolean b1) {
- return b0 && b1;
- }
-
- public static byte greater(byte b0, byte b1) {
- return b0 > b1 ? b0 : b1;
- }
-
- public static byte lesser(byte b0, byte b1) {
- return b0 > b1 ? b1 : b0;
- }
-
- public static char greater(char b0, char b1) {
- return b0 > b1 ? b0 : b1;
- }
-
- public static char lesser(char b0, char b1) {
- return b0 > b1 ? b1 : b0;
- }
-
- public static short greater(short b0, short b1) {
- return b0 > b1 ? b0 : b1;
- }
-
- public static short lesser(short b0, short b1) {
- return b0 > b1 ? b1 : b0;
- }
-
- public static int greater(int b0, int b1) {
- return b0 > b1 ? b0 : b1;
- }
-
- public static int lesser(int b0, int b1) {
- return b0 > b1 ? b1 : b0;
- }
-
- public static long greater(long b0, long b1) {
- return b0 > b1 ? b0 : b1;
- }
-
- public static long lesser(long b0, long b1) {
- return b0 > b1 ? b1 : b0;
- }
-
- public static float greater(float b0, float b1) {
- return b0 > b1 ? b0 : b1;
- }
-
- public static float lesser(float b0, float b1) {
- return b0 > b1 ? b1 : b0;
- }
-
- public static double greater(double b0, double b1) {
- return b0 > b1 ? b0 : b1;
- }
-
- public static double lesser(double b0, double b1) {
- return b0 > b1 ? b1 : b0;
- }
-
- /** Helper for implementing MAX. Somewhat similar to GREATEST operator. */
- public static <T extends Comparable<T>> T greater(T b0, T b1) {
- return b0 == null || b0.compareTo(b1) < 0 ? b1 : b0;
- }
-
- /** GREATEST operator. */
- public static <T extends Comparable<T>> T greatest(T b0, T b1) {
- return b0 == null || b1 != null && b0.compareTo(b1) < 0 ? b1 : b0;
- }
-
- /** Boolean comparison. */
- public static int compare(boolean x, boolean y) {
- return x == y ? 0 : x ? 1 : -1;
- }
-
- /** CAST(FLOAT AS VARCHAR). */
- public static String toString(float x) {
- if (x == 0) {
- return "0E0";
- }
- BigDecimal bigDecimal =
- new BigDecimal(x, MathContext.DECIMAL32).stripTrailingZeros();
- final String s = bigDecimal.toString();
- return s.replaceAll("0*E", "E").replace("E+", "E");
- }
-
- /** CAST(DOUBLE AS VARCHAR). */
- public static String toString(double x) {
- if (x == 0) {
- return "0E0";
- }
- BigDecimal bigDecimal =
- new BigDecimal(x, MathContext.DECIMAL64).stripTrailingZeros();
- final String s = bigDecimal.toString();
- return s.replaceAll("0*E", "E").replace("E+", "E");
- }
-
- /** CAST(DECIMAL AS VARCHAR). */
- public static String toString(BigDecimal x) {
- final String s = x.toString();
- if (s.startsWith("0")) {
- // we want ".1" not "0.1"
- return s.substring(1);
- } else if (s.startsWith("-0")) {
- // we want "-.1" not "-0.1"
- return "-" + s.substring(2);
- } else {
- return s;
- }
- }
-
- /** CAST(BOOLEAN AS VARCHAR). */
- public static String toString(boolean x) {
- // Boolean.toString returns lower case -- no good.
- return x ? "TRUE" : "FALSE";
- }
-
- @NonDeterministic
- private static Object cannotConvert(Object o, Class toType) {
- throw new RuntimeException("Cannot convert " + o + " to " + toType);
- }
-
- /** CAST(VARCHAR AS BOOLEAN). */
- public static boolean toBoolean(String s) {
- s = trim_(s, true, true, ' ');
- if (s.equalsIgnoreCase("TRUE")) {
- return true;
- } else if (s.equalsIgnoreCase("FALSE")) {
- return false;
- } else {
- throw new RuntimeException("Invalid character for cast");
- }
- }
-
- public static boolean toBoolean(Number number) {
- return !number.equals(0);
- }
-
- public static boolean toBoolean(Object o) {
- return o instanceof Boolean ? (Boolean) o
- : o instanceof Number ? toBoolean((Number) o)
- : o instanceof String ? toBoolean((String) o)
- : (Boolean) cannotConvert(o, boolean.class);
- }
-
- // Don't need parseByte etc. - Byte.parseByte is sufficient.
-
- public static byte toByte(Object o) {
- return o instanceof Byte ? (Byte) o
- : o instanceof Number ? toByte((Number) o)
- : Byte.parseByte(o.toString());
- }
-
- public static byte toByte(Number number) {
- return number.byteValue();
- }
-
- public static char toChar(String s) {
- return s.charAt(0);
- }
-
- public static Character toCharBoxed(String s) {
- return s.charAt(0);
- }
-
- public static short toShort(String s) {
- return Short.parseShort(s.trim());
- }
-
- public static short toShort(Number number) {
- return number.shortValue();
- }
-
- public static short toShort(Object o) {
- return o instanceof Short ? (Short) o
- : o instanceof Number ? toShort((Number) o)
- : o instanceof String ? toShort((String) o)
- : (Short) cannotConvert(o, short.class);
- }
-
- public static int toInt(java.util.Date v) {
- return toInt(v, LOCAL_TZ);
- }
-
- public static int toInt(java.util.Date v, TimeZone timeZone) {
- return (int) (toLong(v, timeZone) / DateTimeUtil.MILLIS_PER_DAY);
- }
-
- public static Integer toIntOptional(java.util.Date v) {
- return v == null ? null : toInt(v);
- }
-
- public static Integer toIntOptional(java.util.Date v, TimeZone timeZone) {
- return v == null
- ? null
- : toInt(v, timeZone);
- }
-
- public static long toLong(Date v) {
- return toLong(v, LOCAL_TZ);
- }
-
- public static int toInt(java.sql.Time v) {
- return (int) (toLong(v) % DateTimeUtil.MILLIS_PER_DAY);
- }
-
- public static Integer toIntOptional(java.sql.Time v) {
- return v == null ? null : toInt(v);
- }
-
- public static int toInt(String s) {
- return Integer.parseInt(s.trim());
- }
-
- public static int toInt(Number number) {
- return number.intValue();
- }
-
- public static int toInt(Object o) {
- return o instanceof Integer ? (Integer) o
- : o instanceof Number ? toInt((Number) o)
- : o instanceof String ? toInt((String) o)
- : (Integer) cannotConvert(o, int.class);
- }
-
- public static long toLong(Timestamp v) {
- return toLong(v, LOCAL_TZ);
- }
-
- // mainly intended for java.sql.Timestamp but works for other dates also
- public static long toLong(java.util.Date v, TimeZone timeZone) {
- final long time = v.getTime();
- return time + timeZone.getOffset(time);
- }
-
- // mainly intended for java.sql.Timestamp but works for other dates also
- public static Long toLongOptional(java.util.Date v) {
- return v == null ? null : toLong(v, LOCAL_TZ);
- }
-
- public static Long toLongOptional(Timestamp v, TimeZone timeZone) {
- if (v == null) {
- return null;
- }
- return toLong(v, LOCAL_TZ);
- }
-
- public static long toLong(String s) {
- if (s.startsWith("199") && s.contains(":")) {
- return Timestamp.valueOf(s).getTime();
- }
- return Long.parseLong(s.trim());
- }
-
- public static long toLong(Number number) {
- return number.longValue();
- }
-
- public static long toLong(Object o) {
- return o instanceof Long ? (Long) o
- : o instanceof Number ? toLong((Number) o)
- : o instanceof String ? toLong((String) o)
- : (Long) cannotConvert(o, long.class);
- }
-
- public static float toFloat(String s) {
- return Float.parseFloat(s.trim());
- }
-
- public static float toFloat(Number number) {
- return number.floatValue();
- }
-
- public static float toFloat(Object o) {
- return o instanceof Float ? (Float) o
- : o instanceof Number ? toFloat((Number) o)
- : o instanceof String ? toFloat((String) o)
- : (Float) cannotConvert(o, float.class);
- }
-
- public static double toDouble(String s) {
- return Double.parseDouble(s.trim());
- }
-
- public static double toDouble(Number number) {
- return number.doubleValue();
- }
-
- public static double toDouble(Object o) {
- return o instanceof Double ? (Double) o
- : o instanceof Number ? toDouble((Number) o)
- : o instanceof String ? toDouble((String) o)
- : (Double) cannotConvert(o, double.class);
- }
-
- public static BigDecimal toBigDecimal(String s) {
- return new BigDecimal(s.trim());
- }
-
- public static BigDecimal toBigDecimal(Number number) {
- // There are some values of "long" that cannot be represented as "double".
- // Not so "int". If it isn't a long, go straight to double.
- return number instanceof BigDecimal ? (BigDecimal) number
- : number instanceof BigInteger ? new BigDecimal((BigInteger) number)
- : number instanceof Long ? new BigDecimal(number.longValue())
- : new BigDecimal(number.doubleValue());
- }
-
- public static BigDecimal toBigDecimal(Object o) {
- return o instanceof Number ? toBigDecimal((Number) o)
- : toBigDecimal(o.toString());
- }
-
- // Don't need shortValueOf etc. - Short.valueOf is sufficient.
-
- /** Helper for CAST(... AS VARCHAR(maxLength)). */
- public static String truncate(String s, int maxLength) {
- return s == null ? null
- : s.length() > maxLength ? s.substring(0, maxLength)
- : s;
- }
-
- /** Helper for CAST(... AS VARBINARY(maxLength)). */
- public static ByteString truncate(ByteString s, int maxLength) {
- return s == null ? null
- : s.length() > maxLength ? s.substring(0, maxLength)
- : s;
- }
-
- /** SQL {@code POSITION(seek IN string)} function. */
- public static int position(String seek, String s) {
- return s.indexOf(seek) + 1;
- }
-
- /** SQL {@code POSITION(seek IN string)} function. */
- public static int position(ByteString seek, ByteString s) {
- return s.indexOf(seek) + 1;
- }
-
- /** Cheap, unsafe, long power. power(2, 3) returns 8. */
- public static long powerX(long a, long b) {
- long x = 1;
- while (b > 0) {
- x *= a;
- --b;
- }
- return x;
- }
-
- /** Helper for rounding. Truncate(12345, 1000) returns 12000. */
- public static long round(long v, long x) {
- return truncate(v + x / 2, x);
- }
-
- /** Helper for rounding. Truncate(12345, 1000) returns 12000. */
- public static long truncate(long v, long x) {
- long remainder = v % x;
- if (remainder < 0) {
- remainder += x;
- }
- return v - remainder;
- }
-
- /** Helper for rounding. Truncate(12345, 1000) returns 12000. */
- public static int round(int v, int x) {
- return truncate(v + x / 2, x);
- }
-
- /** Helper for rounding. Truncate(12345, 1000) returns 12000. */
- public static int truncate(int v, int x) {
- int remainder = v % x;
- if (remainder < 0) {
- remainder += x;
- }
- return v - remainder;
- }
-
- /** Helper for CAST({timestamp} AS VARCHAR(n)). */
- public static String unixTimestampToString(long timestamp) {
- final StringBuilder buf = new StringBuilder(17);
- int date = (int) (timestamp / DateTimeUtil.MILLIS_PER_DAY);
- int time = (int) (timestamp % DateTimeUtil.MILLIS_PER_DAY);
- if (time < 0) {
- --date;
- time += DateTimeUtil.MILLIS_PER_DAY;
- }
- unixDateToString(buf, date);
- buf.append(' ');
- unixTimeToString(buf, time);
- return buf.toString();
- }
-
- /** Helper for CAST({timestamp} AS VARCHAR(n)). */
- public static String unixTimeToString(int time) {
- final StringBuilder buf = new StringBuilder(8);
- unixTimeToString(buf, time);
- return buf.toString();
- }
-
- private static void unixTimeToString(StringBuilder buf, int time) {
- int h = time / 3600000;
- int time2 = time % 3600000;
- int m = time2 / 60000;
- int time3 = time2 % 60000;
- int s = time3 / 1000;
- int ms = time3 % 1000;
- int2(buf, h);
- buf.append(':');
- int2(buf, m);
- buf.append(':');
- int2(buf, s);
- }
-
- /** SQL {@code CURRENT_TIMESTAMP} function. */
- @NonDeterministic
- public static long currentTimestamp(DataContext root) {
- // Cast required for JDK 1.6.
- return (Long) DataContext.Variable.CURRENT_TIMESTAMP.get(root);
- }
-
- /** SQL {@code CURRENT_TIME} function. */
- @NonDeterministic
- public static int currentTime(DataContext root) {
- int time = (int) (currentTimestamp(root) % DateTimeUtil.MILLIS_PER_DAY);
- if (time < 0) {
- time += DateTimeUtil.MILLIS_PER_DAY;
- }
- return time;
- }
-
- /** SQL {@code CURRENT_DATE} function. */
- @NonDeterministic
- public static int currentDate(DataContext root) {
- final long timestamp = currentTimestamp(root);
- int date = (int) (timestamp / DateTimeUtil.MILLIS_PER_DAY);
- final int time = (int) (timestamp % DateTimeUtil.MILLIS_PER_DAY);
- if (time < 0) {
- --date;
- }
- return date;
- }
-
- /** SQL {@code LOCAL_TIMESTAMP} function. */
- @NonDeterministic
- public static long localTimestamp(DataContext root) {
- // Cast required for JDK 1.6.
- return (Long) DataContext.Variable.LOCAL_TIMESTAMP.get(root);
- }
-
- /** SQL {@code LOCAL_TIME} function. */
- @NonDeterministic
- public static int localTime(DataContext root) {
- return (int) (localTimestamp(root) % DateTimeUtil.MILLIS_PER_DAY);
- }
-
- private static void int2(StringBuilder buf, int i) {
- buf.append((char) ('0' + (i / 10) % 10));
- buf.append((char) ('0' + i % 10));
- }
-
- private static void int4(StringBuilder buf, int i) {
- buf.append((char) ('0' + (i / 1000) % 10));
- buf.append((char) ('0' + (i / 100) % 10));
- buf.append((char) ('0' + (i / 10) % 10));
- buf.append((char) ('0' + i % 10));
- }
-
- public static int dateStringToUnixDate(String s) {
- int hyphen1 = s.indexOf('-');
- int y;
- int m;
- int d;
- if (hyphen1 < 0) {
- y = Integer.parseInt(s.trim());
- m = 1;
- d = 1;
- } else {
- y = Integer.parseInt(s.substring(0, hyphen1).trim());
- final int hyphen2 = s.indexOf('-', hyphen1 + 1);
- if (hyphen2 < 0) {
- m = Integer.parseInt(s.substring(hyphen1 + 1).trim());
- d = 1;
- } else {
- m = Integer.parseInt(s.substring(hyphen1 + 1, hyphen2).trim());
- d = Integer.parseInt(s.substring(hyphen2 + 1).trim());
- }
- }
- return ymdToUnixDate(y, m, d);
- }
-
- public static int timeStringToUnixDate(String v) {
- return timeStringToUnixDate(v, 0);
- }
-
- public static int timeStringToUnixDate(String v, int start) {
- final int colon1 = v.indexOf(':', start);
- int hour;
- int minute;
- int second;
- int milli;
- if (colon1 < 0) {
- hour = Integer.parseInt(v.trim());
- minute = 1;
- second = 1;
- milli = 0;
- } else {
- hour = Integer.parseInt(v.substring(start, colon1).trim());
- final int colon2 = v.indexOf(':', colon1 + 1);
- if (colon2 < 0) {
- minute = Integer.parseInt(v.substring(colon1 + 1).trim());
- second = 1;
- milli = 0;
- } else {
- minute = Integer.parseInt(v.substring(colon1 + 1, colon2).trim());
- int dot = v.indexOf('.', colon2);
- if (dot < 0) {
- second = Integer.parseInt(v.substring(colon2 + 1).trim());
- milli = 0;
- } else {
- second = Integer.parseInt(v.substring(colon2 + 1, dot).trim());
- milli = Integer.parseInt(v.substring(dot + 1).trim());
- }
- }
- }
- return hour * (int) DateTimeUtil.MILLIS_PER_HOUR
- + minute * (int) DateTimeUtil.MILLIS_PER_MINUTE
- + second * (int) DateTimeUtil.MILLIS_PER_SECOND
- + milli;
- }
-
- public static long timestampStringToUnixDate(String s) {
- final long d;
- final long t;
- s = s.trim();
- int space = s.indexOf(' ');
- if (space >= 0) {
- d = dateStringToUnixDate(s.substring(0, space));
- t = timeStringToUnixDate(s, space + 1);
- } else {
- d = dateStringToUnixDate(s);
- t = 0;
- }
- return d * DateTimeUtil.MILLIS_PER_DAY + t;
- }
-
- /** Helper for CAST({date} AS VARCHAR(n)). */
- public static String unixDateToString(int date) {
- final StringBuilder buf = new StringBuilder(10);
- unixDateToString(buf, date);
- return buf.toString();
- }
-
- private static void unixDateToString(StringBuilder buf, int date) {
- julianToString(buf, date + EPOCH_JULIAN);
- }
-
- private static void julianToString(StringBuilder buf, int julian) {
- // this shifts the epoch back to astronomical year -4800 instead of the
- // start of the Christian era in year AD 1 of the proleptic Gregorian
- // calendar.
- int j = julian + 32044;
- int g = j / 146097;
- int dg = j % 146097;
- int c = (dg / 36524 + 1) * 3 / 4;
- int dc = dg - c * 36524;
- int b = dc / 1461;
- int db = dc % 1461;
- int a = (db / 365 + 1) * 3 / 4;
- int da = db - a * 365;
-
- // integer number of full years elapsed since March 1, 4801 BC
- int y = g * 400 + c * 100 + b * 4 + a;
- // integer number of full months elapsed since the last March 1
- int m = (da * 5 + 308) / 153 - 2;
- // number of days elapsed since day 1 of the month
- int d = da - (m + 4) * 153 / 5 + 122;
- int year = y - 4800 + (m + 2) / 12;
- int month = (m + 2) % 12 + 1;
- int day = d + 1;
- int4(buf, year);
- buf.append('-');
- int2(buf, month);
- buf.append('-');
- int2(buf, day);
- }
-
- public static long unixDateExtract(TimeUnitRange range, long date) {
- return julianExtract(range, (int) date + EPOCH_JULIAN);
- }
-
- private static int julianExtract(TimeUnitRange range, int julian) {
- // this shifts the epoch back to astronomical year -4800 instead of the
- // start of the Christian era in year AD 1 of the proleptic Gregorian
- // calendar.
- int j = julian + 32044;
- int g = j / 146097;
- int dg = j % 146097;
- int c = (dg / 36524 + 1) * 3 / 4;
- int dc = dg - c * 36524;
- int b = dc / 1461;
- int db = dc % 1461;
- int a = (db / 365 + 1) * 3 / 4;
- int da = db - a * 365;
-
- // integer number of full years elapsed since March 1, 4801 BC
- int y = g * 400 + c * 100 + b * 4 + a;
- // integer number of full months elapsed since the last March 1
- int m = (da * 5 + 308) / 153 - 2;
- // number of days elapsed since day 1 of the month
- int d = da - (m + 4) * 153 / 5 + 122;
- int year = y - 4800 + (m + 2) / 12;
- int month = (m + 2) % 12 + 1;
- int day = d + 1;
- switch (range) {
- case YEAR:
- return year;
- case MONTH:
- return month;
- case DAY:
- return day;
- default:
- throw new AssertionError(range);
- }
- }
-
- public static int ymdToUnixDate(int year, int month, int day) {
- final int julian = ymdToJulian(year, month, day);
- return julian - EPOCH_JULIAN;
- }
-
- public static int ymdToJulian(int year, int month, int day) {
- int a = (14 - month) / 12;
- int y = year + 4800 - a;
- int m = month + 12 * a - 3;
- int j = day + (153 * m + 2) / 5
- + 365 * y
- + y / 4
- - y / 100
- + y / 400
- - 32045;
- if (j < 2299161) {
- j = day + (153 * m + 2) / 5 + 365 * y + y / 4 - 32083;
- }
- return j;
- }
-
- public static String intervalYearMonthToString(int v, TimeUnitRange range) {
- final StringBuilder buf = new StringBuilder();
- if (v >= 0) {
- buf.append('+');
- } else {
- buf.append('-');
- v = -v;
- }
- final int y;
- final int m;
- switch (range) {
- case YEAR:
- v = roundUp(v, 12);
- y = v / 12;
- buf.append(y);
- break;
- case YEAR_TO_MONTH:
- y = v / 12;
- buf.append(y);
- buf.append('-');
- m = v % 12;
- number(buf, m, 2);
- break;
- case MONTH:
- m = v;
- buf.append(m);
- break;
- default:
- throw new AssertionError(range);
- }
- return buf.toString();
- }
-
- private static StringBuilder number(StringBuilder buf, int v, int n) {
- for (int k = digitCount(v); k < n; k++) {
- buf.append('0');
- }
- return buf.append(v);
- }
-
- public static int digitCount(int v) {
- for (int n = 1;; n++) {
- v /= 10;
- if (v == 0) {
- return n;
- }
- }
- }
-
- public static String intervalDayTimeToString(long v, TimeUnitRange range,
- int scale) {
- final StringBuilder buf = new StringBuilder();
- if (v >= 0) {
- buf.append('+');
- } else {
- buf.append('-');
- v = -v;
- }
- final long ms;
- final long s;
- final long m;
- final long h;
- final long d;
- switch (range) {
- case DAY_TO_SECOND:
- v = roundUp(v, powerX(10, 3 - scale));
- ms = v % 1000;
- v /= 1000;
- s = v % 60;
- v /= 60;
- m = v % 60;
- v /= 60;
- h = v % 24;
- v /= 24;
- d = v;
- buf.append((int) d);
- buf.append(' ');
- number(buf, (int) h, 2);
- buf.append(':');
- number(buf, (int) m, 2);
- buf.append(':');
- number(buf, (int) s, 2);
- fraction(buf, scale, ms);
- break;
- case DAY_TO_MINUTE:
- v = roundUp(v, 1000 * 60);
- v /= 1000;
- v /= 60;
- m = v % 60;
- v /= 60;
- h = v % 24;
- v /= 24;
- d = v;
- buf.append((int) d);
- buf.append(' ');
- number(buf, (int) h, 2);
- buf.append(':');
- number(buf, (int) m, 2);
- break;
- case DAY_TO_HOUR:
- v = roundUp(v, 1000 * 60 * 60);
- v /= 1000;
- v /= 60;
- v /= 60;
- h = v % 24;
- v /= 24;
- d = v;
- buf.append((int) d);
- buf.append(' ');
- number(buf, (int) h, 2);
- break;
- case DAY:
- v = roundUp(v, 1000 * 60 * 60 * 24);
- d = v / (1000 * 60 * 60 * 24);
- buf.append((int) d);
- break;
- case HOUR:
- v = roundUp(v, 1000 * 60 * 60);
- v /= 1000;
- v /= 60;
- v /= 60;
- h = v;
- buf.append((int) h);
- break;
- case HOUR_TO_MINUTE:
- v = roundUp(v, 1000 * 60);
- v /= 1000;
- v /= 60;
- m = v % 60;
- v /= 60;
- h = v;
- buf.append((int) h);
- buf.append(':');
- number(buf, (int) m, 2);
- break;
- case HOUR_TO_SECOND:
- v = roundUp(v, powerX(10, 3 - scale));
- ms = v % 1000;
- v /= 1000;
- s = v % 60;
- v /= 60;
- m = v % 60;
- v /= 60;
- h = v;
- buf.append((int) h);
- buf.append(':');
- number(buf, (int) m, 2);
- buf.append(':');
- number(buf, (int) s, 2);
- fraction(buf, scale, ms);
- break;
- case MINUTE_TO_SECOND:
- v = roundUp(v, powerX(10, 3 - scale));
- ms = v % 1000;
- v /= 1000;
- s = v % 60;
- v /= 60;
- m = v;
- buf.append((int) m);
- buf.append(':');
- number(buf, (int) s, 2);
- fraction(buf, scale, ms);
- break;
- case MINUTE:
- v = roundUp(v, 1000 * 60);
- v /= 1000;
- v /= 60;
- m = v;
- buf.append((int) m);
- break;
- case SECOND:
- v = roundUp(v, powerX(10, 3 - scale));
- ms = v % 1000;
- v /= 1000;
- s = v;
- buf.append((int) s);
- fraction(buf, scale, ms);
- break;
- default:
- throw new AssertionError(range);
- }
- return buf.toString();
- }
-
- /**
- * Rounds a dividend to the nearest divisor.
- * For example roundUp(31, 10) yields 30; roundUp(37, 10) yields 40.
- * @param dividend Number to be divided
- * @param divisor Number to divide by
- * @return Rounded dividend
- */
- private static long roundUp(long dividend, long divisor) {
- long remainder = dividend % divisor;
- dividend -= remainder;
- if (remainder * 2 > divisor) {
- dividend += divisor;
- }
- return dividend;
- }
-
- private static int roundUp(int dividend, int divisor) {
- int remainder = dividend % divisor;
- dividend -= remainder;
- if (remainder * 2 > divisor) {
- dividend += divisor;
- }
- return dividend;
- }
-
- private static void fraction(StringBuilder buf, int scale, long ms) {
- if (scale > 0) {
- buf.append('.');
- long v1 = scale == 3 ? ms
- : scale == 2 ? ms / 10
- : scale == 1 ? ms / 100
- : 0;
- number(buf, (int) v1, scale);
- }
- }
-
- /** Helper for "array element reference". Caller has already ensured that
- * array and index are not null. Index is 1-based, per SQL. */
- public static Object arrayItem(List list, int item) {
- if (item < 1 || item > list.size()) {
- return null;
- }
- return list.get(item - 1);
- }
-
- /** Helper for "map element reference". Caller has already ensured that
- * array and index are not null. Index is 1-based, per SQL. */
- public static Object mapItem(Map map, Object item) {
- return map.get(item);
- }
-
- /** Implements the {@code [ ... ]} operator on an object whose type is not
- * known until runtime.
- */
- public static Object item(Object object, Object index) {
- if (object instanceof Map) {
- return ((Map) object).get(index);
- }
- if (object instanceof List && index instanceof Number) {
- List list = (List) object;
- return list.get(((Number) index).intValue());
- }
- return null;
- }
-
- /** NULL → FALSE, FALSE → FALSE, TRUE → TRUE. */
- public static boolean isTrue(Boolean b) {
- return b != null && b;
- }
-
- /** NULL → TRUE, FALSE → FALSE, TRUE → TRUE. */
- public static boolean isNotFalse(Boolean b) {
- return b == null || b;
- }
-
- /** NULL → NULL, FALSE → TRUE, TRUE → FALSE. */
- public static Boolean not(Boolean b) {
- return (b == null) ? null : !b;
- }
-
- /** Converts a JDBC array to a list. */
- public static List arrayToList(final java.sql.Array a) {
- if (a == null) {
- return null;
- }
- try {
- return Primitive.asList(a.getArray());
- } catch (SQLException e) {
- throw new RuntimeException(e);
- }
- }
-
- /** Support the SLICE function. */
- public static List slice(List list) {
- return list;
- }
-
- /** Support the ELEMENT function. */
- public static Object element(List list) {
- switch (list.size()) {
- case 0:
- return null;
- case 1:
- return list.get(0);
- default:
- throw new RuntimeException("more than one value");
- }
- }
-
- /** Returns a lambda that converts a list to an enumerable. */
- public static <E> Function1<List<E>, Enumerable<E>> listToEnumerable() {
- //noinspection unchecked
- return (Function1<List<E>, Enumerable<E>>) (Function1) LIST_AS_ENUMERABLE;
- }
-
- /** A range of time units. The first is more significant than the
- * other (e.g. year-to-day) or the same as the other
- * (e.g. month). */
- public enum TimeUnitRange {
- YEAR,
- YEAR_TO_MONTH,
- MONTH,
- DAY,
- DAY_TO_HOUR,
- DAY_TO_MINUTE,
- DAY_TO_SECOND,
- HOUR,
- HOUR_TO_MINUTE,
- HOUR_TO_SECOND,
- MINUTE,
- MINUTE_TO_SECOND,
- SECOND;
-
- /** Whether this is in the YEAR-TO-MONTH family of intervals. */
- public boolean monthly() {
- return ordinal() <= MONTH.ordinal();
- }
- }
-}
-
-// End SqlFunctions.java
diff --git a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
new file mode 100644
index 0000000000..c184f0bb6d
--- /dev/null
+++ b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -0,0 +1,4040 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.calcite.sql2rel;
+
+import org.apache.calcite.linq4j.Ord;
+import org.apache.calcite.plan.Convention;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelOptSamplingParameters;
+import org.apache.calcite.plan.RelOptTable;
+import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.prepare.Prepare;
+import org.apache.calcite.prepare.RelOptTableImpl;
+import org.apache.calcite.rel.RelCollation;
+import org.apache.calcite.rel.RelCollations;
+import org.apache.calcite.rel.RelFieldCollation;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.RelShuttle;
+import org.apache.calcite.rel.core.Aggregate;
+import org.apache.calcite.rel.core.AggregateCall;
+import org.apache.calcite.rel.core.Collect;
+import org.apache.calcite.rel.core.CorrelationId;
+import org.apache.calcite.rel.core.Join;
+import org.apache.calcite.rel.core.JoinInfo;
+import org.apache.calcite.rel.core.JoinRelType;
+import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.core.RelFactories;
+import org.apache.calcite.rel.core.Sample;
+import org.apache.calcite.rel.core.Uncollect;
+import org.apache.calcite.rel.logical.LogicalAggregate;
+import org.apache.calcite.rel.logical.LogicalCorrelate;
+import org.apache.calcite.rel.logical.LogicalIntersect;
+import org.apache.calcite.rel.logical.LogicalJoin;
+import org.apache.calcite.rel.logical.LogicalMinus;
+import org.apache.calcite.rel.logical.LogicalProject;
+import org.apache.calcite.rel.logical.LogicalSort;
+import org.apache.calcite.rel.logical.LogicalTableFunctionScan;
+import org.apache.calcite.rel.logical.LogicalTableModify;
+import org.apache.calcite.rel.logical.LogicalTableScan;
+import org.apache.calcite.rel.logical.LogicalUnion;
+import org.apache.calcite.rel.logical.LogicalValues;
+import org.apache.calcite.rel.metadata.RelColumnMapping;
+import org.apache.calcite.rel.stream.LogicalDelta;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexCallBinding;
+import org.apache.calcite.rex.RexCorrelVariable;
+import org.apache.calcite.rex.RexDynamicParam;
+import org.apache.calcite.rex.RexFieldAccess;
+import org.apache.calcite.rex.RexFieldCollation;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexRangeRef;
+import org.apache.calcite.rex.RexShuttle;
+import org.apache.calcite.rex.RexUtil;
+import org.apache.calcite.rex.RexVisitorImpl;
+import org.apache.calcite.rex.RexWindowBound;
+import org.apache.calcite.schema.ModifiableTable;
+import org.apache.calcite.schema.ModifiableView;
+import org.apache.calcite.schema.Table;
+import org.apache.calcite.schema.TranslatableTable;
+import org.apache.calcite.sql.JoinConditionType;
+import org.apache.calcite.sql.JoinType;
+import org.apache.calcite.sql.SemiJoinType;
+import org.apache.calcite.sql.SqlAggFunction;
+import org.apache.calcite.sql.SqlBasicCall;
+import org.apache.calcite.sql.SqlCall;
+import org.apache.calcite.sql.SqlDataTypeSpec;
+import org.apache.calcite.sql.SqlDelete;
+import org.apache.calcite.sql.SqlDynamicParam;
+import org.apache.calcite.sql.SqlExplainLevel;
+import org.apache.calcite.sql.SqlFunction;
+import org.apache.calcite.sql.SqlIdentifier;
+import org.apache.calcite.sql.SqlInsert;
+import org.apache.calcite.sql.SqlIntervalQualifier;
+import org.apache.calcite.sql.SqlJoin;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlLiteral;
+import org.apache.calcite.sql.SqlMerge;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlNodeList;
+import org.apache.calcite.sql.SqlNumericLiteral;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.SqlOperatorTable;
+import org.apache.calcite.sql.SqlSampleSpec;
+import org.apache.calcite.sql.SqlSelect;
+import org.apache.calcite.sql.SqlSelectKeyword;
+import org.apache.calcite.sql.SqlSetOperator;
+import org.apache.calcite.sql.SqlUpdate;
+import org.apache.calcite.sql.SqlUtil;
+import org.apache.calcite.sql.SqlValuesOperator;
+import org.apache.calcite.sql.SqlWindow;
+import org.apache.calcite.sql.SqlWith;
+import org.apache.calcite.sql.SqlWithItem;
+import org.apache.calcite.sql.fun.SqlCountAggFunction;
+import org.apache.calcite.sql.fun.SqlInOperator;
+import org.apache.calcite.sql.fun.SqlRowOperator;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.parser.SqlParserPos;
+import org.apache.calcite.sql.type.SqlReturnTypeInference;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.sql.type.SqlTypeUtil;
+import org.apache.calcite.sql.type.TableFunctionReturnTypeInference;
+import org.apache.calcite.sql.util.SqlBasicVisitor;
+import org.apache.calcite.sql.util.SqlVisitor;
+import org.apache.calcite.sql.validate.AggregatingSelectScope;
+import org.apache.calcite.sql.validate.CollectNamespace;
+import org.apache.calcite.sql.validate.DelegatingScope;
+import org.apache.calcite.sql.validate.ListScope;
+import org.apache.calcite.sql.validate.ParameterScope;
+import org.apache.calcite.sql.validate.SelectScope;
+import org.apache.calcite.sql.validate.SqlMonotonicity;
+import org.apache.calcite.sql.validate.SqlQualified;
+import org.apache.calcite.sql.validate.SqlUserDefinedTableFunction;
+import org.apache.calcite.sql.validate.SqlUserDefinedTableMacro;
+import org.apache.calcite.sql.validate.SqlValidator;
+import org.apache.calcite.sql.validate.SqlValidatorImpl;
+import org.apache.calcite.sql.validate.SqlValidatorNamespace;
+import org.apache.calcite.sql.validate.SqlValidatorScope;
+import org.apache.calcite.sql.validate.SqlValidatorUtil;
+import org.apache.calcite.util.ImmutableBitSet;
+import org.apache.calcite.util.ImmutableIntList;
+import org.apache.calcite.util.NlsString;
+import org.apache.calcite.util.NumberUtil;
+import org.apache.calcite.util.Pair;
+import org.apache.calcite.util.Util;
+import org.apache.calcite.util.trace.CalciteTrace;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+import java.lang.reflect.Type;
+import java.math.BigDecimal;
+import java.util.AbstractList;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.Stack;
+import java.util.TreeSet;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import static org.apache.calcite.sql.SqlUtil.stripAs;
+import static org.apache.calcite.util.Static.RESOURCE;
+
+/*
+ * OVERRIDE POINT:
+ * - getInSubqueryThreshold(), was `20`, now `Integer.MAX_VALUE`
+ * - isTrimUnusedFields(), override to false
+ * - AggConverter.visit(SqlCall), skip column reading for COUNT(COL), for https://jirap.corp.ebay.com/browse/KYLIN-104
+ */
+
+/**
+ * Converts a SQL parse tree (consisting of
+ * {@link org.apache.calcite.sql.SqlNode} objects) into a relational algebra
+ * expression (consisting of {@link org.apache.calcite.rel.RelNode} objects).
+ *
+ * <p>The public entry points are: {@link #convertQuery},
+ * {@link #convertExpression(SqlNode)}.
+ */
+public class SqlToRelConverter {
+ //~ Static fields/initializers ---------------------------------------------
+
+ protected static final Logger SQL2REL_LOGGER = CalciteTrace.getSqlToRelTracer();
+
+ private static final BigDecimal TWO = BigDecimal.valueOf(2L);
+
+ //~ Instance fields --------------------------------------------------------
+
+ protected final SqlValidator validator;
+ protected final RexBuilder rexBuilder;
+ protected final Prepare.CatalogReader catalogReader;
+ protected final RelOptCluster cluster;
+ private DefaultValueFactory defaultValueFactory;
+ private SubqueryConverter subqueryConverter;
+ protected final List<RelNode> leaves = new ArrayList<>();
+ private final List<SqlDynamicParam> dynamicParamSqlNodes = new ArrayList<>();
+ private final SqlOperatorTable opTab;
+ private boolean shouldConvertTableAccess;
+ protected final RelDataTypeFactory typeFactory;
+ private final SqlNodeToRexConverter exprConverter;
+ private boolean decorrelationEnabled;
+ private boolean trimUnusedFields;
+ private boolean shouldCreateValuesRel;
+ private boolean isExplain;
+ private int nDynamicParamsInExplain;
+
+ /**
+ * Fields used in name resolution for correlated subqueries.
+ */
+ private final Map<String, DeferredLookup> mapCorrelToDeferred = new HashMap<>();
+ private int nextCorrel = 0;
+
+ private static final String CORREL_PREFIX = "$cor";
+
+ /**
+ * Stack of names of datasets requested by the <code>
+ * TABLE(SAMPLE(<datasetName>, <query>))</code> construct.
+ */
+ private final Stack<String> datasetStack = new Stack<>();
+
+ /**
+ * Mapping of non-correlated subqueries that have been converted to their
+ * equivalent constants. Used to avoid re-evaluating the subquery if it's
+ * already been evaluated.
+ */
+ private final Map<SqlNode, RexNode> mapConvertedNonCorrSubqs = new HashMap<>();
+
+ public final RelOptTable.ViewExpander viewExpander;
+
+ //~ Constructors -----------------------------------------------------------
+ /**
+ * Creates a converter.
+ *
+ * @param viewExpander Preparing statement
+ * @param validator Validator
+ * @param catalogReader Schema
+ * @param planner Planner
+ * @param rexBuilder Rex builder
+ * @param convertletTable Expression converter
+ */
+ @Deprecated
+ // will be removed before 2.0
+ public SqlToRelConverter(RelOptTable.ViewExpander viewExpander, SqlValidator validator, Prepare.CatalogReader catalogReader, RelOptPlanner planner, RexBuilder rexBuilder, SqlRexConvertletTable convertletTable) {
+ this(viewExpander, validator, catalogReader, RelOptCluster.create(planner, rexBuilder), convertletTable);
+ }
+
+ /* Creates a converter. */
+ public SqlToRelConverter(RelOptTable.ViewExpander viewExpander, SqlValidator validator, Prepare.CatalogReader catalogReader, RelOptCluster cluster, SqlRexConvertletTable convertletTable) {
+ this.viewExpander = viewExpander;
+ this.opTab = (validator == null) ? SqlStdOperatorTable.instance() : validator.getOperatorTable();
+ this.validator = validator;
+ this.catalogReader = catalogReader;
+ this.defaultValueFactory = new NullDefaultValueFactory();
+ this.subqueryConverter = new NoOpSubqueryConverter();
+ this.rexBuilder = cluster.getRexBuilder();
+ this.typeFactory = rexBuilder.getTypeFactory();
+ this.cluster = Preconditions.checkNotNull(cluster);
+ this.shouldConvertTableAccess = true;
+ this.exprConverter = new SqlNodeToRexConverterImpl(convertletTable);
+ decorrelationEnabled = true;
+ trimUnusedFields = false;
+ shouldCreateValuesRel = true;
+ isExplain = false;
+ nDynamicParamsInExplain = 0;
+ }
+
+ //~ Methods ----------------------------------------------------------------
+
+ /**
+ * @return the RelOptCluster in use.
+ */
+ public RelOptCluster getCluster() {
+ return cluster;
+ }
+
+ /**
+ * Returns the row-expression builder.
+ */
+ public RexBuilder getRexBuilder() {
+ return rexBuilder;
+ }
+
+ /**
+ * Returns the number of dynamic parameters encountered during translation;
+ * this must only be called after {@link #convertQuery}.
+ *
+ * @return number of dynamic parameters
+ */
+ public int getDynamicParamCount() {
+ return dynamicParamSqlNodes.size();
+ }
+
+ /**
+ * Returns the type inferred for a dynamic parameter.
+ *
+ * @param index 0-based index of dynamic parameter
+ * @return inferred type, never null
+ */
+ public RelDataType getDynamicParamType(int index) {
+ SqlNode sqlNode = dynamicParamSqlNodes.get(index);
+ if (sqlNode == null) {
+ throw Util.needToImplement("dynamic param type inference");
+ }
+ return validator.getValidatedNodeType(sqlNode);
+ }
+
+ /**
+ * Returns the current count of the number of dynamic parameters in an
+ * EXPLAIN PLAN statement.
+ *
+ * @param increment if true, increment the count
+ * @return the current count before the optional increment
+ */
+ public int getDynamicParamCountInExplain(boolean increment) {
+ int retVal = nDynamicParamsInExplain;
+ if (increment) {
+ ++nDynamicParamsInExplain;
+ }
+ return retVal;
+ }
+
+ /**
+ * @return mapping of non-correlated subqueries that have been converted to
+ * the constants that they evaluate to
+ */
+ public Map<SqlNode, RexNode> getMapConvertedNonCorrSubqs() {
+ return mapConvertedNonCorrSubqs;
+ }
+
+ /**
+ * Adds to the current map of non-correlated converted subqueries the
+ * elements from another map that contains non-correlated subqueries that
+ * have been converted by another SqlToRelConverter.
+ *
+ * @param alreadyConvertedNonCorrSubqs the other map
+ */
+ public void addConvertedNonCorrSubqs(Map<SqlNode, RexNode> alreadyConvertedNonCorrSubqs) {
+ mapConvertedNonCorrSubqs.putAll(alreadyConvertedNonCorrSubqs);
+ }
+
+ /**
+ * Set a new DefaultValueFactory. To have any effect, this must be called
+ * before any convert method.
+ *
+ * @param factory new DefaultValueFactory
+ */
+ public void setDefaultValueFactory(DefaultValueFactory factory) {
+ defaultValueFactory = factory;
+ }
+
+ /**
+ * Sets a new SubqueryConverter. To have any effect, this must be called
+ * before any convert method.
+ *
+ * @param converter new SubqueryConverter
+ */
+ public void setSubqueryConverter(SubqueryConverter converter) {
+ subqueryConverter = converter;
+ }
+
+ /**
+ * Indicates that the current statement is part of an EXPLAIN PLAN statement
+ *
+ * @param nDynamicParams number of dynamic parameters in the statement
+ */
+ public void setIsExplain(int nDynamicParams) {
+ isExplain = true;
+ nDynamicParamsInExplain = nDynamicParams;
+ }
+
+ /**
+ * Controls whether table access references are converted to physical rels
+ * immediately. The optimizer doesn't like leaf rels to have
+ * {@link Convention#NONE}. However, if we are doing further conversion
+ * passes (e.g. {@link RelStructuredTypeFlattener}), then we may need to
+ * defer conversion. To have any effect, this must be called before any
+ * convert method.
+ *
+ * @param enabled true for immediate conversion (the default); false to
+ * generate logical LogicalTableScan instances
+ */
+ public void enableTableAccessConversion(boolean enabled) {
+ shouldConvertTableAccess = enabled;
+ }
+
+ /**
+ * Controls whether instances of
+ * {@link org.apache.calcite.rel.logical.LogicalValues} are generated. These
+ * may not be supported by all physical implementations. To have any effect,
+ * this must be called before any convert method.
+ *
+ * @param enabled true to allow LogicalValues to be generated (the default);
+ * false to force substitution of Project+OneRow instead
+ */
+ public void enableValuesRelCreation(boolean enabled) {
+ shouldCreateValuesRel = enabled;
+ }
+
+ private void checkConvertedType(SqlNode query, RelNode result) {
+ if (!query.isA(SqlKind.DML)) {
+ // Verify that conversion from SQL to relational algebra did
+ // not perturb any type information. (We can't do this if the
+ // SQL statement is something like an INSERT which has no
+ // validator type information associated with its result,
+ // hence the namespace check above.)
+ RelDataType convertedRowType = result.getRowType();
+ if (!checkConvertedRowType(query, convertedRowType)) {
+ RelDataType validatedRowType = validator.getValidatedNodeType(query);
+ validatedRowType = uniquifyFields(validatedRowType);
+ throw Util.newInternal("Conversion to relational algebra failed to " + "preserve datatypes:\n" + "validated type:\n" + validatedRowType.getFullTypeString() + "\nconverted type:\n" + convertedRowType.getFullTypeString() + "\nrel:\n" + RelOptUtil.toString(result));
+ }
+ }
+ }
+
+ public RelNode flattenTypes(RelNode rootRel, boolean restructure) {
+ RelStructuredTypeFlattener typeFlattener = new RelStructuredTypeFlattener(rexBuilder, createToRelContext());
+ return typeFlattener.rewrite(rootRel, restructure);
+ }
+
+ /**
+ * If subquery is correlated and decorrelation is enabled, performs
+ * decorrelation.
+ *
+ * @param query Query
+ * @param rootRel Root relational expression
+ * @return New root relational expression after decorrelation
+ */
+ public RelNode decorrelate(SqlNode query, RelNode rootRel) {
+ if (!enableDecorrelation()) {
+ return rootRel;
+ }
+ final RelNode result = decorrelateQuery(rootRel);
+ if (result != rootRel) {
+ checkConvertedType(query, result);
+ }
+ return result;
+ }
+
+ /**
+ * Walks over a tree of relational expressions, replacing each
+ * {@link RelNode} with a 'slimmed down' relational expression that projects
+ * only the fields required by its consumer.
+ *
+ * <p>This may make things easier for the optimizer, by removing crud that
+ * would expand the search space, but is difficult for the optimizer itself
+ * to do it, because optimizer rules must preserve the number and type of
+ * fields. Hence, this transform that operates on the entire tree, similar
+ * to the {@link RelStructuredTypeFlattener type-flattening transform}.
+ *
+ * <p>Currently this functionality is disabled in farrago/luciddb; the
+ * default implementation of this method does nothing.
+ *
+ * @param rootRel Relational expression that is at the root of the tree
+ * @return Trimmed relational expression
+ */
+ public RelNode trimUnusedFields(RelNode rootRel) {
+ // Trim fields that are not used by their consumer.
+ if (isTrimUnusedFields()) {
+ final RelFieldTrimmer trimmer = newFieldTrimmer();
+ rootRel = trimmer.trim(rootRel);
+ boolean dumpPlan = SQL2REL_LOGGER.isLoggable(Level.FINE);
+ if (dumpPlan) {
+ SQL2REL_LOGGER.fine(RelOptUtil.dumpPlan("Plan after trimming unused fields", rootRel, false, SqlExplainLevel.EXPPLAN_ATTRIBUTES));
+ }
+ }
+ return rootRel;
+ }
+
+ /**
+ * Creates a RelFieldTrimmer.
+ *
+ * @return Field trimmer
+ */
+ protected RelFieldTrimmer newFieldTrimmer() {
+ return new RelFieldTrimmer(validator);
+ }
+
+ /**
+ * Converts an unvalidated query's parse tree into a relational expression.
+ *
+ * @param query Query to convert
+ * @param needsValidation Whether to validate the query before converting;
+ * <code>false</code> if the query has already been
+ * validated.
+ * @param top Whether the query is top-level, say if its result
+ * will become a JDBC result set; <code>false</code> if
+ * the query will be part of a view.
+ */
+ public RelNode convertQuery(SqlNode query, final boolean needsValidation, final boolean top) {
+ if (needsValidation) {
+ query = validator.validate(query);
+ }
+
+ RelNode result = convertQueryRecursive(query, top, null);
+ if (top && isStream(query)) {
+ result = new LogicalDelta(cluster, result.getTraitSet(), result);
+ }
+ checkConvertedType(query, result);
+
+ boolean dumpPlan = SQL2REL_LOGGER.isLoggable(Level.FINE);
+ if (dumpPlan) {
+ SQL2REL_LOGGER.fine(RelOptUtil.dumpPlan("Plan after converting SqlNode to RelNode", result, false, SqlExplainLevel.EXPPLAN_ATTRIBUTES));
+ }
+
+ return result;
+ }
+
+ private static boolean isStream(SqlNode query) {
+ return query instanceof SqlSelect && ((SqlSelect) query).isKeywordPresent(SqlSelectKeyword.STREAM);
+ }
+
+ protected boolean checkConvertedRowType(SqlNode query, RelDataType convertedRowType) {
+ RelDataType validatedRowType = validator.getValidatedNodeType(query);
+ validatedRowType = uniquifyFields(validatedRowType);
+
+ return RelOptUtil.equal("validated row type", validatedRowType, "converted row type", convertedRowType, false);
+ }
+
+ protected RelDataType uniquifyFields(RelDataType rowType) {
+ return validator.getTypeFactory().createStructType(RelOptUtil.getFieldTypeList(rowType), SqlValidatorUtil.uniquify(rowType.getFieldNames()));
+ }
+
+ /**
+ * Converts a SELECT statement's parse tree into a relational expression.
+ */
+ public RelNode convertSelect(SqlSelect select) {
+ final SqlValidatorScope selectScope = validator.getWhereScope(select);
+ final Blackboard bb = createBlackboard(selectScope, null);
+ convertSelectImpl(bb, select);
+ return bb.root;
+ }
+
+ /**
+ * Factory method for creating translation workspace.
+ */
+ protected Blackboard createBlackboard(SqlValidatorScope scope, Map<String, RexNode> nameToNodeMap) {
+ return new Blackboard(scope, nameToNodeMap);
+ }
+
+ /**
+ * Implementation of {@link #convertSelect(SqlSelect)}; derived class may
+ * override.
+ */
+ protected void convertSelectImpl(final Blackboard bb, SqlSelect select) {
+ convertFrom(bb, select.getFrom());
+ convertWhere(bb, select.getWhere());
+
+ final List<SqlNode> orderExprList = new ArrayList<>();
+ final List<RelFieldCollation> collationList = new ArrayList<>();
+ gatherOrderExprs(bb, select, select.getOrderList(), orderExprList, collationList);
+ final RelCollation collation = cluster.traitSet().canonize(RelCollations.of(collationList));
+
+ if (validator.isAggregate(select)) {
+ convertAgg(bb, select, orderExprList);
+ } else {
+ convertSelectList(bb, select, orderExprList);
+ }
+
+ if (select.isDistinct()) {
+ distinctify(bb, true);
+ }
+ convertOrder(select, bb, collation, orderExprList, select.getOffset(), select.getFetch());
+ bb.setRoot(bb.root, true);
+ }
+
+ /**
+ * Having translated 'SELECT ... FROM ... [GROUP BY ...] [HAVING ...]', adds
+ * a relational expression to make the results unique.
+ *
+ * <p>If the SELECT clause contains duplicate expressions, adds
+ * {@link org.apache.calcite.rel.logical.LogicalProject}s so that we are
+ * grouping on the minimal set of keys. The performance gain isn't huge, but
+ * it is difficult to detect these duplicate expressions later.
+ *
+ * @param bb Blackboard
+ * @param checkForDupExprs Check for duplicate expressions
+ */
+ private void distinctify(Blackboard bb, boolean checkForDupExprs) {
+ // Look for duplicate expressions in the project.
+ // Say we have 'select x, y, x, z'.
+ // Then dups will be {[2, 0]}
+ // and oldToNew will be {[0, 0], [1, 1], [2, 0], [3, 2]}
+ RelNode rel = bb.root;
+ if (checkForDupExprs && (rel instanceof LogicalProject)) {
+ LogicalProject project = (LogicalProject) rel;
+ final List<RexNode> projectExprs = project.getProjects();
+ final List<Integer> origins = new ArrayList<>();
+ int dupCount = 0;
+ for (int i = 0; i < projectExprs.size(); i++) {
+ int x = findExpr(projectExprs.get(i), projectExprs, i);
+ if (x >= 0) {
+ origins.add(x);
+ ++dupCount;
+ } else {
+ origins.add(i);
+ }
+ }
+ if (dupCount == 0) {
+ distinctify(bb, false);
+ return;
+ }
+
+ final Map<Integer, Integer> squished = Maps.newHashMap();
+ final List<RelDataTypeField> fields = rel.getRowType().getFieldList();
+ final List<Pair<RexNode, String>> newProjects = Lists.newArrayList();
+ for (int i = 0; i < fields.size(); i++) {
+ if (origins.get(i) == i) {
+ squished.put(i, newProjects.size());
+ newProjects.add(RexInputRef.of2(i, fields));
+ }
+ }
+ rel = LogicalProject.create(rel, Pair.left(newProjects), Pair.right(newProjects));
+ bb.root = rel;
+ distinctify(bb, false);
+ rel = bb.root;
+
+ // Create the expressions to reverse the mapping.
+ // Project($0, $1, $0, $2).
+ final List<Pair<RexNode, String>> undoProjects = Lists.newArrayList();
+ for (int i = 0; i < fields.size(); i++) {
+ final int origin = origins.get(i);
+ RelDataTypeField field = fields.get(i);
+ undoProjects.add(Pair.of((RexNode) new RexInputRef(squished.get(origin), field.getType()), field.getName()));
+ }
+
+ rel = LogicalProject.create(rel, Pair.left(undoProjects), Pair.right(undoProjects));
+ bb.setRoot(rel, false);
+
+ return;
+ }
+
+ // Usual case: all of the expressions in the SELECT clause are
+ // different.
+ final ImmutableBitSet groupSet = ImmutableBitSet.range(rel.getRowType().getFieldCount());
+ rel = createAggregate(bb, false, groupSet, ImmutableList.of(groupSet), ImmutableList.<AggregateCall> of());
+
+ bb.setRoot(rel, false);
+ }
+
+ private int findExpr(RexNode seek, List<RexNode> exprs, int count) {
+ for (int i = 0; i < count; i++) {
+ RexNode expr = exprs.get(i);
+ if (expr.toString().equals(seek.toString())) {
+ return i;
+ }
+ }
+ return -1;
+ }
+
+ /**
+ * Converts a query's ORDER BY clause, if any.
+ *
+ * @param select Query
+ * @param bb Blackboard
+ * @param collation Collation list
+ * @param orderExprList Method populates this list with orderBy expressions
+ * not present in selectList
+ * @param offset Expression for number of rows to discard before
+ * returning first row
+ * @param fetch Expression for number of rows to fetch
+ */
+ protected void convertOrder(SqlSelect select, Blackboard bb, RelCollation collation, List<SqlNode> orderExprList, SqlNode offset, SqlNode fetch) {
+ if (select.getOrderList() == null) {
+ assert collation.getFieldCollations().isEmpty();
+ if (offset == null && fetch == null) {
+ return;
+ }
+ }
+
+ // Create a sorter using the previously constructed collations.
+ bb.setRoot(LogicalSort.create(bb.root, collation, offset == null ? null : convertExpression(offset), fetch == null ? null : convertExpression(fetch)), false);
+
+ // If extra expressions were added to the project list for sorting,
+ // add another project to remove them.
+ if (orderExprList.size() > 0) {
+ final List<RexNode> exprs = new ArrayList<>();
+ final RelDataType rowType = bb.root.getRowType();
+ final int fieldCount = rowType.getFieldCount() - orderExprList.size();
+ for (int i = 0; i < fieldCount; i++) {
+ exprs.add(rexBuilder.makeInputRef(bb.root, i));
+ }
+ bb.setRoot(new LogicalProject(cluster, cluster.traitSetOf(RelCollations.PRESERVE), bb.root, exprs, cluster.getTypeFactory().createStructType(rowType.getFieldList().subList(0, fieldCount))), false);
+ }
+ }
+
+ /**
+ * Returns whether a given node contains a {@link SqlInOperator}.
+ *
+ * @param node a RexNode tree
+ */
+ private static boolean containsInOperator(SqlNode node) {
+ try {
+ SqlVisitor<Void> visitor = new SqlBasicVisitor<Void>() {
+ public Void visit(SqlCall call) {
+ if (call.getOperator() instanceof SqlInOperator) {
+ throw new Util.FoundOne(call);
+ }
+ return super.visit(call);
+ }
+ };
+ node.accept(visitor);
+ return false;
+ } catch (Util.FoundOne e) {
+ Util.swallow(e, null);
+ return true;
+ }
+ }
+
+ /**
+ * Push down all the NOT logical operators into any IN/NOT IN operators.
+ *
+ * @param sqlNode the root node from which to look for NOT operators
+ * @return the transformed SqlNode representation with NOT pushed down.
+ */
+ private static SqlNode pushDownNotForIn(SqlNode sqlNode) {
+ if ((sqlNode instanceof SqlCall) && containsInOperator(sqlNode)) {
+ SqlCall sqlCall = (SqlCall) sqlNode;
+ if ((sqlCall.getOperator() == SqlStdOperatorTable.AND) || (sqlCall.getOperator() == SqlStdOperatorTable.OR)) {
+ SqlNode[] sqlOperands = ((SqlBasicCall) sqlCall).operands;
+ for (int i = 0; i < sqlOperands.length; i++) {
+ sqlOperands[i] = pushDownNotForIn(sqlOperands[i]);
+ }
+ return sqlNode;
+ } else if (sqlCall.getOperator() == SqlStdOperatorTable.NOT) {
+ SqlNode childNode = sqlCall.operand(0);
+ assert childNode instanceof SqlCall;
+ SqlBasicCall childSqlCall = (SqlBasicCall) childNode;
+ if (childSqlCall.getOperator() == SqlStdOperatorTable.AND) {
+ SqlNode[] andOperands = childSqlCall.getOperands();
+ SqlNode[] orOperands = new SqlNode[andOperands.length];
+ for (int i = 0; i < orOperands.length; i++) {
+ orOperands[i] = SqlStdOperatorTable.NOT.createCall(SqlParserPos.ZERO, andOperands[i]);
+ }
+ for (int i = 0; i < orOperands.length; i++) {
+ orOperands[i] = pushDownNotForIn(orOperands[i]);
+ }
+ return SqlStdOperatorTable.OR.createCall(SqlParserPos.ZERO, orOperands[0], orOperands[1]);
+ } else if (childSqlCall.getOperator() == SqlStdOperatorTable.OR) {
+ SqlNode[] orOperands = childSqlCall.getOperands();
+ SqlNode[] andOperands = new SqlNode[orOperands.length];
+ for (int i = 0; i < andOperands.length; i++) {
+ andOperands[i] = SqlStdOperatorTable.NOT.createCall(SqlParserPos.ZERO, orOperands[i]);
+ }
+ for (int i = 0; i < andOperands.length; i++) {
+ andOperands[i] = pushDownNotForIn(andOperands[i]);
+ }
+ return SqlStdOperatorTable.AND.createCall(SqlParserPos.ZERO, andOperands[0], andOperands[1]);
+ } else if (childSqlCall.getOperator() == SqlStdOperatorTable.NOT) {
+ SqlNode[] notOperands = childSqlCall.getOperands();
+ assert notOperands.length == 1;
+ return pushDownNotForIn(notOperands[0]);
+ } else if (childSqlCall.getOperator() instanceof SqlInOperator) {
+ SqlNode[] inOperands = childSqlCall.getOperands();
+ SqlInOperator inOp = (SqlInOperator) childSqlCall.getOperator();
+ if (inOp.isNotIn()) {
+ return SqlStdOperatorTable.IN.createCall(SqlParserPos.ZERO, inOperands[0], inOperands[1]);
+ } else {
+ return SqlStdOperatorTable.NOT_IN.createCall(SqlParserPos.ZERO, inOperands[0], inOperands[1]);
+ }
+ } else {
+ // childSqlCall is "leaf" node in a logical expression tree
+ // (only considering AND, OR, NOT)
+ return sqlNode;
+ }
+ } else {
+ // sqlNode is "leaf" node in a logical expression tree
+ // (only considering AND, OR, NOT)
+ return sqlNode;
+ }
+ } else {
+ // tree rooted at sqlNode does not contain inOperator
+ return sqlNode;
+ }
+ }
+
+ /**
+ * Converts a WHERE clause.
+ *
+ * @param bb Blackboard
+ * @param where WHERE clause, may be null
+ */
+ private void convertWhere(final Blackboard bb, final SqlNode where) {
+ if (where == null) {
+ return;
+ }
+ SqlNode newWhere = pushDownNotForIn(where);
+ replaceSubqueries(bb, newWhere, RelOptUtil.Logic.UNKNOWN_AS_FALSE);
+ final RexNode convertedWhere = bb.convertExpression(newWhere);
+
+ // only allocate filter if the condition is not TRUE
+ if (!convertedWhere.isAlwaysTrue()) {
+ bb.setRoot(RelOptUtil.createFilter(bb.root, convertedWhere), false);
+ }
+ }
+
+ private void replaceSubqueries(final Blackboard bb, final SqlNode expr, RelOptUtil.Logic logic) {
+ findSubqueries(bb, expr, logic, false);
+ for (SubQuery node : bb.subqueryList) {
+ substituteSubquery(bb, node);
+ }
+ }
+
+ private void substituteSubquery(Blackboard bb, SubQuery subQuery) {
+ final RexNode expr = subQuery.expr;
+ if (expr != null) {
+ // Already done.
+ return;
+ }
+
+ final SqlBasicCall call;
+ final RelNode rel;
+ final SqlNode query;
+ final Pair<RelNode, Boolean> converted;
+ switch (subQuery.node.getKind()) {
+ case CURSOR:
+ convertCursor(bb, subQuery);
+ return;
+
+ case MULTISET_QUERY_CONSTRUCTOR:
+ case MULTISET_VALUE_CONSTRUCTOR:
+ rel = convertMultisets(ImmutableList.of(subQuery.node), bb);
+ subQuery.expr = bb.register(rel, JoinRelType.INNER);
+ return;
+
+ case IN:
+ call = (SqlBasicCall) subQuery.node;
+ final SqlNode[] operands = call.getOperands();
+
+ SqlNode leftKeyNode = operands[0];
+ query = operands[1];
+
+ final List<RexNode> leftKeys;
+ switch (leftKeyNode.getKind()) {
+ case ROW:
+ leftKeys = Lists.newArrayList();
+ for (SqlNode sqlExpr : ((SqlBasicCall) leftKeyNode).getOperandList()) {
+ leftKeys.add(bb.convertExpression(sqlExpr));
+ }
+ break;
+ default:
+ leftKeys = ImmutableList.of(bb.convertExpression(leftKeyNode));
+ }
+
+ final boolean isNotIn = ((SqlInOperator) call.getOperator()).isNotIn();
+ if (query instanceof SqlNodeList) {
+ SqlNodeList valueList = (SqlNodeList) query;
+ if (!containsNullLiteral(valueList) && valueList.size() < getInSubqueryThreshold()) {
+ // We're under the threshold, so convert to OR.
+ subQuery.expr = convertInToOr(bb, leftKeys, valueList, isNotIn);
+ return;
+ }
+
+ // Otherwise, let convertExists translate
+ // values list into an inline table for the
+ // reference to Q below.
+ }
+
+ // Project out the search columns from the left side
+
+ // Q1:
+ // "select from emp where emp.deptno in (select col1 from T)"
+ //
+ // is converted to
+ //
+ // "select from
+ // emp inner join (select distinct col1 from T)) q
+ // on emp.deptno = q.col1
+ //
+ // Q2:
+ // "select from emp where emp.deptno not in (Q)"
+ //
+ // is converted to
+ //
+ // "select from
+ // emp left outer join (select distinct col1, TRUE from T) q
+ // on emp.deptno = q.col1
+ // where emp.deptno <> null
+ // and q.indicator <> TRUE"
+ //
+ final boolean outerJoin = bb.subqueryNeedsOuterJoin || isNotIn || subQuery.logic == RelOptUtil.Logic.TRUE_FALSE_UNKNOWN;
+ converted = convertExists(query, RelOptUtil.SubqueryType.IN, subQuery.logic, outerJoin);
+ if (converted.right) {
+ // Generate
+ // emp CROSS JOIN (SELECT COUNT(*) AS c,
+ // COUNT(deptno) AS ck FROM dept)
+ final RelDataType longType = typeFactory.createSqlType(SqlTypeName.BIGINT);
+ final RelNode seek = converted.left.getInput(0); // fragile
+ final int keyCount = leftKeys.size();
+ final List<Integer> args = ImmutableIntList.range(0, keyCount);
+ LogicalAggregate aggregate = LogicalAggregate.create(seek, false, ImmutableBitSet.of(), null, ImmutableList.of(AggregateCall.create(SqlStdOperatorTable.COUNT, false, ImmutableList.<Integer> of(), -1, longType, null), AggregateCall.create(SqlStdOperatorTable.COUNT, false, args, -1, longType, null)));
+ LogicalJoin join = LogicalJoin.create(bb.root, aggregate, rexBuilder.makeLiteral(true), JoinRelType.INNER, ImmutableSet.<String> of());
+ bb.setRoot(join, false);
+ }
+ RexNode rex = bb.register(converted.left, outerJoin ? JoinRelType.LEFT : JoinRelType.INNER, leftKeys);
+
+ subQuery.expr = translateIn(subQuery, bb.root, rex);
+ if (isNotIn) {
+ subQuery.expr = rexBuilder.makeCall(SqlStdOperatorTable.NOT, subQuery.expr);
+ }
+ return;
+
+ case EXISTS:
+ // "select from emp where exists (select a from T)"
+ //
+ // is converted to the following if the subquery is correlated:
+ //
+ // "select from emp left outer join (select AGG_TRUE() as indicator
+ // from T group by corr_var) q where q.indicator is true"
+ //
+ // If there is no correlation, the expression is replaced with a
+ // boolean indicating whether the subquery returned 0 or >= 1 row.
+ call = (SqlBasicCall) subQuery.node;
+ query = call.getOperands()[0];
+ converted = convertExists(query, RelOptUtil.SubqueryType.EXISTS, subQuery.logic, true);
+ assert !converted.right;
+ if (convertNonCorrelatedSubQuery(subQuery, bb, converted.left, true)) {
+ return;
+ }
+ subQuery.expr = bb.register(converted.left, JoinRelType.LEFT);
+ return;
+
+ case SCALAR_QUERY:
+ // Convert the subquery. If it's non-correlated, convert it
+ // to a constant expression.
+ call = (SqlBasicCall) subQuery.node;
+ query = call.getOperands()[0];
+ converted = convertExists(query, RelOptUtil.SubqueryType.SCALAR, subQuery.logic, true);
+ assert !converted.right;
+ if (convertNonCorrelatedSubQuery(subQuery, bb, converted.left, false)) {
+ return;
+ }
+ rel = convertToSingleValueSubq(query, converted.left);
+ subQuery.expr = bb.register(rel, JoinRelType.LEFT);
+ return;
+
+ case SELECT:
+ // This is used when converting multiset queries:
+ //
+ // select * from unnest(select multiset[deptno] from emps);
+ //
+ converted = convertExists(subQuery.node, RelOptUtil.SubqueryType.SCALAR, subQuery.logic, true);
+ assert !converted.right;
+ subQuery.expr = bb.register(converted.left, JoinRelType.LEFT);
+ return;
+
+ default:
+ throw Util.newInternal("unexpected kind of subquery :" + subQuery.node);
+ }
+ }
+
+ private RexNode translateIn(SubQuery subQuery, RelNode root, final RexNode rex) {
+ switch (subQuery.logic) {
+ case TRUE:
+ return rexBuilder.makeLiteral(true);
+
+ case UNKNOWN_AS_FALSE:
+ assert rex instanceof RexRangeRef;
+ final int fieldCount = rex.getType().getFieldCount();
+ RexNode rexNode = rexBuilder.makeFieldAccess(rex, fieldCount - 1);
+ rexNode = rexBuilder.makeCall(SqlStdOperatorTable.IS_TRUE, rexNode);
+
+ // Then append the IS NOT NULL(leftKeysForIn).
+ //
+ // RexRangeRef contains the following fields:
+ // leftKeysForIn,
+ // rightKeysForIn (the original subquery select list),
+ // nullIndicator
+ //
+ // The first two lists contain the same number of fields.
+ final int k = (fieldCount - 1) / 2;
+ for (int i = 0; i < k; i++) {
+ rexNode = rexBuilder.makeCall(SqlStdOperatorTable.AND, rexNode, rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, rexBuilder.makeFieldAccess(rex, i)));
+ }
+ return rexNode;
+
+ case TRUE_FALSE_UNKNOWN:
+ case UNKNOWN_AS_TRUE:
+ // select e.deptno,
+ // case
+ // when ct.c = 0 then false
+ // when dt.i is not null then true
+ // when e.deptno is null then null
+ // when ct.ck < ct.c then null
+ // else false
+ // end
+ // from e
+ // cross join (select count(*) as c, count(deptno) as ck from v) as ct
+ // left join (select distinct deptno, true as i from v) as dt
+ // on e.deptno = dt.deptno
+ final Join join = (Join) root;
+ final Project left = (Project) join.getLeft();
+ final RelNode leftLeft = ((Join) left.getInput()).getLeft();
+ final int leftLeftCount = leftLeft.getRowType().getFieldCount();
+ final RelDataType nullableBooleanType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.BOOLEAN), true);
+ final RelDataType longType = typeFactory.createSqlType(SqlTypeName.BIGINT);
+ final RexNode cRef = rexBuilder.makeInputRef(root, leftLeftCount);
+ final RexNode ckRef = rexBuilder.makeInputRef(root, leftLeftCount + 1);
+ final RexNode iRef = rexBuilder.makeInputRef(root, root.getRowType().getFieldCount() - 1);
+
+ final RexLiteral zero = rexBuilder.makeExactLiteral(BigDecimal.ZERO, longType);
+ final RexLiteral trueLiteral = rexBuilder.makeLiteral(true);
+ final RexLiteral falseLiteral = rexBuilder.makeLiteral(false);
+ final RexNode unknownLiteral = rexBuilder.makeNullLiteral(SqlTypeName.BOOLEAN);
+
+ final ImmutableList.Builder<RexNode> args = ImmutableList.builder();
+ args.add(rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, cRef, zero), falseLiteral, rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, iRef), trueLiteral);
+ final JoinInfo joinInfo = join.analyzeCondition();
+ for (int leftKey : joinInfo.leftKeys) {
+ final RexNode kRef = rexBuilder.makeInputRef(root, leftKey);
+ args.add(rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, kRef), unknownLiteral);
+ }
+ args.add(rexBuilder.makeCall(SqlStdOperatorTable.LESS_THAN, ckRef, cRef), unknownLiteral, falseLiteral);
+
+ return rexBuilder.makeCall(nullableBooleanType, SqlStdOperatorTable.CASE, args.build());
+
+ default:
+ throw new AssertionError(subQuery.logic);
+ }
+ }
+
+ private static boolean containsNullLiteral(SqlNodeList valueList) {
+ for (SqlNode node : valueList.getList()) {
+ if (node instanceof SqlLiteral) {
+ SqlLiteral lit = (SqlLiteral) node;
+ if (lit.getValue() == null) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Determines if a subquery is non-correlated and if so, converts it to a
+ * constant.
+ *
+ * @param subQuery the call that references the subquery
+ * @param bb blackboard used to convert the subquery
+ * @param converted RelNode tree corresponding to the subquery
+ * @param isExists true if the subquery is part of an EXISTS expression
+ * @return if the subquery can be converted to a constant
+ */
+ private boolean convertNonCorrelatedSubQuery(SubQuery subQuery, Blackboard bb, RelNode converted, boolean isExists) {
+ SqlCall call = (SqlBasicCall) subQuery.node;
+ if (subqueryConverter.canConvertSubquery() && isSubQueryNonCorrelated(converted, bb)) {
+ // First check if the subquery has already been converted
+ // because it's a nested subquery. If so, don't re-evaluate
+ // it again.
+ RexNode constExpr = mapConvertedNonCorrSubqs.get(call);
+ if (constExpr == null) {
+ constExpr = subqueryConverter.convertSubquery(call, this, isExists, isExplain);
+ }
+ if (constExpr != null) {
+ subQuery.expr = constExpr;
+ mapConvertedNonCorrSubqs.put(call, constExpr);
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Converts the RelNode tree for a select statement to a select that
+ * produces a single value.
+ *
+ * @param query the query
+ * @param plan the original RelNode tree corresponding to the statement
+ * @return the converted RelNode tree
+ */
+ public RelNode convertToSingleValueSubq(SqlNode query, RelNode plan) {
+ // Check whether query is guaranteed to produce a single value.
+ if (query instanceof SqlSelect) {
+ SqlSelect select = (SqlSelect) query;
+ SqlNodeList selectList = select.getSelectList();
+ SqlNodeList groupList = select.getGroup();
+
+ if ((selectList.size() == 1) && ((groupList == null) || (groupList.size() == 0))) {
+ SqlNode selectExpr = selectList.get(0);
+ if (selectExpr instanceof SqlCall) {
+ SqlCall selectExprCall = (SqlCall) selectExpr;
+ if (Util.isSingleValue(selectExprCall)) {
+ return plan;
+ }
+ }
+
+ // If there is a limit with 0 or 1,
+ // it is ensured to produce a single value
+ if (select.getFetch() != null && select.getFetch() instanceof SqlNumericLiteral) {
+ SqlNumericLiteral limitNum = (SqlNumericLiteral) select.getFetch();
+ if (((BigDecimal) limitNum.getValue()).intValue() < 2) {
+ return plan;
+ }
+ }
+ }
+ } else if (query instanceof SqlCall) {
+ // If the query is (values ...),
+ // it is necessary to look into the operands to determine
+ // whether SingleValueAgg is necessary
+ SqlCall exprCall = (SqlCall) query;
+ if (exprCall.getOperator() instanceof SqlValuesOperator && Util.isSingleValue(exprCall)) {
+ return plan;
+ }
+ }
+
+ // If not, project SingleValueAgg
+ return RelOptUtil.createSingleValueAggRel(cluster, plan);
+ }
+
+ /**
+ * Converts "x IN (1, 2, ...)" to "x=1 OR x=2 OR ...".
+ *
+ * @param leftKeys LHS
+ * @param valuesList RHS
+ * @param isNotIn is this a NOT IN operator
+ * @return converted expression
+ */
+ private RexNode convertInToOr(final Blackboard bb, final List<RexNode> leftKeys, SqlNodeList valuesList, boolean isNotIn) {
+ final List<RexNode> comparisons = new ArrayList<>();
+ for (SqlNode rightVals : valuesList) {
+ RexNode rexComparison;
+ if (leftKeys.size() == 1) {
+ rexComparison = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, leftKeys.get(0), rexBuilder.ensureType(leftKeys.get(0).getType(), bb.convertExpression(rightVals), true));
+ } else {
+ assert rightVals instanceof SqlCall;
+ final SqlBasicCall call = (SqlBasicCall) rightVals;
+ assert (call.getOperator() instanceof SqlRowOperator) && call.getOperands().length == leftKeys.size();
+ rexComparison = RexUtil.composeConjunction(rexBuilder, Iterables.transform(Pair.zip(leftKeys, call.getOperandList()), new Function<Pair<RexNode, SqlNode>, RexNode>() {
+ public RexNode apply(Pair<RexNode, SqlNode> pair) {
+ return rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, pair.left, rexBuilder.ensureType(pair.left.getType(), bb.convertExpression(pair.right), true));
+ }
+ }), false);
+ }
+ comparisons.add(rexComparison);
+ }
+
+ RexNode result = RexUtil.composeDisjunction(rexBuilder, comparisons, true);
+ assert result != null;
+
+ if (isNotIn) {
+ result = rexBuilder.makeCall(SqlStdOperatorTable.NOT, result);
+ }
+
+ return result;
+ }
+
+ /**
+ * Gets the list size threshold under which {@link #convertInToOr} is used.
+ * Lists of this size or greater will instead be converted to use a join
+ * against an inline table
+ * ({@link org.apache.calcite.rel.logical.LogicalValues}) rather than a
+ * predicate. A threshold of 0 forces usage of an inline table in all cases; a
+ * threshold of Integer.MAX_VALUE forces usage of OR in all cases
+ *
+ * @return threshold, default 20
+ */
+ protected int getInSubqueryThreshold() {
+ /* OVERRIDE POINT */
+ return Integer.MAX_VALUE;
+ }
+
+ /**
+ * Converts an EXISTS or IN predicate into a join. For EXISTS, the subquery
+ * produces an indicator variable, and the result is a relational expression
+ * which outer joins that indicator to the original query. After performing
+ * the outer join, the condition will be TRUE if the EXISTS condition holds,
+ * NULL otherwise.
+ *
+ * @param seek A query, for example 'select * from emp' or
+ * 'values (1,2,3)' or '('Foo', 34)'.
+ * @param subqueryType Whether sub-query is IN, EXISTS or scalar
+ * @param logic Whether the answer needs to be in full 3-valued logic (TRUE,
+ * FALSE, UNKNOWN) will be required, or whether we can accept an
+ * approximation (say representing UNKNOWN as FALSE)
+ * @param needsOuterJoin Whether an outer join is needed
+ * @return join expression
+ * @pre extraExpr == null || extraName != null
+ */
+ private Pair<RelNode, Boolean> convertExists(SqlNode seek, RelOptUtil.SubqueryType subqueryType, RelOptUtil.Logic logic, boolean needsOuterJoin) {
+ final SqlValidatorScope seekScope = (seek instanceof SqlSelect) ? validator.getSelectScope((SqlSelect) seek) : null;
+ final Blackboard seekBb = createBlackboard(seekScope, null);
+ RelNode seekRel = convertQueryOrInList(seekBb, seek);
+
+ return RelOptUtil.createExistsPlan(seekRel, subqueryType, logic, needsOuterJoin);
+ }
+
+ private RelNode convertQueryOrInList(Blackboard bb, SqlNode seek) {
+ // NOTE: Once we start accepting single-row queries as row constructors,
+ // there will be an ambiguity here for a case like X IN ((SELECT Y FROM
+ // Z)). The SQL standard resolves the ambiguity by saying that a lone
+ // select should be interpreted as a table expression, not a row
+ // expression. The semantic difference is that a table expression can
+ // return multiple rows.
+ if (seek instanceof SqlNodeList) {
+ return convertRowValues(bb, seek, ((SqlNodeList) seek).getList(), false, null);
+ } else {
+ return convertQueryRecursive(seek, false, null);
+ }
+ }
+
+ private RelNode convertRowValues(Blackboard bb, SqlNode rowList, Collection<SqlNode> rows, boolean allowLiteralsOnly, RelDataType targetRowType) {
+ // NOTE jvs 30-Apr-2006: We combine all rows consisting entirely of
+ // literals into a single LogicalValues; this gives the optimizer a smaller
+ // input tree. For everything else (computed expressions, row
+ // subqueries), we union each row in as a projection on top of a
+ // LogicalOneRow.
+
+ final ImmutableList.Builder<ImmutableList<RexLiteral>> tupleList = ImmutableList.builder();
+ final RelDataType rowType;
+ if (targetRowType != null) {
+ rowType = targetRowType;
+ } else {
+ rowType = SqlTypeUtil.promoteToRowType(typeFactory, validator.getValidatedNodeType(rowList), null);
+ }
+
+ final List<RelNode> unionInputs = new ArrayList<>();
+ for (SqlNode node : rows) {
+ SqlBasicCall call;
+ if (isRowConstructor(node)) {
+ call = (SqlBasicCall) node;
+ ImmutableList.Builder<RexLiteral> tuple = ImmutableList.builder();
+ for (Ord<SqlNode> operand : Ord.zip(call.operands)) {
+ RexLiteral rexLiteral = convertLiteralInValuesList(operand.e, bb, rowType, operand.i);
+ if ((rexLiteral == null) && allowLiteralsOnly) {
+ return null;
+ }
+ if ((rexLiteral == null) || !shouldCreateValuesRel) {
+ // fallback to convertRowConstructor
+ tuple = null;
+ break;
+ }
+ tuple.add(rexLiteral);
+ }
+ if (tuple != null) {
+ tupleList.add(tuple.build());
+ continue;
+ }
+ } else {
+ RexLiteral rexLiteral = convertLiteralInValuesList(node, bb, rowType, 0);
+ if ((rexLiteral != null) && shouldCreateValuesRel) {
+ tupleList.add(ImmutableList.of(rexLiteral));
+ continue;
+ } else {
+ if ((rexLiteral == null) && allowLiteralsOnly) {
+ return null;
+ }
+ }
+
+ // convert "1" to "row(1)"
+ call = (SqlBasicCall) SqlStdOperatorTable.ROW.createCall(SqlParserPos.ZERO, node);
+ }
+ unionInputs.add(convertRowConstructor(bb, call));
+ }
+ LogicalValues values = LogicalValues.create(cluster, rowType, tupleList.build());
+ RelNode resultRel;
+ if (unionInputs.isEmpty()) {
+ resultRel = values;
+ } else {
+ if (!values.getTuples().isEmpty()) {
+ unionInputs.add(values);
+ }
+ resultRel = LogicalUnion.create(unionInputs, true);
+ }
+ leaves.add(resultRel);
+ return resultRel;
+ }
+
+ private RexLiteral convertLiteralInValuesList(SqlNode sqlNode, Blackboard bb, RelDataType rowType, int iField) {
+ if (!(sqlNode instanceof SqlLiteral)) {
+ return null;
+ }
+ RelDataTypeField field = rowType.getFieldList().get(iField);
+ RelDataType type = field.getType();
+ if (type.isStruct()) {
+ // null literals for weird stuff like UDT's need
+ // special handling during type flattening, so
+ // don't use LogicalValues for those
+ return null;
+ }
+
+ RexNode literalExpr = exprConverter.convertLiteral(bb, (SqlLiteral) sqlNode);
+
+ if (!(literalExpr instanceof RexLiteral)) {
+ assert literalExpr.isA(SqlKind.CAST);
+ RexNode child = ((RexCall) literalExpr).getOperands().get(0);
+ assert RexLiteral.isNullLiteral(child);
+
+ // NOTE jvs 22-Nov-2006: we preserve type info
+ // in LogicalValues digest, so it's OK to lose it here
+ return (RexLiteral) child;
+ }
+
+ RexLiteral literal = (RexLiteral) literalExpr;
+
+ Comparable value = literal.getValue();
+
+ if (SqlTypeUtil.isExactNumeric(type)) {
+ BigDecimal roundedValue = NumberUtil.rescaleBigDecimal((BigDecimal) value, type.getScale());
+ return rexBuilder.makeExactLiteral(roundedValue, type);
+ }
+
+ if ((value instanceof NlsString) && (type.getSqlTypeName() == SqlTypeName.CHAR)) {
+ // pad fixed character type
+ NlsString unpadded = (NlsString) value;
+ return rexBuilder.makeCharLiteral(new NlsString(Util.rpad(unpadded.getValue(), type.getPrecision()), unpadded.getCharsetName(), unpadded.getCollation()));
+ }
+ return literal;
+ }
+
+ private boolean isRowConstructor(SqlNode node) {
+ if (!(node.getKind() == SqlKind.ROW)) {
+ return false;
+ }
+ SqlCall call = (SqlCall) node;
+ return call.getOperator().getName().equalsIgnoreCase("row");
+ }
+
+ /**
+ * Builds a list of all <code>IN</code> or <code>EXISTS</code> operators
+ * inside SQL parse tree. Does not traverse inside queries.
+ *
+ * @param bb blackboard
+ * @param node the SQL parse tree
+ * @param logic Whether the answer needs to be in full 3-valued logic (TRUE,
+ * FALSE, UNKNOWN) will be required, or whether we can accept
+ * an approximation (say representing UNKNOWN as FALSE)
+ * @param registerOnlyScalarSubqueries if set to true and the parse tree
+ * corresponds to a variation of a select
+ * node, only register it if it's a scalar
+ * subquery
+ */
+ private void findSubqueries(Blackboard bb, SqlNode node, RelOptUtil.Logic logic, boolean registerOnlyScalarSubqueries) {
+ final SqlKind kind = node.getKind();
+ switch (kind) {
+ case EXISTS:
+ case SELECT:
+ case MULTISET_QUERY_CONSTRUCTOR:
+ case MULTISET_VALUE_CONSTRUCTOR:
+ case CURSOR:
+ case SCALAR_QUERY:
+ if (!registerOnlyScalarSubqueries || (kind == SqlKind.SCALAR_QUERY)) {
+ bb.registerSubquery(node, RelOptUtil.Logic.TRUE_FALSE);
+ }
+ return;
+ case IN:
+ if (((SqlCall) node).getOperator() == SqlStdOperatorTable.NOT_IN) {
+ logic = logic.negate();
+ }
+ break;
+ case NOT:
+ logic = logic.negate();
+ break;
+ }
+ if (node instanceof SqlCall) {
+ if (kind == SqlKind.OR || kind == SqlKind.NOT) {
+ // It's always correct to outer join subquery with
+ // containing query; however, when predicates involve Or
+ // or NOT, outer join might be necessary.
+ bb.subqueryNeedsOuterJoin = true;
+ }
+ for (SqlNode operand : ((SqlCall) node).getOperandList()) {
+ if (operand != null) {
+ // In the case of an IN expression, locate scalar
+ // subqueries so we can convert them to constants
+ findSubqueries(bb, operand, logic, kind == SqlKind.IN || registerOnlyScalarSubqueries);
+ }
+ }
+ } else if (node instanceof SqlNodeList) {
+ for (SqlNode child : (SqlNodeList) node) {
+ findSubqueries(bb, child, logic, kind == SqlKind.IN || registerOnlyScalarSubqueries);
+ }
+ }
+
+ // Now that we've located any scalar subqueries inside the IN
+ // expression, register the IN expression itself. We need to
+ // register the scalar subqueries first so they can be converted
+ // before the IN expression is converted.
+ if (kind == SqlKind.IN) {
+ if (logic == RelOptUtil.Logic.TRUE_FALSE_UNKNOWN && !validator.getValidatedNodeType(node).isNullable()) {
+ logic = RelOptUtil.Logic.UNKNOWN_AS_FALSE;
+ }
+ // TODO: This conversion is only valid in the WHERE clause
+ if (logic == RelOptUtil.Logic.UNKNOWN_AS_FALSE && !bb.subqueryNeedsOuterJoin) {
+ logic = RelOptUtil.Logic.TRUE;
+ }
+ bb.registerSubquery(node, logic);
+ }
+ }
+
+ /**
+ * Converts an expression from {@link SqlNode} to {@link RexNode} format.
+ *
+ * @param node Expression to translate
+ * @return Converted expression
+ */
+ public RexNode convertExpression(SqlNode node) {
+ Map<String, RelDataType> nameToTypeMap = Collections.emptyMap();
+ Blackboard bb = createBlackboard(new ParameterScope((SqlValidatorImpl) validator, nameToTypeMap), null);
+ return bb.convertExpression(node);
+ }
+
+ /**
+ * Converts an expression from {@link SqlNode} to {@link RexNode} format,
+ * mapping identifier references to predefined expressions.
+ *
+ * @param node Expression to translate
+ * @param nameToNodeMap map from String to {@link RexNode}; when an
+ * {@link SqlIdentifier} is encountered, it is used as a
+ * key and translated to the corresponding value from
+ * this map
+ * @return Converted expression
+ */
+ public RexNode convertExpression(SqlNode node, Map<String, RexNode> nameToNodeMap) {
+ final Map<String, RelDataType> nameToTypeMap = new HashMap<>();
+ for (Map.Entry<String, RexNode> entry : nameToNodeMap.entrySet()) {
+ nameToTypeMap.put(entry.getKey(), entry.getValue().getType());
+ }
+ Blackboard bb = createBlackboard(new ParameterScope((SqlValidatorImpl) validator, nameToTypeMap), nameToNodeMap);
+ return bb.convertExpression(node);
+ }
+
+ /**
+ * Converts a non-standard expression.
+ *
+ * <p>This method is an extension-point that derived classes can override. If
+ * this method returns a null result, the normal expression translation
+ * process will proceed. The default implementation always returns null.
+ *
+ * @param node Expression
+ * @param bb Blackboard
+ * @return null to proceed with the usual expression translation process
+ */
+ protected RexNode convertExtendedExpression(SqlNode node, Blackboard bb) {
+ return null;
+ }
+
+ private RexNode convertOver(Blackboard bb, SqlNode node) {
+ SqlCall call = (SqlCall) node;
+ SqlCall aggCall = call.operand(0);
+ SqlNode windowOrRef = call.operand(1);
+ final SqlWindow window = validator.resolveWindow(windowOrRef, bb.scope, true);
+ // ROW_NUMBER() expects specific kind of framing.
+ if (aggCall.getOperator() == SqlStdOperatorTable.ROW_NUMBER) {
+ window.setLowerBound(SqlWindow.createUnboundedPreceding(SqlParserPos.ZERO));
+ window.setUpperBound(SqlWindow.createCurrentRow(SqlParserPos.ZERO));
+ window.setRows(SqlLiteral.createBoolean(true, SqlParserPos.ZERO));
+ }
+ final SqlNodeList partitionList = window.getPartitionList();
+ final ImmutableList.Builder<RexNode> partitionKeys = ImmutableList.builder();
+ for (SqlNode partition : partitionList) {
+ partitionKeys.add(bb.convertExpression(partition));
+ }
+ RexNode lowerBound = bb.convertExpression(window.getLowerBound());
+ RexNode upperBound = bb.convertExpression(window.getUpperBound());
+ SqlNodeList orderList = window.getOrderList();
+ if ((orderList.size() == 0) && !window.isRows()) {
+ // A logical range requires an ORDER BY clause. Use the implicit
+ // ordering of this relation. There must be one, otherwise it would
+ // have failed validation.
+ orderList = bb.scope.getOrderList();
+ if (orderList == null) {
+ throw new AssertionError("Relation should have sort key for implicit ORDER BY");
+ }
+ }
+ final ImmutableList.Builder<RexFieldCollation> orderKeys = ImmutableList.builder();
+ final Set<SqlKind> flags = EnumSet.noneOf(SqlKind.class);
+ for (SqlNode order : orderList) {
+ flags.clear();
+ RexNode e = bb.convertSortExpression(order, flags);
+ orderKeys.add(new RexFieldCollation(e, flags));
+ }
+ try {
+ Util.permAssert(bb.window == null, "already in window agg mode");
+ bb.window = window;
+ RexNode rexAgg = exprConverter.convertCall(bb, aggCall);
+ rexAgg = rexBuilder.ensureType(validator.getValidatedNodeType(call), rexAgg, false);
+
+ // Walk over the tree and apply 'over' to all agg functions. This is
+ // necessary because the returned expression is not necessarily a call
+ // to an agg function. For example, AVG(x) becomes SUM(x) / COUNT(x).
+ final RexShuttle visitor = new HistogramShuttle(partitionKeys.build(), orderKeys.build(), RexWindowBound.create(window.getLowerBound(), lowerBound), RexWindowBound.create(window.getUpperBound(), upperBound), window);
+ return rexAgg.accept(visitor);
+ } finally {
+ bb.window = null;
+ }
+ }
+
+ /**
+ * Converts a FROM clause into a relational expression.
+ *
+ * @param bb Scope within which to resolve identifiers
+ * @param from FROM clause of a query. Examples include:
+ *
+ * <ul>
+ * <li>a single table ("SALES.EMP"),
+ * <li>an aliased table ("EMP AS E"),
+ * <li>a list of tables ("EMP, DEPT"),
+ * <li>an ANSI Join expression ("EMP JOIN DEPT ON EMP.DEPTNO =
+ * DEPT.DEPTNO"),
+ * <li>a VALUES clause ("VALUES ('Fred', 20)"),
+ * <li>a query ("(SELECT * FROM EMP WHERE GENDER = 'F')"),
+ * <li>or any combination of the above.
+ * </ul>
+ */
+ protected void convertFrom(Blackboard bb, SqlNode from) {
+ SqlCall call;
+ final SqlNode[] operands;
+ switch (from.getKind()) {
+ case AS:
+ operands = ((SqlBasicCall) from).getOperands();
+ convertFrom(bb, operands[0]);
+ return;
+
+ case WITH_ITEM:
+ convertFrom(bb, ((SqlWithItem) from).query);
+ return;
+
+ case WITH:
+ convertFrom(bb, ((SqlWith) from).body);
+ return;
+
+ case TABLESAMPLE:
+ operands = ((SqlBasicCall) from).getOperands();
+ SqlSampleSpec sampleSpec = SqlLiteral.sampleValue(operands[1]);
+ if (sampleSpec instanceof SqlSampleSpec.SqlSubstitutionSampleSpec) {
+ String sampleName = ((SqlSampleSpec.SqlSubstitutionSampleSpec) sampleSpec).getName();
+ datasetStack.push(sampleName);
+ convertFrom(bb, operands[0]);
+ datasetStack.pop();
+ } else if (sampleSpec instanceof SqlSampleSpec.SqlTableSampleSpec) {
+ SqlSampleSpec.SqlTableSampleSpec tableSampleSpec = (SqlSampleSpec.SqlTableSampleSpec) sampleSpec;
+ convertFrom(bb, operands[0]);
+ RelOptSamplingParameters params = new RelOptSamplingParameters(tableSampleSpec.isBernoulli(), tableSampleSpec.getSamplePercentage(), tableSampleSpec.isRepeatable(), tableSampleSpec.getRepeatableSeed());
+ bb.setRoot(new Sample(cluster, bb.root, params), false);
+ } else {
+ throw Util.newInternal("unknown TABLESAMPLE type: " + sampleSpec);
+ }
+ return;
+
+ case IDENTIFIER:
+ final SqlValidatorNamespace fromNamespace = validator.getNamespace(from).resolve();
+ if (fromNamespace.getNode() != null) {
+ convertFrom(bb, fromNamespace.getNode());
+ return;
+ }
+ final String datasetName = datasetStack.isEmpty() ? null : datasetStack.peek();
+ boolean[] usedDataset = { false };
+ RelOptTable table = SqlValidatorUtil.getRelOptTable(fromNamespace, catalogReader, datasetName, usedDataset);
+ final RelNode tableRel;
+ if (shouldConvertTableAccess) {
+ tableRel = toRel(table);
+ } else {
+ tableRel = LogicalTableScan.create(cluster, table);
+ }
+ bb.setRoot(tableRel, true);
+ if (usedDataset[0]) {
+ bb.setDataset(datasetName);
+ }
+ return;
+
+ case JOIN:
+ final SqlJoin join = (SqlJoin) from;
+ final Blackboard fromBlackboard = createBlackboard(validator.getJoinScope(from), null);
+ SqlNode left = join.getLeft();
+ SqlNode right = join.getRight();
+ final boolean isNatural = join.isNatural();
+ final JoinType joinType = join.getJoinType();
+ final Blackboard leftBlackboard = createBlackboard(Util.first(validator.getJoinScope(left), ((DelegatingScope) bb.scope).getParent()), null);
+ final Blackboard rightBlackboard = createBlackboard(Util.first(validator.getJoinScope(right), ((DelegatingScope) bb.scope).getParent()), null);
+ convertFrom(leftBlackboard, left);
+ RelNode leftRel = leftBlackboard.root;
+ convertFrom(rightBlackboard, right);
+ RelNode rightRel = rightBlackboard.root;
+ JoinRelType convertedJoinType = convertJoinType(joinType);
+ RexNode conditionExp;
+ final SqlValidatorNamespace leftNamespace = validator.getNamespace(left);
+ final SqlValidatorNamespace rightNamespace = validator.getNamespace(right);
+ if (isNatural) {
+ final RelDataType leftRowType = leftNamespace.getRowType();
+ final RelDataType rightRowType = rightNamespace.getRowType();
+ final List<String> columnList = SqlValidatorUtil.deriveNaturalJoinColumnList(leftRowType, rightRowType);
+ conditionExp = convertUsing(leftNamespace, rightNamespace, columnList);
+ } else {
+ conditionExp = convertJoinCondition(fromBlackboard, leftNamespace, rightNamespace, join.getCondition(), join.getConditionType(), leftRel, rightRel);
+ }
+
+ final RelNode joinRel = createJoin(fromBlackboard, leftRel, rightRel, conditionExp, convertedJoinType);
+ bb.setRoot(joinRel, false);
+ return;
+
+ case SELECT:
+ case INTERSECT:
+ case EXCEPT:
+ case UNION:
+ final RelNode rel = convertQueryRecursive(from, false, null);
+ bb.setRoot(rel, true);
+ return;
+
+ case VALUES:
+ convertValuesImpl(bb, (SqlCall) from, null);
+ return;
+
+ case UNNEST:
+ final SqlNode node = ((SqlCall) from).operand(0);
+ replaceSubqueries(bb, node, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
+ final RelNode childRel = RelOptUtil.createProject((null != bb.root) ? bb.root : LogicalValues.createOneRow(cluster), Collections.singletonList(bb.convertExpression(node)), Collections.singletonList(validator.deriveAlias(node, 0)), true);
+
+ Uncollect uncollect = new Uncollect(cluster, cluster.traitSetOf(Convention.NONE), childRel);
+ bb.setRoot(uncollect, true);
+ return;
+
+ case COLLECTION_TABLE:
+ call = (SqlCall) from;
+
+ // Dig out real call; TABLE() wrapper is just syntactic.
+ assert call.getOperandList().size() == 1;
+ call = call.operand(0);
+ convertCollectionTable(bb, call);
+ return;
+
+ default:
+ throw Util.newInternal("not a join operator " + from);
+ }
+ }
+
+ protected void convertCollectionTable(Blackboard bb, SqlCall call) {
+ final SqlOperator operator = call.getOperator();
+ if (operator == SqlStdOperatorTable.TABLESAMPLE) {
+ final String sampleName = SqlLiteral.stringValue(call.operand(0));
+ datasetStack.push(sampleName);
+ SqlCall cursorCall = call.operand(1);
+ SqlNode query = cursorCall.operand(0);
+ RelNode converted = convertQuery(query, false, false);
+ bb.setRoot(converted, false);
+ datasetStack.pop();
+ return;
+ }
+ replaceSubqueries(bb, call, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
+
+ // Expand table macro if possible. It's more efficient than
+ // LogicalTableFunctionScan.
+ if (operator instanceof SqlUserDefinedTableMacro) {
+ final SqlUserDefinedTableMacro udf = (SqlUserDefinedTableMacro) operator;
+ final TranslatableTable table = udf.getTable(typeFactory, call.getOperandList());
+ final RelDataType rowType = table.getRowType(typeFactory);
+ RelOptTable relOptTable = RelOptTableImpl.create(null, rowType, table);
+ RelNode converted = toRel(relOptTable);
+ bb.setRoot(converted, true);
+ return;
+ }
+
+ Type elementType;
+ if (operator instanceof SqlUserDefinedTableFunction) {
+ SqlUserDefinedTableFunction udtf = (SqlUserDefinedTableFunction) operator;
+ elementType = udtf.getElementType(typeFactory, call.getOperandList());
+ } else {
+ elementType = null;
+ }
+
+ RexNode rexCall = bb.convertExpression(call);
+ final List<RelNode> inputs = bb.retrieveCursors();
+ Set<RelColumnMapping> columnMappings = getColumnMappings(operator);
+ LogicalTableFunctionScan callRel = LogicalTableFunctionScan.create(cluster, inputs, rexCall, elementType, validator.getValidatedNodeType(call), columnMappings);
+ bb.setRoot(callRel, true);
+ afterTableFunction(bb, call, callRel);
+ }
+
+ protected void afterTableFunction(SqlToRelConverter.Blackboard bb, SqlCall call, LogicalTableFunctionScan callRel) {
+ }
+
+ private Set<RelColumnMapping> getColumnMappings(SqlOperator op) {
+ SqlReturnTypeInference rti = op.getReturnTypeInference();
+ if (rti == null) {
+ return null;
+ }
+ if (rti instanceof TableFunctionReturnTypeInference) {
+ TableFunctionReturnTypeInference tfrti = (TableFunctionReturnTypeInference) rti;
+ return tfrti.getColumnMappings();
+ } else {
+ return null;
+ }
+ }
+
+ protected RelNode createJoin(Blackboard bb, RelNode leftRel, RelNode rightRel, RexNode joinCond, JoinRelType joinType) {
+ assert joinCond != null;
+
+ Set<String> correlatedVariables = RelOptUtil.getVariablesUsed(rightRel);
+ if (correlatedVariables.size() > 0) {
+ final ImmutableBitSet.Builder requiredColumns = ImmutableBitSet.builder();
+ final List<String> correlNames = Lists.newArrayList();
+
+ // All correlations must refer the same namespace since correlation
+ // produces exactly one correlation source.
+ // The same source might be referenced by different variables since
+ // DeferredLookups are not de-duplicated at create time.
+ SqlValidatorNamespace prevNs = null;
+
+ for (String correlName : correlatedVariables) {
+ DeferredLookup lookup = mapCorrelToDeferred.get(correlName);
+ RexFieldAccess fieldAccess = lookup.getFieldAccess(correlName);
+ String originalRelName = lookup.getOriginalRelName();
+ String originalFieldName = fieldAccess.getField().getName();
+
+ int[] nsIndexes = { -1 };
+ final SqlValidatorScope[] ancestorScopes = { null };
+ SqlValidatorNamespace foundNs = lookup.bb.scope.resolve(ImmutableList.of(originalRelName), ancestorScopes, nsIndexes);
+
+ assert foundNs != null;
+ assert nsIndexes.length == 1;
+
+ int childNamespaceIndex = nsIndexes[0];
+
+ SqlValidatorScope ancestorScope = ancestorScopes[0];
+ boolean correlInCurrentScope = ancestorScope == bb.scope;
+
+ if (!correlInCurrentScope) {
+ continue;
+ }
+
+ if (prevNs == null) {
+ prevNs = foundNs;
+ } else {
+ assert prevNs == foundNs : "All correlation variables should resolve" + " to the same namespace." + " Prev ns=" + prevNs + ", new ns=" + foundNs;
+ }
+
+ int namespaceOffset = 0;
+ if (childNamespaceIndex > 0) {
+ // If not the first child, need to figure out the width
+ // of output types from all the preceding namespaces
+ assert ancestorScope instanceof ListScope;
+ List<SqlValidatorNamespace> children = ((ListScope) ancestorScope).getChildren();
+
+ for (int i = 0; i < childNamespaceIndex; i++) {
+ SqlValidatorNamespace child = children.get(i);
+ namespaceOffset += child.getRowType().getFieldCount();
+ }
+ }
+
+ RelDataTypeField field = catalogReader.field(foundNs.getRowType(), originalFieldName);
+ int pos = namespaceOffset + field.getIndex();
+
+ assert field.getType() == lookup.getFieldAccess(correlName).getField().getType();
+
+ assert pos != -1;
+
+ if (bb.mapRootRelToFieldProjection.containsKey(bb.root)) {
+ // bb.root is an aggregate and only projects group by
+ // keys.
+ Map<Integer, Integer> exprProjection = bb.mapRootRelToFieldProjection.get(bb.root);
+
+ // subquery can reference group by keys projected from
+ // the root of the outer relation.
+ if (exprProjection.containsKey(pos)) {
+ pos = exprProjection.get(pos);
+ } else {
+ // correl not grouped
+ throw Util.newInternal("Identifier '" + originalRelName + "." + originalFieldName + "' is not a group expr");
+ }
+ }
+
+ requiredColumns.set(pos);
+ correlNames.add(correlName);
+ }
+
+ if (!correlNames.isEmpty()) {
+ if (correlNames.size() > 1) {
+ // The same table was referenced more than once.
+ // So we deduplicate
+ RelShuttle dedup = new DeduplicateCorrelateVariables(rexBuilder, correlNames.get(0), ImmutableSet.copyOf(Util.skip(correlNames)));
+ rightRel = rightRel.accept(dedup);
+ }
+ LogicalCorrelate corr = LogicalCorrelate.create(leftRel, rightRel, new CorrelationId(correlNames.get(0)), requiredColumns.build(), SemiJoinType.of(joinType));
+ if (!joinCond.isAlwaysTrue()) {
+ return RelOptUtil.createFilter(corr, joinCond);
+ }
+ return corr;
+ }
+ }
+
+ final Join originalJoin = (Join) RelFactories.DEFAULT_JOIN_FACTORY.createJoin(leftRel, rightRel, joinCond, joinType, ImmutableSet.<String> of(), false);
+
+ return RelOptUtil.pushDownJoinConditions(originalJoin);
+ }
+
+ private static boolean containsGet(RexNode node) {
+ try {
+ node.accept(new RexVisitorImpl<Void>(true) {
+ @Override
+ public Void visitCall(RexCall call) {
+ if (call.getOperator() == RexBuilder.GET_OPERATOR) {
+ throw Util.FoundOne.NULL;
+ }
+ return super.visitCall(call);
+ }
+ });
+ return false;
+ } catch (Util.FoundOne e) {
+ return true;
+ }
+ }
+
+ /**
+ * Determines whether a subquery is non-correlated. Note that a
+ * non-correlated subquery can contain correlated references, provided those
+ * references do not reference select statements that are parents of the
+ * subquery.
+ *
+ * @param subq the subquery
+ * @param bb blackboard used while converting the subquery, i.e., the
+ * blackboard of the parent query of this subquery
+ * @return true if the subquery is non-correlated.
+ */
+ private boolean isSubQueryNonCorrelated(RelNode subq, Blackboard bb) {
+ Set<String> correlatedVariables = RelOptUtil.getVariablesUsed(subq);
+ for (String correlName : correlatedVariables) {
+ DeferredLookup lookup = mapCorrelToDeferred.get(correlName);
+ String originalRelName = lookup.getOriginalRelName();
+
+ int[] nsIndexes = { -1 };
+ final SqlValidatorScope[] ancestorScopes = { null };
+ SqlValidatorNamespace foundNs = lookup.bb.scope.resolve(ImmutableList.of(originalRelName), ancestorScopes, nsIndexes);
+
+ assert foundNs != null;
+ assert nsIndexes.length == 1;
+
+ SqlValidatorScope ancestorScope = ancestorScopes[0];
+
+ // If the correlated reference is in a scope that's "above" the
+ // subquery, then this is a correlated subquery.
+ SqlValidatorScope parentScope = bb.scope;
+ do {
+ if (ancestorScope == parentScope) {
+ return false;
+ }
+ if (parentScope instanceof DelegatingScope) {
+ parentScope = ((DelegatingScope) parentScope).getParent();
+ } else {
+ break;
+ }
+ } while (parentScope != null);
+ }
+ return true;
+ }
+
+ /**
+ * Returns a list of fields to be prefixed to each relational expression.
+ *
+ * @return List of system fields
+ */
+ protected List<RelDataTypeField> getSystemFields() {
+ return Collections.emptyList();
+ }
+
+ private RexNode convertJoinCondition(Blackboard bb, SqlValidatorNamespace leftNamespace, SqlValidatorNamespace rightNamespace, SqlNode condition, JoinConditionType conditionType, RelNode leftRel, RelNode rightRel) {
+ if (condition == null) {
+ return rexBuilder.makeLiteral(true);
+ }
+ bb.setRoot(ImmutableList.of(leftRel, rightRel));
+ replaceSubqueries(bb, condition, RelOptUtil.Logic.UNKNOWN_AS_FALSE);
+ switch (conditionType) {
+ case ON:
+ bb.setRoot(ImmutableList.of(leftRel, rightRel));
+ return bb.convertExpression(condition);
+ case USING:
+ final SqlNodeList list = (SqlNodeList) condition;
+ final List<String> nameList = new ArrayList<>();
+ for (SqlNode columnName : list) {
+ final SqlIdentifier id = (SqlIdentifier) columnName;
+ String name = id.getSimple();
+ nameList.add(name);
+ }
+ return convertUsing(leftNamespace, rightNamespace, nameList);
+ default:
+ throw Util.unexpected(conditionType);
+ }
+ }
+
+ /**
+ * Returns an expression for matching columns of a USING clause or inferred
+ * from NATURAL JOIN. "a JOIN b USING (x, y)" becomes "a.x = b.x AND a.y =
+ * b.y". Returns null if the column list is empty.
+ *
+ * @param leftNamespace Namespace of left input to join
+ * @param rightNamespace Namespace of right input to join
+ * @param nameList List of column names to join on
+ * @return Expression to match columns from name list, or true if name list
+ * is empty
+ */
+ private RexNode convertUsing(SqlValidatorNamespace leftNamespace, SqlValidatorNamespace rightNamespace, List<String> nameList) {
+ final List<RexNode> list = Lists.newArrayList();
+ for (String name : nameList) {
+ List<RexNode> operands = new ArrayList<>();
+ int offset = 0;
+ for (SqlValidatorNamespace n : ImmutableList.of(leftNamespace, rightNamespace)) {
+ final RelDataType rowType = n.getRowType();
+ final RelDataTypeField field = catalogReader.field(rowType, name);
+ operands.add(rexBuilder.makeInputRef(field.getType(), offset + field.getIndex()));
+ offset += rowType.getFieldList().size();
+ }
+ list.add(rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, operands));
+ }
+ return RexUtil.composeConjunction(rexBuilder, list, false);
+ }
+
+ private static JoinRelType convertJoinType(JoinType joinType) {
+ switch (joinType) {
+ case COMMA:
+ case INNER:
+ case CROSS:
+ return JoinRelType.INNER;
+ case FULL:
+ return JoinRelType.FULL;
+ case LEFT:
+ return JoinRelType.LEFT;
+ case RIGHT:
+ return JoinRelType.RIGHT;
+ default:
+ throw Util.unexpected(joinType);
+ }
+ }
+
+ /**
+ * Converts the SELECT, GROUP BY and HAVING clauses of an aggregate query.
+ *
+ * <p>This method extracts SELECT, GROUP BY and HAVING clauses, and creates
+ * an {@link AggConverter}, then delegates to {@link #createAggImpl}.
+ * Derived class may override this method to change any of those clauses or
+ * specify a different {@link AggConverter}.
+ *
+ * @param bb Scope within which to resolve identifiers
+ * @param select Query
+ * @param orderExprList Additional expressions needed to implement ORDER BY
+ */
+ protected void convertAgg(Blackboard bb, SqlSelect select, List<SqlNode> orderExprList) {
+ assert bb.root != null : "precondition: child != null";
+ SqlNodeList groupList = select.getGroup();
+ SqlNodeList selectList = select.getSelectList();
+ SqlNode having = select.getHaving();
+
+ final AggConverter aggConverter = new AggConverter(bb, select);
+ createAggImpl(bb, aggConverter, selectList, groupList, having, orderExprList);
+ }
+
+ protected final void createAggImpl(Blackboard bb, final AggConverter aggConverter, SqlNodeList selectList, SqlNodeList groupList, SqlNode having, List<SqlNode> orderExprList) {
+ // Find aggregate functions in SELECT and HAVING clause
+ final AggregateFinder aggregateFinder = new AggregateFinder();
+ selectList.accept(aggregateFinder);
+ if (having != null) {
+ having.accept(aggregateFinder);
+ }
+
+ // first replace the subqueries inside the aggregates
+ // because they will provide input rows to the aggregates.
+ replaceSubqueries(bb, aggregateFinder.list, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
+
+ // If group-by clause is missing, pretend that it has zero elements.
+ if (groupList == null) {
+ groupList = SqlNodeList.EMPTY;
+ }
+
+ replaceSubqueries(bb, groupList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
+
+ // register the group exprs
+
+ // build a map to remember the projections from the top scope to the
+ // output of the current root.
+ //
+ // Currently farrago allows expressions, not just column references in
+ // group by list. This is not SQL 2003 compliant.
+
+ final AggregatingSelectScope scope = aggConverter.aggregatingSelectScope;
+ for (SqlNode groupExpr : scope.groupExprList) {
+ aggConverter.addGroupExpr(groupExpr);
+ }
+
+ RexNode havingExpr = null;
+ final List<Pair<RexNode, String>> projects = Lists.newArrayList();
+
+ try {
+ Util.permAssert(bb.agg == null, "already in agg mode");
+ bb.agg = aggConverter;
+
+ // convert the select and having expressions, so that the
+ // agg converter knows which aggregations are required
+
+ selectList.accept(aggConverter);
+ for (SqlNode expr : orderExprList) {
+ expr.accept(aggConverter);
+ }
+ if (having != null) {
+ having.accept(aggConverter);
+ }
+
+ // compute inputs to the aggregator
+ List<RexNode> preExprs = aggConverter.getPreExprs();
+ List<String> preNames = aggConverter.getPreNames();
+
+ if (preExprs.size() == 0) {
+ // Special case for COUNT(*), where we can end up with no inputs
+ // at all. The rest of the system doesn't like 0-tuples, so we
+ // select a dummy constant here.
+ preExprs = Collections.singletonList((RexNode) rexBuilder.makeExactLiteral(BigDecimal.ZERO));
+ preNames = Collections.singletonList(null);
+ }
+
+ final RelNode inputRel = bb.root;
+
+ // Project the expressions required by agg and having.
+ bb.setRoot(RelOptUtil.createProject(inputRel, preExprs, preNames, true), false);
+ bb.mapRootRelToFieldProjection.put(bb.root, scope.groupExprProjection);
+
+ // REVIEW jvs 31-Oct-2007: doesn't the declaration of
+ // monotonicity here assume sort-based aggregation at
+ // the physical level?
+
+ // Tell bb which of group columns are sorted.
+ bb.columnMonotonicities.clear();
+ for (SqlNode groupItem : groupList) {
+ bb.columnMonotonicities.add(bb.scope.getMonotonicity(groupItem));
+ }
+
+ // Add the aggregator
+ bb.setRoot(createAggregate(bb, aggConverter.aggregatingSelectScope.indicator, scope.groupSet, scope.groupSets, aggConverter.getAggCalls()), false);
+
+ // Generate NULL values for rolled-up not-null fields.
+ final Aggregate aggregate = (Aggregate) bb.root;
+ if (aggregate.getGroupType() != Aggregate.Group.SIMPLE) {
+ assert aggregate.indicator;
+ List<Pair<RexNode, String>> projects2 = Lists.newArrayList();
+ int converted = 0;
+ final int groupCount = aggregate.getGroupSet().cardinality();
+ for (RelDataTypeField field : aggregate.getRowType().getFieldList()) {
+ final int i = field.getIndex();
+ final RexNode rex;
+ if (i < groupCount && scope.isNullable(i)) {
+ ++converted;
+
+ rex = rexBuilder.makeCall(SqlStdOperatorTable.CASE, rexBuilder.makeInputRef(aggregate, groupCount + i), rexBuilder.makeCast(typeFactory.createTypeWithNullability(field.getType(), true), rexBuilder.constantNull()), rexBuilder.makeInputRef(aggregate, i));
+ } else {
+ rex = rexBuilder.makeInputRef(aggregate, i);
+ }
+ projects2.add(Pair.of(rex, field.getName()));
+ }
+ if (converted > 0) {
+ bb.setRoot(RelOptUtil.createProject(bb.root, projects2, true), false);
+ }
+ }
+
+ bb.mapRootRelToFieldProjection.put(bb.root, scope.groupExprProjection);
+
+ // Replace subqueries in having here and modify having to use
+ // the replaced expressions
+ if (having != null) {
+ SqlNode newHaving = pushDownNotForIn(having);
+ replaceSubqueries(bb, newHaving, RelOptUtil.Logic.UNKNOWN_AS_FALSE);
+ havingExpr = bb.convertExpression(newHaving);
+ if (havingExpr.isAlwaysTrue()) {
+ havingExpr = null;
+ }
+ }
+
+ // Now convert the other subqueries in the select list.
+ // This needs to be done separately from the subquery inside
+ // any aggregate in the select list, and after the aggregate rel
+ // is allocated.
+ replaceSubqueries(bb, selectList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
+
+ // Now subqueries in the entire select list have been converted.
+ // Convert the select expressions to get the final list to be
+ // projected.
+ int k = 0;
+
+ // For select expressions, use the field names previously assigned
+ // by the validator. If we derive afresh, we might generate names
+ // like "EXPR$2" that don't match the names generated by the
+ // validator. This is especially the case when there are system
+ // fields; system fields appear in the relnode's rowtype but do not
+ // (yet) appear in the validator type.
+ final SelectScope selectScope = SqlValidatorUtil.getEnclosingSelectScope(bb.scope);
+ assert selectScope != null;
+ final SqlValidatorNamespace selectNamespace = validator.getNamespace(selectScope.getNode());
+ final List<String> names = selectNamespace.getRowType().getFieldNames();
+ int sysFieldCount = selectList.size() - names.size();
+ for (SqlNode expr : selectList) {
+ projects.add(Pair.of(bb.convertExpression(expr), k < sysFieldCount ? validator.deriveAlias(expr, k++) : names.get(k++ - sysFieldCount)));
+ }
+
+ for (SqlNode expr : orderExprList) {
+ projects.add(Pair.of(bb.convertExpression(expr), validator.deriveAlias(expr, k++)));
+ }
+ } finally {
+ bb.agg = null;
+ }
+
+ // implement HAVING (we have already checked that it is non-trivial)
+ if (havingExpr != null) {
+ bb.setRoot(RelOptUtil.createFilter(bb.root, havingExpr), false);
+ }
+
+ // implement the SELECT list
+ bb.setRoot(RelOptUtil.createProject(bb.root, projects, true), false);
+
+ // Tell bb which of group columns are sorted.
+ bb.columnMonotonicities.clear();
+ for (SqlNode selectItem : selectList) {
+ bb.columnMonotonicities.add(bb.scope.getMonotonicity(selectItem));
+ }
+ }
+
+ /**
+ * Creates an Aggregate.
+ *
+ * <p>In case the aggregate rel changes the order in which it projects
+ * fields, the <code>groupExprProjection</code> parameter is provided, and
+ * the implementation of this method may modify it.
+ *
+ * <p>The <code>sortedCount</code> parameter is the number of expressions
+ * known to be monotonic. These expressions must be on the leading edge of
+ * the grouping keys. The default implementation of this method ignores this
+ * parameter.
+ *
+ * @param bb Blackboard
+ * @param indicator Whether to output fields indicating grouping sets
+ * @param groupSet Bit set of ordinals of grouping columns
+ * @param groupSets Grouping sets
+ * @param aggCalls Array of calls to aggregate functions
+ * @return LogicalAggregate
+ */
+ protected RelNode createAggregate(Blackboard bb, boolean indicator, ImmutableBitSet groupSet, ImmutableList<ImmutableBitSet> groupSets, List<AggregateCall> aggCalls) {
+ return LogicalAggregate.create(bb.root, indicator, groupSet, groupSets, aggCalls);
+ }
+
+ public RexDynamicParam convertDynamicParam(final SqlDynamicParam dynamicParam) {
+ // REVIEW jvs 8-Jan-2005: dynamic params may be encountered out of
+ // order. Should probably cross-check with the count from the parser
+ // at the end and make sure they all got filled in. Why doesn't List
+ // have a resize() method?!? Make this a utility.
+ while (dynamicParam.getIndex() >= dynamicParamSqlNodes.size()) {
+ dynamicParamSqlNodes.add(null);
+ }
+
+ dynamicParamSqlNodes.set(dynamicParam.getIndex(), dynamicParam);
+ return rexBuilder.makeDynamicParam(getDynamicParamType(dynamicParam.getIndex()), dynamicParam.getIndex());
+ }
+
+ /**
+ * Creates a list of collations required to implement the ORDER BY clause,
+ * if there is one. Populates <code>extraOrderExprs</code> with any sort
+ * expressions which are not in the select clause.
+ *
+ * @param bb Scope within which to resolve identifiers
+ * @param select Select clause. Never null, because we invent a
+ * dummy SELECT if ORDER BY is applied to a set
+ * operation (UNION etc.)
+ * @param orderList Order by clause, may be null
+ * @param extraOrderExprs Sort expressions which are not in the select
+ * clause (output)
+ * @param collationList List of collations (output)
+ */
+ protected void gatherOrderExprs(Blackboard bb, SqlSelect select, SqlNodeList orderList, List<SqlNode> extraOrderExprs, List<RelFieldCollation> collationList) {
+ // TODO: add validation rules to SqlValidator also
+ assert bb.root != null : "precondition: child != null";
+ assert select != null;
+ if (orderList == null) {
+ return;
+ }
+ for (SqlNode orderItem : orderList) {
+ collationList.add(convertOrderItem(select, orderItem, extraOrderExprs, RelFieldCollation.Direction.ASCENDING, RelFieldCollation.NullDirection.UNSPECIFIED));
+ }
+ }
+
+ protected RelFieldCollation convertOrderItem(SqlSelect select, SqlNode orderItem, List<SqlNode> extraExprs, RelFieldCollation.Direction direction, RelFieldCollation.NullDirection nullDirection) {
+ assert select != null;
+ // Handle DESC keyword, e.g. 'select a, b from t order by a desc'.
+ switch (orderItem.getKind()) {
+ case DESCENDING:
+ return convertOrderItem(select, ((SqlCall) orderItem).operand(0), extraExprs, RelFieldCollation.Direction.DESCENDING, nullDirection);
+ case NULLS_FIRST:
+ return convertOrderItem(select, ((SqlCall) orderItem).operand(0), extraExprs, direction, RelFieldCollation.NullDirection.FIRST);
+ case NULLS_LAST:
+ return convertOrderItem(select, ((SqlCall) orderItem).operand(0), extraExprs, direction, RelFieldCollation.NullDirection.LAST);
+ }
+
+ SqlNode converted = validator.expandOrderExpr(select, orderItem);
+
+ // Scan the select list and order exprs for an identical expression.
+ final SelectScope selectScope = validator.getRawSelectScope(select);
+ int ordinal = -1;
+ for (SqlNode selectItem : selectScope.getExpandedSelectList()) {
+ ++ordinal;
+ if (converted.equalsDeep(stripAs(selectItem), false)) {
+ return new RelFieldCollation(ordinal, direction, nullDirection);
+ }
+ }
+
+ for (SqlNode extraExpr : extraExprs) {
+ ++ordinal;
+ if (converted.equalsDeep(extraExpr, false)) {
+ return new RelFieldCollation(ordinal, direction, nullDirection);
+ }
+ }
+
+ // TODO: handle collation sequence
+ // TODO: flag expressions as non-standard
+
+ extraExprs.add(converted);
+ return new RelFieldCollation(ordinal + 1, direction, nullDirection);
+ }
+
+ protected boolean enableDecorrelation() {
+ // disable subquery decorrelation when needed.
+ // e.g. if outer joins are not supported.
+ return decorrelationEnabled;
+ }
+
+ protected RelNode decorrelateQuery(RelNode rootRel) {
+ return RelDecorrelator.decorrelateQuery(rootRel);
+ }
+
+ /**
+ * Sets whether to trim unused fields as part of the conversion process.
+ *
+ * @param trim Whether to trim unused fields
+ */
+ public void setTrimUnusedFields(boolean trim) {
+ this.trimUnusedFields = trim;
+ }
+
+ /**
+ * Returns whether to trim unused fields as part of the conversion process.
+ *
+ * @return Whether to trim unused fields
+ */
+ public boolean isTrimUnusedFields() {
+ /* OVERRIDE POINT */
+ return false;
+ }
+
+ /**
+ * Recursively converts a query to a relational expression.
+ *
+ * @param query Query
+ * @param top Whether this query is the top-level query of the
+ * statement
+ * @param targetRowType Target row type, or null
+ * @return Relational expression
+ */
+ protected RelNode convertQueryRecursive(SqlNode query, boolean top, RelDataType targetRowType) {
+ switch (query.getKind()) {
+ case SELECT:
+ return convertSelect((SqlSelect) query);
+ case INSERT:
+ return convertInsert((SqlInsert) query);
+ case DELETE:
+ return convertDelete((SqlDelete) query);
+ case UPDATE:
+ return convertUpdate((SqlUpdate) query);
+ case MERGE:
+ return convertMerge((SqlMerge) query);
+ case UNION:
+ case INTERSECT:
+ case EXCEPT:
+ return convertSetOp((SqlCall) query);
+ case WITH:
+ return convertWith((SqlWith) query);
+ case VALUES:
+ return convertValues((SqlCall) query, targetRowType);
+ default:
+ throw Util.newInternal("not a query: " + query);
+ }
+ }
+
+ /**
+ * Converts a set operation (UNION, INTERSECT, MINUS) into relational
+ * expressions.
+ *
+ * @param call Call to set operator
+ * @return Relational expression
+ */
+ protected RelNode convertSetOp(SqlCall call) {
+ final RelNode left = convertQueryRecursive(call.operand(0), false, null);
+ final RelNode right = convertQueryRecursive(call.operand(1), false, null);
+ boolean all = false;
+ if (call.getOperator() instanceof SqlSetOperator) {
+ all = ((SqlSetOperator) (call.getOperator())).isAll();
+ }
+ switch (call.getKind()) {
+ case UNION:
+ return LogicalUnion.create(ImmutableList.of(left, right), all);
+
+ case INTERSECT:
+ // TODO: all
+ if (!all) {
+ return LogicalIntersect.create(ImmutableList.of(left, right), all);
+ } else {
+ throw Util.newInternal("set operator INTERSECT ALL not suported");
+ }
+
+ case EXCEPT:
+ // TODO: all
+ if (!all) {
+ return LogicalMinus.create(ImmutableList.of(left, right), all);
+ } else {
+ throw Util.newInternal("set operator EXCEPT ALL not suported");
+ }
+
+ default:
+ throw Util.unexpected(call.getKind());
+ }
+ }
+
+ protected RelNode convertInsert(SqlInsert call) {
+ RelOptTable targetTable = getTargetTable(call);
+
+ final RelDataType targetRowType = validator.getValidatedNodeType(call);
+ assert targetRowType != null;
+ RelNode sourceRel = convertQueryRecursive(call.getSource(), false, targetRowType);
+ RelNode massagedRel = convertColumnList(call, sourceRel);
+
+ return createModify(targetTable, massagedRel);
+ }
+
+ /** Creates a relational expression to modify a table or modifiable view. */
+ private RelNode createModify(RelOptTable targetTable, RelNode source) {
+ final ModifiableTable modifiableTable = targetTable.unwrap(ModifiableTable.class);
+ if (modifiableTable != null) {
+ return modifiableTable.toModificationRel(cluster, targetTable, catalogReader, source, LogicalTableModify.Operation.INSERT, null, false);
+ }
+ final ModifiableView modifiableView = targetTable.unwrap(ModifiableView.class);
+ if (modifiableView != null) {
+ final Table delegateTable = modifiableView.getTable();
+ final RelDataType delegateRowType = delegateTable.getRowType(typeFactory);
+ final RelOptTable delegateRelOptTable = RelOptTableImpl.create(null, delegateRowType, delegateTable, modifiableView.getTablePath());
+ final RelNode newSource = createSource(targetTable, source, modifiableView, delegateRowType);
+ return createModify(delegateRelOptTable, newSource);
+ }
+ return LogicalTableModify.create(targetTable, catalogReader, source, LogicalTableModify.Operation.INSERT, null, false);
+ }
+
+ /** Wraps a relational expression in the projects and filters implied by
+ * a {@link ModifiableView}.
+ *
+ * <p>The input relational expression is suitable for inserting into the view,
+ * and the returned relational expression is suitable for inserting into its
+ * delegate table.
+ *
+ * <p>In principle, the delegate table of a view might be another modifiable
+ * view, and if so, the process can be repeated. */
+ private RelNode createSource(RelOptTable targetTable, RelNode source, ModifiableView modifiableView, RelDataType delegateRowType) {
+ final ImmutableIntList mapping = modifiableView.getColumnMapping();
+ assert mapping.size() == targetTable.getRowType().getFieldCount();
+
+ // For columns represented in the mapping, the expression is just a field
+ // reference.
+ final Map<Integer, RexNode> projectMap = new HashMap<>();
+ final List<RexNode> filters = new ArrayList<>();
+ for (int i = 0; i < mapping.size(); i++) {
+ int target = mapping.get(i);
+ if (target >= 0) {
+ projectMap.put(target, RexInputRef.of(i, source.getRowType()));
+ }
+ }
+
+ // For columns that are not in the mapping, and have a constraint of the
+ // form "column = value", the expression is the literal "value".
+ //
+ // If a column has multiple constraints, the extra ones will become a
+ // filter.
+ final RexNode constraint = modifiableView.getConstraint(rexBuilder, delegateRowType);
+ RelOptUtil.inferViewPredicates(projectMap, filters, constraint);
+ final List<Pair<RexNode, String>> projects = new ArrayList<>();
+ for (RelDataTypeField field : delegateRowType.getFieldList()) {
+ RexNode node = projectMap.get(field.getIndex());
+ if (node == null) {
+ node = rexBuilder.makeNullLiteral(field.getType().getSqlTypeName());
+ }
+ projects.add(Pair.of(rexBuilder.ensureType(field.getType(), node, false), field.getName()));
+ }
+
+ source = RelOptUtil.createProject(source, projects, true);
+ if (filters.size() > 0) {
+ source = RelOptUtil.createFilter(source, filters);
+ }
+ return source;
+ }
+
+ private RelOptTable.ToRelContext createToRelContext() {
+ return new RelOptTable.ToRelContext() {
+ public RelOptCluster getCluster() {
+ return cluster;
+ }
+
+ public RelNode expandView(RelDataType rowType, String queryString, List<String> schemaPath) {
+ return viewExpander.expandView(rowType, queryString, schemaPath);
+ }
+ };
+ }
+
+ public RelNode toRel(RelOptTable table) {
+ return table.toRel(createToRelContext());
+ }
+
+ protected RelOptTable getTargetTable(SqlNode call) {
+ SqlValidatorNamespace targetNs = validator.getNamespace(call).resolve();
+ return SqlValidatorUtil.getRelOptTable(targetNs, catalogReader, null, null);
+ }
+
+ /**
+ * Creates a source for an INSERT statement.
+ *
+ * <p>If the column list is not specified, source expressions match target
+ * columns in order.
+ *
+ * <p>If the column list is specified, Source expressions are mapped to
+ * target columns by name via targetColumnList, and may not cover the entire
+ * target table. So, we'll make up a full row, using a combination of
+ * default values and the source expressions provided.
+ *
+ * @param call Insert expression
+ * @param sourceRel Source relational expression
+ * @return Converted INSERT statement
+ */
+ protected RelNode convertColumnList(SqlInsert call, RelNode sourceRel) {
+ RelDataType sourceRowType = sourceRel.getRowType();
+ final RexNode sourceRef = rexBuilder.makeRangeReference(sourceRowType, 0, false);
+ final List<String> targetColumnNames = new ArrayList<>();
+ final List<RexNode> columnExprs = new ArrayList<>();
+ collectInsertTargets(call, sourceRef, targetColumnNames, columnExprs);
+
+ final RelOptTable targetTable = getTargetTable(call);
+ final RelDataType targetRowType = targetTable.getRowType();
+ final List<RelDataTypeField> targetFields = targetRowType.getFieldList();
+ final List<RexNode> sourceExps = new ArrayList<>(Collections.<RexNode> nCopies(targetFields.size(), null));
+ final List<String> fieldNames = new ArrayList<>(Collections.<String> nCopies(targetFields.size(), null));
+
+ // Walk the name list and place the associated value in the
+ // expression list according to the ordinal value returned from
+ // the table construct, leaving nulls in the list for columns
+ // that are not referenced.
+ for (Pair<String, RexNode> p : Pair.zip(targetColumnNames, columnExprs)) {
+ RelDataTypeField field = catalogReader.field(targetRowType, p.left);
+ assert field != null : "column " + p.left + " not found";
+ sourceExps.set(field.getIndex(), p.right);
+ }
+
+ // Walk the expression list and get default values for any columns
+ // that were not supplied in the statement. Get field names too.
+ for (int i = 0; i < targetFields.size(); ++i) {
+ final RelDataTypeField field = targetFields.get(i);
+ final String fieldName = field.getName();
+ fieldNames.set(i, fieldName);
+ if (sourceExps.get(i) != null) {
+ if (defaultValueFactory.isGeneratedAlways(targetTable, i)) {
+ throw RESOURCE.insertIntoAlwaysGenerated(fieldName).ex();
+ }
+ continue;
+ }
+ sourceExps.set(i, defaultValueFactory.newColumnDefaultValue(targetTable, i));
+
+ // bare nulls are dangerous in the wrong hands
+ sourceExps.set(i, castNullLiteralIfNeeded(sourceExps.get(i), field.getType()));
+ }
+
+ return RelOptUtil.createProject(sourceRel, sourceExps, fieldNames, true);
+ }
+
+ private RexNode castNullLiteralIfNeeded(RexNode node, RelDataType type) {
+ if (!RexLiteral.isNullLiteral(node)) {
+ return node;
+ }
+ return rexBuilder.makeCast(type, node);
+ }
+
+ /**
+ * Given an INSERT statement, collects the list of names to be populated and
+ * the expressions to put in them.
+ *
+ * @param call Insert statement
+ * @param sourceRef Expression representing a row from the source
+ * relational expression
+ * @param targetColumnNames List of target column names, to be populated
+ * @param columnExprs List of expressions, to be populated
+ */
+ protected void collectInsertTargets(SqlInsert call, final RexNode sourceRef, final List<String> targetColumnNames, List<RexNode> columnExprs) {
+ final RelOptTable targetTable = getTargetTable(call);
+ final RelDataType targetRowType = targetTable.getRowType();
+ SqlNodeList targetColumnList = call.getTargetColumnList();
+ if (targetColumnList == null) {
+ targetColumnNames.addAll(targetRowType.getFieldNames());
+ } else {
+ for (int i = 0; i < targetColumnList.size(); i++) {
+ SqlIdentifier id = (SqlIdentifier) targetColumnList.get(i);
+ targetColumnNames.add(id.getSimple());
+ }
+ }
+
+ for (int i = 0; i < targetColumnNames.size(); i++) {
+ final RexNode expr = rexBuilder.makeFieldAccess(sourceRef, i);
+ columnExprs.add(expr);
+ }
+ }
+
+ private RelNode convertDelete(SqlDelete call) {
+ RelOptTable targetTable = getTargetTable(call);
+ RelNode sourceRel = convertSelect(call.getSourceSelect());
+ return LogicalTableModify.create(targetTable, catalogReader, sourceRel, LogicalTableModify.Operation.DELETE, null, false);
+ }
+
+ private RelNode convertUpdate(SqlUpdate call) {
+ RelOptTable targetTable = getTargetTable(call);
+
+ // convert update column list from SqlIdentifier to String
+ final List<String> targetColumnNameList = new ArrayList<>();
+ for (SqlNode node : call.getTargetColumnList()) {
+ SqlIdentifier id = (SqlIdentifier) node;
+ String name = id.getSimple();
+ targetColumnNameList.add(name);
+ }
+
+ RelNode sourceRel = convertSelect(call.getSourceSelect());
+
+ return LogicalTableModify.create(targetTable, catalogReader, sourceRel, LogicalTableModify.Operation.UPDATE, targetColumnNameList, false);
+ }
+
+ private RelNode convertMerge(SqlMerge call) {
+ RelOptTable targetTable = getTargetTable(call);
+
+ // convert update column list from SqlIdentifier to String
+ final List<String> targetColumnNameList = new ArrayList<>();
+ SqlUpdate updateCall = call.getUpdateCall();
+ if (updateCall != null) {
+ for (SqlNode targetColumn : updateCall.getTargetColumnList()) {
+ SqlIdentifier id = (SqlIdentifier) targetColumn;
+ String name = id.getSimple();
+ targetColumnNameList.add(name);
+ }
+ }
+
+ // replace the projection of the source select with a
+ // projection that contains the following:
+ // 1) the expressions corresponding to the new insert row (if there is
+ // an insert)
+ // 2) all columns from the target table (if there is an update)
+ // 3) the set expressions in the update call (if there is an update)
+
+ // first, convert the merge's source select to construct the columns
+ // from the target table and the set expressions in the update call
+ RelNode mergeSourceRel = convertSelect(call.getSourceSelect());
+
+ // then, convert the insert statement so we can get the insert
+ // values expressions
+ SqlInsert insertCall = call.getInsertCall();
+ int nLevel1Exprs = 0;
+ List<RexNode> level1InsertExprs = null;
+ List<RexNode> level2InsertExprs = null;
+ if (insertCall != null) {
+ RelNode insertRel = convertInsert(insertCall);
+
+ // if there are 2 level of projections in the insert source, combine
+ // them into a single project; level1 refers to the topmost project;
+ // the level1 projection contains references to the level2
+ // expressions, except in the case where no target expression was
+ // provided, in which case, the expression is the default value for
+ // the column; or if the expressions directly map to the source
+ // table
+ level1InsertExprs = ((LogicalProject) insertRel.getInput(0)).getProjects();
+ if (insertRel.getInput(0).getInput(0) instanceof LogicalProject) {
+ level2InsertExprs = ((LogicalProject) insertRel.getInput(0).getInput(0)).getProjects();
+ }
+ nLevel1Exprs = level1InsertExprs.size();
+ }
+
+ LogicalJoin join = (LogicalJoin) mergeSourceRel.getInput(0);
+ int nSourceFields = join.getLeft().getRowType().getFieldCount();
+ final List<RexNode> projects = new ArrayList<>();
+ for (int level1Idx = 0; level1Idx < nLevel1Exprs; level1Idx++) {
+ if ((level2InsertExprs != null) && (level1InsertExprs.get(level1Idx) instanceof RexInputRef)) {
+ int level2Idx = ((RexInputRef) level1InsertExprs.get(level1Idx)).getIndex();
+ projects.add(level2InsertExprs.get(level2Idx));
+ } else {
+ projects.add(level1InsertExprs.get(level1Idx));
+ }
+ }
+ if (updateCall != null) {
+ final LogicalProject project = (LogicalProject) mergeSourceRel;
+ projects.addAll(Util.skip(project.getProjects(), nSourceFields));
+ }
+
+ RelNode massagedRel = RelOptUtil.createProject(join, projects, null, true);
+
+ return LogicalTableModify.create(targetTable, catalogReader, massagedRel, LogicalTableModify.Operation.MERGE, targetColumnNameList, false);
+ }
+
+ /**
+ * Converts an identifier into an expression in a given scope. For example,
+ * the "empno" in "select empno from emp join dept" becomes "emp.empno".
+ */
+ private RexNode convertIdentifier(Blackboard bb, SqlIdentifier identifier) {
+ // first check for reserved identifiers like CURRENT_USER
+ final SqlCall call = SqlUtil.makeCall(opTab, identifier);
+ if (call != null) {
+ return bb.convertExpression(call);
+ }
+
+ final SqlQualified qualified;
+ if (bb.scope != null) {
+ qualified = bb.scope.fullyQualify(identifier);
+ identifier = qualified.identifier;
+ } else {
+ qualified = SqlQualified.create(null, 1, null, identifier);
+ }
+ RexNode e = bb.lookupExp(qualified);
+ final String correlationName;
+ if (e instanceof RexCorrelVariable) {
+ correlationName = ((RexCorrelVariable) e).getName();
+ } else {
+ correlationName = null;
+ }
+
+ for (String name : qualified.suffixTranslated()) {
+ final boolean caseSensitive = true; // name already fully-qualified
+ e = rexBuilder.makeFieldAccess(e, name, caseSensitive);
+ }
+ if (e instanceof RexInputRef) {
+ // adjust the type to account for nulls introduced by outer joins
+ e = adjustInputRef(bb, (RexInputRef) e);
+ }
+
+ if (null != correlationName) {
+ // REVIEW: make mapCorrelateVariableToRexNode map to RexFieldAccess
+ assert e instanceof RexFieldAccess;
+ final RexNode prev = bb.mapCorrelateVariableToRexNode.put(correlationName, e);
+ assert prev == null;
+ }
+ return e;
+ }
+
+ /**
+ * Adjusts the type of a reference to an input field to account for nulls
+ * introduced by outer joins; and adjusts the offset to match the physical
+ * implementation.
+ *
+ * @param bb Blackboard
+ * @param inputRef Input ref
+ * @return Adjusted input ref
+ */
+ protected RexNode adjustInputRef(Blackboard bb, RexInputRef inputRef) {
+ RelDataTypeField field = bb.getRootField(inputRef);
+ if (field != null) {
+ return rexBuilder.makeInputRef(field.getType(), inputRef.getIndex());
+ }
+ return inputRef;
+ }
+
+ /**
+ * Converts a row constructor into a relational expression.
+ *
+ * @param bb Blackboard
+ * @param rowConstructor Row constructor expression
+ * @return Relational expression which returns a single row.
+ * @pre isRowConstructor(rowConstructor)
+ */
+ private RelNode convertRowConstructor(Blackboard bb, SqlCall rowConstructor) {
+ assert isRowConstructor(rowConstructor) : rowConstructor;
+ final List<SqlNode> operands = rowConstructor.getOperandList();
+ return convertMultisets(operands, bb);
+ }
+
+ private RelNode convertCursor(Blackboard bb, SubQuery subQuery) {
+ final SqlCall cursorCall = (SqlCall) subQuery.node;
+ assert cursorCall.operandCount() == 1;
+ SqlNode query = cursorCall.operand(0);
+ RelNode converted = convertQuery(query, false, false);
+ int iCursor = bb.cursors.size();
+ bb.cursors.add(converted);
+ subQuery.expr = new RexInputRef(iCursor, converted.getRowType());
+ return converted;
+ }
+
+ private RelNode convertMultisets(final List<SqlNode> operands, Blackboard bb) {
+ // NOTE: Wael 2/04/05: this implementation is not the most efficient in
+ // terms of planning since it generates XOs that can be reduced.
+ final List<Object> joinList = new ArrayList<>();
+ List<SqlNode> lastList = new ArrayList<>();
+ for (int i = 0; i < operands.size(); i++) {
+ SqlNode operand = operands.get(i);
+ if (!(operand instanceof SqlCall)) {
+ lastList.add(operand);
+ continue;
+ }
+
+ final SqlCall call = (SqlCall) operand;
+ final SqlOperator op = call.getOperator();
+ if ((op != SqlStdOperatorTable.MULTISET_VALUE) && (op != SqlStdOperatorTable.MULTISET_QUERY)) {
+ lastList.add(operand);
+ continue;
+ }
+ final RelNode input;
+ if (op == SqlStdOperatorTable.MULTISET_VALUE) {
+ final SqlNodeList list = new SqlNodeList(call.getOperandList(), call.getParserPosition());
+ // assert bb.scope instanceof SelectScope : bb.scope;
+ CollectNamespace nss = (CollectNamespace) validator.getNamespace(call);
+ Blackboard usedBb;
+ if (null != nss) {
+ usedBb = createBlackboard(nss.getScope(), null);
+ } else {
+ usedBb = createBlackboard(new ListScope(bb.scope) {
+ public SqlNode getNode() {
+ return call;
+ }
+ }, null);
+ }
+ RelDataType multisetType = validator.getValidatedNodeType(call);
+ validator.setValidatedNodeType(list, multisetType.getComponentType());
+ input = convertQueryOrInList(usedBb, list);
+ } else {
+ input = convertQuery(call.operand(0), false, true);
+ }
+
+ if (lastList.size() > 0) {
+ joinList.add(lastList);
+ }
+ lastList = new ArrayList<>();
+ Collect collect = new Collect(cluster, cluster.traitSetOf(Convention.NONE), input, validator.deriveAlias(call, i));
+ joinList.add(collect);
+ }
+
+ if (joinList.size() == 0) {
+ joinList.add(lastList);
+ }
+
+ for (int i = 0; i < joinList.size(); i++) {
+ Object o = joinList.get(i);
+ if (o instanceof List) {
+ @SuppressWarnings("unchecked")
+ List<SqlNode> projectList = (List<SqlNode>) o;
+ final List<RexNode> selectList = new ArrayList<>();
+ final List<String> fieldNameList = new ArrayList<>();
+ for (int j = 0; j < projectList.size(); j++) {
+ SqlNode operand = projectList.get(j);
+ selectList.add(bb.convertExpression(operand));
+
+ // REVIEW angel 5-June-2005: Use deriveAliasFromOrdinal
+ // instead of deriveAlias to match field names from
+ // SqlRowOperator. Otherwise, get error Type
+ // 'RecordType(INTEGER EMPNO)' has no field 'EXPR$0' when
+ // doing select * from unnest( select multiset[empno]
+ // from sales.emps);
+
+ fieldNameList.add(SqlUtil.deriveAliasFromOrdinal(j));
+ }
+
+ RelNode projRel = RelOptUtil.createProject(LogicalValues.createOneRow(cluster), selectList, fieldNameList);
+
+ joinList.set(i, projRel);
+ }
+ }
+
+ RelNode ret = (RelNode) joinList.get(0);
+ for (int i = 1; i < joinList.size(); i++) {
+ RelNode relNode = (RelNode) joinList.get(i);
+ ret = RelFactories.DEFAULT_JOIN_FACTORY.createJoin(ret, relNode, rexBuilder.makeLiteral(true), JoinRelType.INNER, ImmutableSet.<String> of(), false);
+ }
+ return ret;
+ }
+
+ private void convertSelectList(Blackboard bb, SqlSelect select, List<SqlNode> orderList) {
+ SqlNodeList selectList = select.getSelectList();
+ selectList = validator.expandStar(selectList, select, false);
+
+ replaceSubqueries(bb, selectList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
+
+ List<String> fieldNames = new ArrayList<>();
+ final List<RexNode> exprs = new ArrayList<>();
+ final Collection<String> aliases = new TreeSet<>();
+
+ // Project any system fields. (Must be done before regular select items,
+ // because offsets may be affected.)
+ final List<SqlMonotonicity> columnMonotonicityList = new ArrayList<>();
+ extraSelectItems(bb, select, exprs, fieldNames, aliases, columnMonotonicityList);
+
+ // Project select clause.
+ int i = -1;
+ for (SqlNode expr : selectList) {
+ ++i;
+ exprs.add(bb.convertExpression(expr));
+ fieldNames.add(deriveAlias(expr, aliases, i));
+ }
+
+ // Project extra fields for sorting.
+ for (SqlNode expr : orderList) {
+ ++i;
+ SqlNode expr2 = validator.expandOrderExpr(select, expr);
+ exprs.add(bb.convertExpression(expr2));
+ fieldNames.add(deriveAlias(expr, aliases, i));
+ }
+
+ fieldNames = SqlValidatorUtil.uniquify(fieldNames);
+
+ RelNode inputRel = bb.root;
+ bb.setRoot(RelOptUtil.createProject(bb.root, exprs, fieldNames), false);
+
+ assert bb.columnMonotonicities.isEmpty();
+ bb.columnMonotonicities.addAll(columnMonotonicityList);
+ for (SqlNode selectItem : selectList) {
+ bb.columnMonotonicities.add(selectItem.getMonotonicity(bb.scope));
+ }
+ }
+
+ /**
+ * Adds extra select items. The default implementation adds nothing; derived
+ * classes may add columns to exprList, nameList, aliasList and
+ * columnMonotonicityList.
+ *
+ * @param bb Blackboard
+ * @param select Select statement being translated
+ * @param exprList List of expressions in select clause
+ * @param nameList List of names, one per column
+ * @param aliasList Collection of aliases that have been used
+ * already
+ * @param columnMonotonicityList List of monotonicity, one per column
+ */
+ protected void extraSelectItems(Blackboard bb, SqlSelect select, List<RexNode> exprList, List<String> nameList, Collection<String> aliasList, List<SqlMonotonicity> columnMonotonicityList) {
+ }
+
+ private String deriveAlias(final SqlNode node, Collection<String> aliases, final int ordinal) {
+ String alias = validator.deriveAlias(node, ordinal);
+ if ((alias == null) || aliases.contains(alias)) {
+ String aliasBase = (alias == null) ? "EXPR$" : alias;
+ for (int j = 0;; j++) {
+ alias = aliasBase + j;
+ if (!aliases.contains(alias)) {
+ break;
+ }
+ }
+ }
+ aliases.add(alias);
+ return alias;
+ }
+
+ /**
+ * Converts a WITH sub-query into a relational expression.
+ */
+ public RelNode convertWith(SqlWith with) {
+ return convertQuery(with.body, false, false);
+ }
+
+ /**
+ * Converts a SELECT statement's parse tree into a relational expression.
+ */
+ public RelNode convertValues(SqlCall values, RelDataType targetRowType) {
+ final SqlValidatorScope scope = validator.getOverScope(values);
+ assert scope != null;
+ final Blackboard bb = createBlackboard(scope, null);
+ convertValuesImpl(bb, values, targetRowType);
+ return bb.root;
+ }
+
+ /**
+ * Converts a values clause (as in "INSERT INTO T(x,y) VALUES (1,2)") into a
+ * relational expression.
+ *
+ * @param bb Blackboard
+ * @param values Call to SQL VALUES operator
+ * @param targetRowType Target row type
+ */
+ private void convertValuesImpl(Blackboard bb, SqlCall values, RelDataType targetRowType) {
+ // Attempt direct conversion to LogicalValues; if that fails, deal with
+ // fancy stuff like subqueries below.
+ RelNode valuesRel = convertRowValues(bb, values, values.getOperandList(), true, targetRowType);
+ if (valuesRel != null) {
+ bb.setRoot(valuesRel, true);
+ return;
+ }
+
+ final List<RelNode> unionRels = new ArrayList<>();
+ for (SqlNode rowConstructor1 : values.getOperandList()) {
+ SqlCall rowConstructor = (SqlCall) rowConstructor1;
+ Blackboard tmpBb = createBlackboard(bb.scope, null);
+ replaceSubqueries(tmpBb, rowConstructor, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
+ final List<Pair<RexNode, String>> exps = new ArrayList<>();
+ for (Ord<SqlNode> operand : Ord.zip(rowConstructor.getOperandList())) {
+ exps.add(Pair.of(tmpBb.convertExpression(operand.e), validator.deriveAlias(operand.e, operand.i)));
+ }
+ RelNode in = (null == tmpBb.root) ? LogicalValues.createOneRow(cluster) : tmpBb.root;
+ unionRels.add(RelOptUtil.createProject(in, Pair.left(exps), Pair.right(exps), true));
+ }
+
+ if (unionRels.size() == 0) {
+ throw Util.newInternal("empty values clause");
+ } else if (unionRels.size() == 1) {
+ bb.setRoot(unionRels.get(0), true);
+ } else {
+ bb.setRoot(LogicalUnion.create(unionRels, true), true);
+ }
+
+ // REVIEW jvs 22-Jan-2004: should I add
+ // mapScopeToLux.put(validator.getScope(values),bb.root);
+ // ?
+ }
+
+ private String createCorrel() {
+ int n = nextCorrel++;
+ return CORREL_PREFIX + n;
+ }
+
+ private int getCorrelOrdinal(String correlName) {
+ assert correlName.startsWith(CORREL_PREFIX);
+ return Integer.parseInt(correlName.substring(CORREL_PREFIX.length()));
+ }
+
+ //~ Inner Classes ----------------------------------------------------------
+
+ /**
+ * Workspace for translating an individual SELECT statement (or sub-SELECT).
+ */
+ protected class Blackboard implements SqlRexContext, SqlVisitor<RexNode> {
+ /**
+ * Collection of {@link RelNode} objects which correspond to a SELECT
+ * statement.
+ */
+ public final SqlValidatorScope scope;
+ private final Map<String, RexNode> nameToNodeMap;
+ public RelNode root;
+ private List<RelNode> inputs;
+ private final Map<String, RexNode> mapCorrelateVariableToRexNode = new HashMap<>();
+
+ List<RelNode> cursors;
+
+ /**
+ * List of <code>IN</code> and <code>EXISTS</code> nodes inside this
+ * <code>SELECT</code> statement (but not inside sub-queries).
+ */
+ private final Set<SubQuery> subqueryList = Sets.newLinkedHashSet();
+
+ private boolean subqueryNeedsOuterJoin;
+
+ /**
+ * Workspace for building aggregates.
+ */
+ AggConverter agg;
+
+ /**
+ * When converting window aggregate, we need to know if the window is
+ * guaranteed to be non-empty.
+ */
+ SqlWindow window;
+
+ /**
+ * Project the groupby expressions out of the root of this sub-select.
+ * Subqueries can reference group by expressions projected from the
+ * "right" to the subquery.
+ */
+ private final Map<RelNode, Map<Integer, Integer>> mapRootRelToFieldProjection = new HashMap<>();
+
+ private final List<SqlMonotonicity> columnMonotonicities = new ArrayList<>();
+
+ private final List<RelDataTypeField> systemFieldList = new ArrayList<>();
+
+ /**
+ * Creates a Blackboard.
+ *
+ * @param scope Name-resolution scope for expressions validated
+ * within this query. Can be null if this Blackboard is
+ * for a leaf node, say
+ * @param nameToNodeMap Map which translates the expression to map a
+ * given parameter into, if translating expressions;
+ * null otherwise
+ */
+ protected Blackboard(SqlValidatorScope scope, Map<String, RexNode> nameToNodeMap) {
+ this.scope = scope;
+ this.nameToNodeMap = nameToNodeMap;
+ this.cursors = new ArrayList<>();
+ subqueryNeedsOuterJoin = false;
+ }
+
+ public RexNode register(RelNode rel, JoinRelType joinType) {
+ return register(rel, joinType, null);
+ }
+
+ /**
+ * Registers a relational expression.
+ *
+ * @param rel Relational expression
+ * @param joinType Join type
+ * @param leftKeys LHS of IN clause, or null for expressions
+ * other than IN
+ * @return Expression with which to refer to the row (or partial row)
+ * coming from this relational expression's side of the join
+ */
+ public RexNode register(RelNode rel, JoinRelType joinType, List<RexNode> leftKeys) {
+ assert joinType != null;
+ if (root == null) {
+ assert leftKeys == null;
+ setRoot(rel, false);
+ return rexBuilder.makeRangeReference(root.getRowType(), 0, false);
+ }
+
+ final RexNode joinCond;
+ final int origLeftInputCount = root.getRowType().getFieldCount();
+ if (leftKeys != null) {
+ List<RexNode> newLeftInputExpr = Lists.newArrayList();
+ for (int i = 0; i < origLeftInputCount; i++) {
+ newLeftInputExpr.add(rexBuilder.makeInputRef(root, i));
+ }
+
+ final List<Integer> leftJoinKeys = Lists.newArrayList();
+ for (RexNode leftKey : leftKeys) {
+ newLeftInputExpr.add(leftKey);
+ leftJoinKeys.add(origLeftInputCount + leftJoinKeys.size());
+ }
+
+ LogicalProject newLeftInput = (LogicalProject) RelOptUtil.createProject(root, newLeftInputExpr, null, true);
+
+ // maintain the group by mapping in the new LogicalProject
+ if (mapRootRelToFieldProjection.containsKey(root)) {
+ mapRootRelToFieldProjection.put(newLeftInput, mapRootRelToFieldProjection.get(root));
+ }
+
+ setRoot(newLeftInput, false);
+
+ // right fields appear after the LHS fields.
+ final int rightOffset = root.getRowType().getFieldCount() - newLeftInput.getRowType().getFieldCount();
+ final List<Integer> rightKeys = Util.range(rightOffset, rightOffset + leftJoinKeys.size());
+
+ joinCond = RelOptUtil.createEquiJoinCondition(newLeftInput, leftJoinKeys, rel, rightKeys, rexBuilder);
+ } else {
+ joinCond = rexBuilder.makeLiteral(true);
+ }
+
+ int leftFieldCount = root.getRowType().getFieldCount();
+ final RelNode join = createJoin(this, root, rel, joinCond, joinType);
+
+ setRoot(join, false);
+
+ if (leftKeys != null && joinType == JoinRelType.LEFT) {
+ final int leftKeyCount = leftKeys.size();
+ int rightFieldLength = rel.getRowType().getFieldCount();
+ assert leftKeyCount == rightFieldLength - 1;
+
+ final int rexRangeRefLength = leftKeyCount + rightFieldLength;
+ RelDataType returnType = typeFactory.createStructType(new AbstractList<Map.Entry<String, RelDataType>>() {
+ public Map.Entry<String, RelDataType> get(int index) {
+ return join.getRowType().getFieldList().get(origLeftInputCount + index);
+ }
+
+ public int size() {
+ return rexRangeRefLength;
+ }
+ });
+
+ return rexBuilder.makeRangeReference(returnType, origLeftInputCount, false);
+ } else {
+ return rexBuilder.makeRangeReference(rel.getRowType(), leftFieldCount, joinType.generatesNullsOnRight());
+ }
+ }
+
+ /**
+ * Sets a new root relational expression, as the translation process
+ * backs its way further up the tree.
+ *
+ * @param root New root relational expression
+ * @param leaf Whether the relational expression is a leaf, that is,
+ * derived from an atomic relational expression such as a table
+ * name in the from clause, or the projection on top of a
+ * select-subquery. In particular, relational expressions
+ * derived from JOIN operators are not leaves, but set
+ * expressions are.
+ */
+ public void setRoot(RelNode root, boolean leaf) {
+ setRoot(Collections.singletonList(root), root, root instanceof LogicalJoin);
+ if (leaf) {
+ leaves.add(root);
+ }
+ this.columnMonotonicities.clear();
+ }
+
+ private void setRoot(List<RelNode> inputs, RelNode root, boolean hasSystemFields) {
+ this.inputs = inputs;
+ this.root = root;
+ this.systemFieldList.clear();
+ if (hasSystemFields) {
+ this.systemFieldList.addAll(getSystemFields());
+ }
+ }
+
+ /**
+ * Notifies this Blackboard that the root just set using
+ * {@link #setRoot(RelNode, boolean)} was derived using dataset
+ * substitution.
+ *
+ * <p>The default implementation is not interested in such
+ * notifications, and does nothing.
+ *
+ * @param datasetName Dataset name
+ */
+ public void setDataset(String datasetName) {
+ }
+
+ void setRoot(List<RelNode> inputs) {
+ setRoot(inputs, null, false);
+ }
+
+ /**
+ * Returns an expression with which to reference a from-list item.
+ *
+ * @param qualified the alias of the from item
+ * @return a {@link RexFieldAccess} or {@link RexRangeRef}, or null if
+ * not found
+ */
+ RexNode lookupExp(SqlQualified qualified) {
+ if (nameToNodeMap != null && qualified.prefixLength == 1) {
+ RexNode node = nameToNodeMap.get(qualified.identifier.names.get(0));
+ if (node == null) {
+ throw Util.newInternal("Unknown identifier '" + qualified.identifier + "' encountered while expanding expression");
+ }
+ return node;
+ }
+ int[] offsets = { -1 };
+ final SqlValidatorScope[] ancestorScopes = { null };
+ SqlValidatorNamespace foundNs = scope.resolve(qualified.prefix(), ancestorScopes, offsets);
+ if (foundNs == null) {
+ return null;
+ }
+
+ // Found in current query's from list. Find which from item.
+ // We assume that the order of the from clause items has been
+ // preserved.
+ SqlValidatorScope ancestorScope = ancestorScopes[0];
+ boolean isParent = ancestorScope != scope;
+ if ((inputs != null) && !isParent) {
+ int offset = offsets[0];
+ final LookupContext rels = new LookupContext(this, inputs, systemFieldList.size());
+ return lookup(offset, rels);
+ } else {
+ // We're referencing a relational expression which has not been
+ // converted yet. This occurs when from items are correlated,
+ // e.g. "select from emp as emp join emp.getDepts() as dept".
+ // Create a temporary expression.
+ assert isParent;
+ DeferredLookup lookup = new DeferredLookup(this, qualified.identifier.names.get(0));
+ String correlName = createCorrel();
+ mapCorrelToDeferred.put(correlName, lookup);
+ final RelDataType rowType = foundNs.getRowType();
+ return rexBuilder.makeCorrel(rowType, correlName);
+ }
+ }
+
+ /**
+ * Creates an expression with which to reference the expression whose
+ * offset in its from-list is {@code offset}.
+ */
+ RexNode lookup(int offset, LookupContext lookupContext) {
+ Pair<RelNode, Integer> pair = lookupContext.findRel(offset);
+ return rexBuilder.makeRangeReference(pair.left.getRowType(), pair.right, false);
+ }
+
+ RelDataTypeField getRootField(RexInputRef inputRef) {
+ int fieldOffset = inputRef.getIndex();
+ for (RelNode input : inputs) {
+ RelDataType rowType = input.getRowType();
+ if (rowType == null) {
+ // TODO: remove this once leastRestrictive
+ // is correctly implemented
+ return null;
+ }
+ if (fieldOffset < rowType.getFieldCount()) {
+ return rowType.getFieldList().get(fieldOffset);
+ }
+ fieldOffset -= rowType.getFieldCount();
+ }
+ throw new AssertionError();
+ }
+
+ public void flatten(List<RelNode> rels, int systemFieldCount, int[] start, List<Pair<RelNode, Integer>> relOffsetList) {
+ for (RelNode rel : rels) {
+ if (leaves.contains(rel)) {
+ relOffsetList.add(Pair.of(rel, start[0]));
+ start[0] += rel.getRowType().getFieldCount();
+ } else {
+ if (rel instanceof LogicalJoin || rel instanceof LogicalAggregate) {
+ start[0] += systemFieldCount;
+ }
+ flatten(rel.getInputs(), systemFieldCount, start, relOffsetList);
+ }
+ }
+ }
+
+ void registerSubquery(SqlNode node, RelOptUtil.Logic logic) {
+ for (SubQuery subQuery : subqueryList) {
+ if (node.equalsDeep(subQuery.node, false)) {
+ return;
+ }
+ }
+ subqueryList.add(new SubQuery(node, logic));
+ }
+
+ SubQuery getSubquery(SqlNode expr) {
+ for (SubQuery subQuery : subqueryList) {
+ if (expr.equalsDeep(subQuery.node, false)) {
+ return subQuery;
+ }
+ }
+
+ return null;
+ }
+
+ ImmutableList<RelNode> retrieveCursors() {
+ try {
+ return ImmutableList.copyOf(cursors);
+ } finally {
+ cursors.clear();
+ }
+ }
+
+ public RexNode convertExpression(SqlNode expr) {
+ // If we're in aggregation mode and this is an expression in the
+ // GROUP BY clause, return a reference to the field.
+ if (agg != null) {
+ final SqlNode expandedGroupExpr = validator.expand(expr, scope);
+ final int ref = agg.lookupGroupExpr(expandedGroupExpr);
+ if (ref >= 0) {
+ return rexBuilder.makeInputRef(root, ref);
+ }
+ if (expr instanceof SqlCall) {
+ final RexNode rex = agg.lookupAggregates((SqlCall) expr);
+ if (rex != null) {
+ return rex;
+ }
+ }
+ }
+
+ // Allow the derived class chance to override the standard
+ // behavior for special kinds of expressions.
+ RexNode rex = convertExtendedExpression(expr, this);
+ if (rex != null) {
+ return rex;
+ }
+
+ // Sub-queries and OVER expressions are not like ordinary
+ // expressions.
+ final SqlKind kind = expr.getKind();
+ final SubQuery subQuery;
+ switch (kind) {
+ case CURSOR:
+ case IN:
+ subQuery = getSubquery(expr);
+
+ assert subQuery != null;
+ rex = subQuery.expr;
+ assert rex != null : "rex != null";
+ return rex;
+
+ case SELECT:
+ case EXISTS:
+ case SCALAR_QUERY:
+ subQuery = getSubquery(expr);
+ assert subQuery != null;
+ rex = subQuery.expr;
+ assert rex != null : "rex != null";
+
+ if (((kind == SqlKind.SCALAR_QUERY) || (kind == SqlKind.EXISTS)) && isConvertedSubq(rex)) {
+ // scalar subquery or EXISTS has been converted to a
+ // constant
+ return rex;
+ }
+
+ // The indicator column is the last field of the subquery.
+ RexNode fieldAccess = rexBuilder.makeFieldAccess(rex, rex.getType().getFieldCount() - 1);
+
+ // The indicator column will be nullable if it comes from
+ // the null-generating side of the join. For EXISTS, add an
+ // "IS TRUE" check so that the result is "BOOLEAN NOT NULL".
+ if (fieldAccess.getType().isNullable() && kind == SqlKind.EXISTS) {
+ fieldAccess = rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, fieldAccess);
+ }
+ return fieldAccess;
+
+ case OVER:
+ return convertOver(this, expr);
+
+ default:
+ // fall through
+ }
+
+ // Apply standard conversions.
+ rex = expr.accept(this);
+ Util.permAssert(rex != null, "conversion result not null");
+ return rex;
+ }
+
+ /**
+ * Converts an item in an ORDER BY clause, extracting DESC, NULLS LAST
+ * and NULLS FIRST flags first.
+ */
+ public RexNode convertSortExpression(SqlNode expr, Set<SqlKind> flags) {
+ switch (expr.getKind()) {
+ case DESCENDING:
+ case NULLS_LAST:
+ case NULLS_FIRST:
+ flags.add(expr.getKind());
+ final SqlNode operand = ((SqlCall) expr).operand(0);
+ return convertSortExpression(operand, flags);
+ default:
+ return convertExpression(expr);
+ }
+ }
+
+ /**
+ * Determines whether a RexNode corresponds to a subquery that's been
+ * converted to a constant.
+ *
+ * @param rex the expression to be examined
+ * @return true if the expression is a dynamic parameter, a literal, or
+ * a literal that is being cast
+ */
+ private boolean isConvertedSubq(RexNode rex) {
+ if ((rex instanceof RexLiteral) || (rex instanceof RexDynamicParam)) {
+ return true;
+ }
+ if (rex instanceof RexCall) {
+ RexCall call = (RexCall) rex;
+ if (call.getOperator() == SqlStdOperatorTable.CAST) {
+ RexNode operand = call.getOperands().get(0);
+ if (operand instanceof RexLiteral) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ // implement SqlRexContext
+ public int getGroupCount() {
+ if (agg != null) {
+ return agg.groupExprs.size();
+ }
+ if (window != null) {
+ return window.isAlwaysNonEmpty() ? 1 : 0;
+ }
+ return -1;
+ }
+
+ // implement SqlRexContext
+ public RexBuilder getRexBuilder() {
+ return rexBuilder;
+ }
+
+ // implement SqlRexContext
+ public RexRangeRef getSubqueryExpr(SqlCall call) {
+ final SubQuery subQuery = getSubquery(call);
+ assert subQuery != null;
+ return (RexRangeRef) subQuery.expr;
+ }
+
+ // implement SqlRexContext
+ public RelDataTypeFactory getTypeFactory() {
+ return typeFactory;
+ }
+
+ // implement SqlRexContext
+ public DefaultValueFactory getDefaultValueFactory() {
+ return defaultValueFactory;
+ }
+
+ // implement SqlRexContext
+ public SqlValidator getValidator() {
+ return validator;
+ }
+
+ // implement SqlRexContext
+ public RexNode convertLiteral(SqlLiteral literal) {
+ return exprConverter.convertLiteral(this, literal);
+ }
+
+ public RexNode convertInterval(SqlIntervalQualifier intervalQualifier) {
+ return exprConverter.convertInterval(this, intervalQualifier);
+ }
+
+ // implement SqlVisitor
+ public RexNode visit(SqlLiteral literal) {
+ return exprConverter.convertLiteral(this, literal);
+ }
+
+ // implement SqlVisitor
+ public RexNode visit(SqlCall call) {
+ if (agg != null) {
+ final SqlOperator op = call.getOperator();
+ if (op.isAggregator() || op.getKind() == SqlKind.FILTER) {
+ return agg.lookupAggregates(call);
+ }
+ }
+ return exprConverter.convertCall(this, call);
+ }
+
+ // implement SqlVisitor
+ public RexNode visit(SqlNodeList nodeList) {
+ throw new UnsupportedOperationException();
+ }
+
+ // implement SqlVisitor
+ public RexNode visit(SqlIdentifier id) {
+ return convertIdentifier(this, id);
+ }
+
+ // implement SqlVisitor
+ public RexNode visit(SqlDataTypeSpec type) {
+ throw new UnsupportedOperationException();
+ }
+
+ // implement SqlVisitor
+ public RexNode visit(SqlDynamicParam param) {
+ return convertDynamicParam(param);
+ }
+
+ // implement SqlVisitor
+ public RexNode visit(SqlIntervalQualifier intervalQualifier) {
+ return convertInterval(intervalQualifier);
+ }
+
+ public List<SqlMonotonicity> getColumnMonotonicities() {
+ return columnMonotonicities;
+ }
+ }
+
+ /** Deferred lookup. */
+ private static class DeferredLookup {
+ Blackboard bb;
+ String originalRelName;
+
+ DeferredLookup(Blackboard bb, String originalRelName) {
+ this.bb = bb;
+ this.originalRelName = originalRelName;
+ }
+
+ public RexFieldAccess getFieldAccess(String name) {
+ return (RexFieldAccess) bb.mapCorrelateVariableToRexNode.get(name);
+ }
+
+ public String getOriginalRelName() {
+ return originalRelName;
+ }
+ }
+
+ /**
+ * An implementation of DefaultValueFactory which always supplies NULL.
+ */
+ class NullDefaultValueFactory implements DefaultValueFactory {
+ public boolean isGeneratedAlways(RelOptTable table, int iColumn) {
+ return false;
+ }
+
+ public RexNode newColumnDefaultValue(RelOptTable table, int iColumn) {
+ return rexBuilder.constantNull();
+ }
+
+ public RexNode newAttributeInitializer(RelDataType type, SqlFunction constructor, int iAttribute, List<RexNode> constructorArgs) {
+ return rexBuilder.constantNull();
+ }
+ }
+
+ /**
+ * A default implementation of SubqueryConverter that does no conversion.
+ */
+ private class NoOpSubqueryConverter implements SubqueryConverter {
+ // implement SubqueryConverter
+ public boolean canConvertSubquery() {
+ return false;
+ }
+
+ // implement SubqueryConverter
+ public RexNode convertSubquery(SqlCall subquery, SqlToRelConverter parentConverter, boolean isExists, boolean isExplain) {
+ throw new IllegalArgumentException();
+ }
+ }
+
+ /**
+ * Converts expressions to aggregates.
+ *
+ * <p>Consider the expression
+ *
+ * <blockquote>
+ * {@code SELECT deptno, SUM(2 * sal) FROM emp GROUP BY deptno}
+ * </blockquote>
+ *
+ * <p>Then:
+ *
+ * <ul>
+ * <li>groupExprs = {SqlIdentifier(deptno)}</li>
+ * <li>convertedInputExprs = {RexInputRef(deptno), 2 *
+ * RefInputRef(sal)}</li>
+ * <li>inputRefs = {RefInputRef(#0), RexInputRef(#1)}</li>
+ * <li>aggCalls = {AggCall(SUM, {1})}</li>
+ * </ul>
+ */
+ protected class AggConverter implements SqlVisitor<Void> {
+ private final Blackboard bb;
+ public final AggregatingSelectScope aggregatingSelectScope;
+
+ private final Map<String, String> nameMap = Maps.newHashMap();
+
+ /**
+ * The group-by expressions, in {@link SqlNode} format.
+ */
+ private final SqlNodeList groupExprs = new SqlNodeList(SqlParserPos.ZERO);
+
+ /**
+ * Input expressions for the group columns and aggregates, in
+ * {@link RexNode} format. The first elements of the list correspond to the
+ * elements in {@link #groupExprs}; the remaining elements are for
+ * aggregates.
+ */
+ private final List<RexNode> convertedInputExprs = Lists.newArrayList();
+
+ /**
+ * Names of {@link #convertedInputExprs}, where the expressions are
+ * simple mappings to input fields.
+ */
+ private final List<String> convertedInputExprNames = Lists.newArrayList();
+
+ private final List<AggregateCall> aggCalls = Lists.newArrayList();
+ private final Map<SqlNode, RexNode> aggMapping = Maps.newHashMap();
+ private final Map<AggregateCall, RexNode> aggCallMapping = Maps.newHashMap();
+
+ /**
+ * Creates an AggConverter.
+ *
+ * <p>The <code>select</code> parameter provides enough context to name
+ * aggregate calls which are top-level select list items.
+ *
+ * @param bb Blackboard
+ * @param select Query being translated; provides context to give
+ */
+ public AggConverter(Blackboard bb, SqlSelect select) {
+ this.bb = bb;
+ this.aggregatingSelectScope = (AggregatingSelectScope) bb.getValidator().getSelectScope(select);
+
+ // Collect all expressions used in the select list so that aggregate
+ // calls can be named correctly.
+ final SqlNodeList selectList = select.getSelectList();
+ for (int i = 0; i < selectList.size(); i++) {
+ SqlNode selectItem = selectList.get(i);
+ String name = null;
+ if (SqlUtil.isCallTo(selectItem, SqlStdOperatorTable.AS)) {
+ final SqlCall call = (SqlCall) selectItem;
+ selectItem = call.operand(0);
+ name = call.operand(1).toString();
+ }
+ if (name == null) {
+ name = validator.deriveAlias(selectItem, i);
+ }
+ nameMap.put(selectItem.toString(), name);
+ }
+ }
+
+ public int addGroupExpr(SqlNode expr) {
+ RexNode convExpr = bb.convertExpression(expr);
+ int ref = lookupGroupExpr(expr);
+ if (ref >= 0) {
+ return ref;
+ }
+ final int index = groupExprs.size();
+ groupExprs.add(expr);
+ String name = nameMap.get(expr.toString());
+ addExpr(convExpr, name);
+ return index;
+ }
+
+ /**
+ * Adds an expression, deducing an appropriate name if possible.
+ *
+ * @param expr Expression
+ * @param name Suggested name
+ */
+ private void addExpr(RexNode expr, String name) {
+ convertedInputExprs.add(expr);
+ if ((name == null) && (expr instanceof RexInputRef)) {
+ final int i = ((RexInputRef) expr).getIndex();
+ name = bb.root.getRowType().getFieldList().get(i).getName();
+ }
+ if (convertedInputExprNames.contains(name)) {
+ // In case like 'SELECT ... GROUP BY x, y, x', don't add
+ // name 'x' twice.
+ name = null;
+ }
+ convertedInputExprNames.add(name);
+ }
+
+ // implement SqlVisitor
+ public Void visit(SqlIdentifier id) {
+ return null;
+ }
+
+ // implement SqlVisitor
+ public Void visit(SqlNodeList nodeList) {
+ for (int i = 0; i < nodeList.size(); i++) {
+ nodeList.get(i).accept(this);
+ }
+ return null;
+ }
+
+ // implement SqlVisitor
+ public Void visit(SqlLiteral lit) {
+ return null;
+ }
+
+ // implement SqlVisitor
+ public Void visit(SqlDataTypeSpec type) {
+ return null;
+ }
+
+ // implement SqlVisitor
+ public Void visit(SqlDynamicParam param) {
+ return null;
+ }
+
+ // implement SqlVisitor
+ public Void visit(SqlIntervalQualifier intervalQualifier) {
+ return null;
+ }
+
+ public Void visit(SqlCall call) {
+ switch (call.getKind()) {
+ case FILTER:
+ translateAgg((SqlCall) call.operand(0), call.operand(1), call);
+ return null;
+ case SELECT:
+ // rchen 2006-10-17:
+ // for now do not detect aggregates in subqueries.
+ return null;
+ }
+ // ignore window aggregates and ranking functions (associated with OVER operator)
+ if (call.getOperator().getKind() == SqlKind.OVER) {
+ return null;
+ }
+ if (call.getOperator().isAggregator()) {
+ translateAgg(call, null, call);
+ return null;
+ }
+ for (SqlNode operand : call.getOperandList()) {
+ // Operands are occasionally null, e.g. switched CASE arg 0.
+ if (operand != null) {
+ operand.accept(this);
+ }
+ }
+ return null;
+ }
+
+ private void translateAgg(SqlCall call, SqlNode filter, SqlCall outerCall) {
+ assert bb.agg == this;
+ final List<Integer> args = new ArrayList<>();
+ int filterArg = -1;
+ final List<RelDataType> argTypes = call.getOperator() instanceof SqlCountAggFunction ? new ArrayList<RelDataType>(call.getOperandList().size()) : null;
+ try {
+ // switch out of agg mode
+ bb.agg = null;
+ for (SqlNode operand : call.getOperandList()) {
+
+ // special case for COUNT(*): delete the *
+ if (operand instanceof SqlIdentifier) {
+ SqlIdentifier id = (SqlIdentifier) operand;
+ if (id.isStar() || isSimpleCount(call)) { /* OVERRIDE POINT */
+ assert call.operandCount() == 1;
+ assert args.isEmpty();
+ break;
+ }
+ }
+ RexNode convertedExpr = bb.convertExpression(operand);
+ assert convertedExpr != null;
+ if (argTypes != null) {
+ argTypes.add(convertedExpr.getType());
+ }
+ args.add(lookupOrCreateGroupExpr(convertedExpr));
+ }
+
+ if (filter != null) {
+ RexNode convertedExpr = bb.convertExpression(filter);
+ assert convertedExpr != null;
+ filterArg = lookupOrCreateGroupExpr(convertedExpr);
+ }
+ } finally {
+ // switch back into agg mode
+ bb.agg = this;
+ }
+
+ final SqlAggFunction aggFunction = (SqlAggFunction) call.getOperator();
+ RelDataType type = validator.deriveType(bb.scope, call);
+ boolean distinct = false;
+ SqlLiteral quantifier = call.getFunctionQuantifier();
+ if ((null != quantifier) && (quantifier.getValue() == SqlSelectKeyword.DISTINCT)) {
+ distinct = true;
+ }
+ final AggregateCall aggCall = AggregateCall.create(aggFunction, distinct, args, filterArg, type, nameMap.get(outerCall.toString()));
+ RexNode rex = rexBuilder.addAggCall(aggCall, groupExprs.size(), aggregatingSelectScope.indicator, aggCalls, aggCallMapping, argTypes);
+ aggMapping.put(outerCall, rex);
+ }
+
+ /* OVERRIDE POINT */
+ private boolean isSimpleCount(SqlCall call) {
+ if (call.getOperator().isName("COUNT") && call.operandCount() == 1) {
+ final SqlNode parm = call.operand(0);
+ if ((parm instanceof SqlIdentifier || parm instanceof SqlNumericLiteral) //
+ && call.getFunctionQuantifier() == null) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private int lookupOrCreateGroupExpr(RexNode expr) {
+ for (int i = 0; i < convertedInputExprs.size(); i++) {
+ RexNode convertedInputExpr = convertedInputExprs.get(i);
+ if (expr.toString().equals(convertedInputExpr.toString())) {
+ return i;
+ }
+ }
+
+ // not found -- add it
+ int index = convertedInputExprs.size();
+ addExpr(expr, null);
+ return index;
+ }
+
+ /**
+ * If an expression is structurally identical to one of the group-by
+ * expressions, returns a reference to the expression, otherwise returns
+ * null.
+ */
+ public int lookupGroupExpr(SqlNode expr) {
+ for (int i = 0; i < groupExprs.size(); i++) {
+ SqlNode groupExpr = groupExprs.get(i);
+ if (expr.equalsDeep(groupExpr, false)) {
+ return i;
+ }
+ }
+ return -1;
+ }
+
+ public RexNode lookupAggregates(SqlCall call) {
+ // assert call.getOperator().isAggregator();
+ assert bb.agg == this;
+
+ switch (call.getKind()) {
+ case GROUPING:
+ case GROUPING_ID:
+ case GROUP_ID:
+ final RelDataType type = validator.getValidatedNodeType(call);
+ if (!aggregatingSelectScope.indicator) {
+ return rexBuilder.makeExactLiteral(TWO.pow(effectiveArgCount(call)).subtract(BigDecimal.ONE), type);
+ } else {
+ final List<Integer> operands;
+ switch (call.getKind()) {
+ case GROUP_ID:
+ operands = ImmutableIntList.range(0, groupExprs.size());
+ break;
+ default:
+ operands = Lists.newArrayList();
+ for (SqlNode operand : call.getOperandList()) {
+ final int x = lookupGroupExpr(operand);
+ assert x >= 0;
+ operands.add(x);
+ }
+ }
+ RexNode node = null;
+ int shift = operands.size();
+ for (int operand : operands) {
+ node = bitValue(node, type, operand, --shift);
+ }
+ return node;
+ }
+ }
+ return aggMapping.get(call);
+ }
+
+ private int effectiveArgCount(SqlCall call) {
+ switch (call.getKind()) {
+ case GROUPING:
+ return 1;
+ case GROUPING_ID:
+ return call.operandCount();
+ case GROUP_ID:
+ return groupExprs.size();
+ default:
+ throw new AssertionError(call.getKind());
+ }
+ }
+
+ private RexNode bitValue(RexNode previous, RelDataType type, int x, int shift) {
+ RexNode node = rexBuilder.makeCall(SqlStdOperatorTable.CASE, rexBuilder.makeInputRef(bb.root, aggregatingSelectScope.groupExprList.size() + x), rexBuilder.makeExactLiteral(BigDecimal.ONE, type), rexBuilder.makeExactLiteral(BigDecimal.ZERO, type));
+ if (shift > 0) {
+ node = rexBuilder.makeCall(SqlStdOperatorTable.MULTIPLY, node, rexBuilder.makeExactLiteral(TWO.pow(shift), type));
+ }
+ if (previous != null) {
+ node = rexBuilder.makeCall(SqlStdOperatorTable.PLUS, previous, node);
+ }
+ return node;
+ }
+
+ public List<RexNode> getPreExprs() {
+ return convertedInputExprs;
+ }
+
+ public List<String> getPreNames() {
+ return convertedInputExprNames;
+ }
+
+ public List<AggregateCall> getAggCalls() {
+ return aggCalls;
+ }
+
+ public RelDataTypeFactory getTypeFactory() {
+ return typeFactory;
+ }
+ }
+
+ /**
+ * Context to find a relational expression to a field offset.
+ */
+ private static class LookupContext {
+ private final List<Pair<RelNode, Integer>> relOffsetList = new ArrayList<>();
+
+ /**
+ * Creates a LookupContext with multiple input relational expressions.
+ *
+ * @param bb Context for translating this subquery
+ * @param rels Relational expressions
+ * @param systemFieldCount Number of system fields
+ */
+ LookupContext(Blackboard bb, List<RelNode> rels, int systemFieldCount) {
+ bb.flatten(rels, systemFieldCount, new int[] { 0 }, relOffsetList);
+ }
+
+ /**
+ * Returns the relational expression with a given offset, and the
+ * ordinal in the combined row of its first field.
+ *
+ * <p>For example, in {@code Emp JOIN Dept}, findRel(1) returns the
+ * relational expression for {@code Dept} and offset 6 (because
+ * {@code Emp} has 6 fields, therefore the first field of {@code Dept}
+ * is field 6.
+ *
+ * @param offset Offset of relational expression in FROM clause
+ * @return Relational expression and the ordinal of its first field
+ */
+ Pair<RelNode, Integer> findRel(int offset) {
+ return relOffsetList.get(offset);
+ }
+ }
+
+ /**
+ * Shuttle which walks over a tree of {@link RexNode}s and applies 'over' to
+ * all agg functions.
+ *
+ * <p>This is necessary because the returned expression is not necessarily a
+ * call to an agg function. For example,
+ *
+ * <blockquote><code>AVG(x)</code></blockquote>
+ *
+ * becomes
+ *
+ * <blockquote><code>SUM(x) / COUNT(x)</code></blockquote>
+ *
+ * <p>Any aggregate functions are converted to calls to the internal <code>
+ * $Histogram</code> aggregation function and accessors such as <code>
+ * $HistogramMin</code>; for example,
+ *
+ * <blockquote><code>MIN(x), MAX(x)</code></blockquote>
+ *
+ * are converted to
+ *
+ * <blockquote><code>$HistogramMin($Histogram(x)),
+ * $HistogramMax($Histogram(x))</code></blockquote>
+ *
+ * Common sub-expression elmination will ensure that only one histogram is
+ * computed.
+ */
+ private class HistogramShuttle extends RexShuttle {
+ /**
+ * Whether to convert calls to MIN(x) to HISTOGRAM_MIN(HISTOGRAM(x)).
+ * Histograms allow rolling computation, but require more space.
+ */
+ static final boolean ENABLE_HISTOGRAM_AGG = false;
+
+ private final List<RexNode> partitionKeys;
+ private final ImmutableList<RexFieldCollation> orderKeys;
+ private final RexWindowBound lowerBound;
+ private final RexWindowBound upperBound;
+ private final SqlWindow window;
+
+ HistogramShuttle(List<RexNode> partitionKeys, ImmutableList<RexFieldCollation> orderKeys, RexWindowBound lowerBound, RexWindowBound upperBound, SqlWindow window) {
+ this.partitionKeys = partitionKeys;
+ this.orderKeys = orderKeys;
+ this.lowerBound = lowerBound;
+ this.upperBound = upperBound;
+ this.window = window;
+ }
+
+ public RexNode visitCall(RexCall call) {
+ final SqlOperator op = call.getOperator();
+ if (!(op instanceof SqlAggFunction)) {
+ return super.visitCall(call);
+ }
+ final SqlAggFunction aggOp = (SqlAggFunction) op;
+ final RelDataType type = call.getType();
+ List<RexNode> exprs = call.getOperands();
+
+ SqlFunction histogramOp = !ENABLE_HISTOGRAM_AGG ? null : getHistogramOp(aggOp);
+
+ if (histogramOp != null) {
+ final RelDataType histogramType = computeHistogramType(type);
+
+ // For DECIMAL, since it's already represented as a bigint we
+ // want to do a reinterpretCast instead of a cast to avoid
+ // losing any precision.
+ boolean reinterpretCast = type.getSqlTypeName() == SqlTypeName.DECIMAL;
+
+ // Replace original expression with CAST of not one
+ // of the supported types
+ if (histogramType != type) {
+ exprs = new ArrayList<>(exprs);
+ exprs.set(0, reinterpretCast ? rexBuilder.makeReinterpretCast(histogramType, exprs.get(0), rexBuilder.makeLiteral(false)) : rexBuilder.makeCast(histogramType, exprs.get(0)));
+ }
+
+ RexCallBinding bind = new RexCallBinding(rexBuilder.getTypeFactory(), SqlStdOperatorTable.HISTOGRAM_AGG, exprs, ImmutableList.<RelCollation> of());
+
+ RexNode over = rexBuilder.makeOver(SqlStdOperatorTable.HISTOGRAM_AGG.inferReturnType(bind), SqlStdOperatorTable.HISTOGRAM_AGG, exprs, partitionKeys, orderKeys, lowerBound, upperBound, window.isRows(), window.isAllowPartial(), false);
+
+ RexNode histogramCall = rexBuilder.makeCall(histogramType, histogramOp, ImmutableList.of(over));
+
+ // If needed, post Cast result back to original
+ // type.
+ if (histogramType != type) {
+ if (reinterpretCast) {
+ histogramCall = rexBuilder.makeReinterpretCast(type, histogramCall, rexBuilder.makeLiteral(false));
+ } else {
+ histogramCall = rexBuilder.makeCast(type, histogramCall);
+ }
+ }
+
+ return histogramCall;
+ } else {
+ boolean needSum0 = aggOp == SqlStdOperatorTable.SUM && type.isNullable();
+ SqlAggFunction aggOpToUse = needSum0 ? SqlStdOperatorTable.SUM0 : aggOp;
+ return rexBuilder.makeOver(type, aggOpToUse, exprs, partitionKeys, orderKeys, lowerBound, upperBound, window.isRows(), window.isAllowPartial(), needSum0);
+ }
+ }
+
+ /**
+ * Returns the histogram operator corresponding to a given aggregate
+ * function.
+ *
+ * <p>For example, <code>getHistogramOp
+ *({@link SqlStdOperatorTable#MIN}}</code> returns
+ * {@link SqlStdOperatorTable#HISTOGRAM_MIN}.
+ *
+ * @param aggFunction An aggregate function
+ * @return Its histogram function, or null
+ */
+ SqlFunction getHistogramOp(SqlAggFunction aggFunction) {
+ if (aggFunction == SqlStdOperatorTable.MIN) {
+ return SqlStdOperatorTable.HISTOGRAM_MIN;
+ } else if (aggFunction == SqlStdOperatorTable.MAX) {
+ return SqlStdOperatorTable.HISTOGRAM_MAX;
+ } else if (aggFunction == SqlStdOperatorTable.FIRST_VALUE) {
+ return SqlStdOperatorTable.HISTOGRAM_FIRST_VALUE;
+ } else if (aggFunction == SqlStdOperatorTable.LAST_VALUE) {
+ return SqlStdOperatorTable.HISTOGRAM_LAST_VALUE;
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Returns the type for a histogram function. It is either the actual
+ * type or an an approximation to it.
+ */
+ private RelDataType computeHistogramType(RelDataType type) {
+ if (SqlTypeUtil.isExactNumeric(type) && type.getSqlTypeName() != SqlTypeName.BIGINT) {
+ return typeFactory.createSqlType(SqlTypeName.BIGINT);
+ } else if (SqlTypeUtil.isApproximateNumeric(type) && type.getSqlTypeName() != SqlTypeName.DOUBLE) {
+ return typeFactory.createSqlType(SqlTypeName.DOUBLE);
+ } else {
+ return type;
+ }
+ }
+ }
+
+ /** A sub-query, whether it needs to be translated using 2- or 3-valued
+ * logic. */
+ private static class SubQuery {
+ final SqlNode node;
+ final RelOptUtil.Logic logic;
+ RexNode expr;
+
+ private SubQuery(SqlNode node, RelOptUtil.Logic logic) {
+ this.node = node;
+ this.logic = logic;
+ }
+ }
+
+ /**
+ * Visitor that collects all aggregate functions in a {@link SqlNode} tree.
+ */
+ private static class AggregateFinder extends SqlBasicVisitor<Void> {
+ final SqlNodeList list = new SqlNodeList(SqlParserPos.ZERO);
+
+ @Override
+ public Void visit(SqlCall call) {
+ // ignore window aggregates and ranking functions (associated with OVER operator)
+ if (call.getOperator().getKind() == SqlKind.OVER) {
+ return null;
+ }
+ if (call.getOperator().isAggregator()) {
+ list.add(call);
+ return null;
+ }
+
+ // Don't traverse into sub-queries, even if they contain aggregate
+ // functions.
+ if (call instanceof SqlSelect) {
+ return null;
+ }
+
+ return call.getOperator().acceptCall(this, call);
+ }
+ }
+}
+
+// End SqlToRelConverter.java
diff --git a/atopcalcite/src/main/java/org/eigenbase/sql2rel/SqlToRelConverter.java b/atopcalcite/src/main/java/org/eigenbase/sql2rel/SqlToRelConverter.java
deleted file mode 100644
index 15077e0526..0000000000
--- a/atopcalcite/src/main/java/org/eigenbase/sql2rel/SqlToRelConverter.java
+++ /dev/null
@@ -1,4782 +0,0 @@
-/*
- * OVERRIDE POINTS:
- * - getInSubqueryThreshold(), was `20`, now `Integer.MAX_VALUE`
- * - isTrimUnusedFields(), override to false
- */
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.eigenbase.sql2rel;
-
-import java.lang.reflect.Type;
-import java.math.*;
-import java.util.*;
-import java.util.logging.*;
-
-import org.eigenbase.rel.*;
-import org.eigenbase.rel.metadata.*;
-import org.eigenbase.relopt.*;
-import org.eigenbase.reltype.*;
-import org.eigenbase.rex.*;
-import org.eigenbase.sql.*;
-import org.eigenbase.sql.fun.*;
-import org.eigenbase.sql.parser.*;
-import org.eigenbase.sql.type.*;
-import org.eigenbase.sql.util.*;
-import org.eigenbase.sql.validate.*;
-import org.eigenbase.trace.*;
-import org.eigenbase.util.*;
-import org.eigenbase.util.mapping.Mappings;
-import org.eigenbase.util14.*;
-
-import net.hydromatic.linq4j.Ord;
-import net.hydromatic.optiq.ModifiableTable;
-import net.hydromatic.optiq.TranslatableTable;
-import net.hydromatic.optiq.prepare.Prepare;
-import net.hydromatic.optiq.prepare.RelOptTableImpl;
-import net.hydromatic.optiq.util.BitSets;
-
-import com.google.common.base.Function;
-import com.google.common.collect.*;
-
-import static org.eigenbase.sql.SqlUtil.stripAs;
-import static org.eigenbase.util.Static.RESOURCE;
-
-/**
- * Converts a SQL parse tree (consisting of {@link org.eigenbase.sql.SqlNode}
- * objects) into a relational algebra expression (consisting of
- * {@link org.eigenbase.rel.RelNode} objects).
- *
- * <p>The public entry points are: {@link #convertQuery},
- * {@link #convertExpression(SqlNode)}.
- */
-@SuppressWarnings({"unused", "rawtypes", "unchecked", "incomplete-switch", "deprecation"})
-public class SqlToRelConverter {
- //~ Static fields/initializers ---------------------------------------------
-
- protected static final Logger SQL2REL_LOGGER =
- EigenbaseTrace.getSqlToRelTracer();
-
- private static final Function<SubQuery, SqlNode> FN =
- new Function<SubQuery, SqlNode>() {
- public SqlNode apply(SubQuery input) {
- return input.node;
- }
- };
-
- //~ Instance fields --------------------------------------------------------
-
- protected final SqlValidator validator;
- protected final RexBuilder rexBuilder;
- protected final Prepare.CatalogReader catalogReader;
- protected final RelOptCluster cluster;
- private DefaultValueFactory defaultValueFactory;
- private SubqueryConverter subqueryConverter;
- protected final List<RelNode> leaves = new ArrayList<RelNode>();
- private final List<SqlDynamicParam> dynamicParamSqlNodes =
- new ArrayList<SqlDynamicParam>();
- private final SqlOperatorTable opTab;
- private boolean shouldConvertTableAccess;
- protected final RelDataTypeFactory typeFactory;
- private final SqlNodeToRexConverter exprConverter;
- private boolean decorrelationEnabled;
- private boolean trimUnusedFields;
- private boolean shouldCreateValuesRel;
- private boolean isExplain;
- private int nDynamicParamsInExplain;
-
- /**
- * Fields used in name resolution for correlated subqueries.
- */
- private final Map<String, DeferredLookup> mapCorrelToDeferred =
- new HashMap<String, DeferredLookup>();
- private int nextCorrel = 0;
-
- private static final String CORREL_PREFIX = "$cor";
-
- /**
- * Stack of names of datasets requested by the <code>
- * TABLE(SAMPLE(<datasetName>, <query>))</code> construct.
- */
- private final Stack<String> datasetStack = new Stack<String>();
-
- /**
- * Mapping of non-correlated subqueries that have been converted to their
- * equivalent constants. Used to avoid re-evaluating the subquery if it's
- * already been evaluated.
- */
- private final Map<SqlNode, RexNode> mapConvertedNonCorrSubqs =
- new HashMap<SqlNode, RexNode>();
-
- public final RelOptTable.ViewExpander viewExpander;
-
- //~ Constructors -----------------------------------------------------------
- /**
- * Creates a converter.
- *
- * @param viewExpander Preparing statement
- * @param validator Validator
- * @param catalogReader Schema
- * @param planner Planner
- * @param rexBuilder Rex builder
- * @param convertletTable Expression converter
- */
- public SqlToRelConverter(
- RelOptTable.ViewExpander viewExpander,
- SqlValidator validator,
- Prepare.CatalogReader catalogReader,
- RelOptPlanner planner,
- RexBuilder rexBuilder,
- SqlRexConvertletTable convertletTable) {
- this.viewExpander = viewExpander;
- this.opTab =
- (validator
- == null) ? SqlStdOperatorTable.instance()
- : validator.getOperatorTable();
- this.validator = validator;
- this.catalogReader = catalogReader;
- this.defaultValueFactory = new NullDefaultValueFactory();
- this.subqueryConverter = new NoOpSubqueryConverter();
- this.rexBuilder = rexBuilder;
- this.typeFactory = rexBuilder.getTypeFactory();
- RelOptQuery query = new RelOptQuery(planner);
- this.cluster = query.createCluster(typeFactory, rexBuilder);
- this.shouldConvertTableAccess = true;
- this.exprConverter =
- new SqlNodeToRexConverterImpl(convertletTable);
- decorrelationEnabled = true;
- trimUnusedFields = false;
- shouldCreateValuesRel = true;
- isExplain = false;
- nDynamicParamsInExplain = 0;
- }
-
- //~ Methods ----------------------------------------------------------------
-
- /**
- * @return the RelOptCluster in use.
- */
- public RelOptCluster getCluster() {
- return cluster;
- }
-
- /**
- * Returns the row-expression builder.
- */
- public RexBuilder getRexBuilder() {
- return rexBuilder;
- }
-
- /**
- * Returns the number of dynamic parameters encountered during translation;
- * this must only be called after {@link #convertQuery}.
- *
- * @return number of dynamic parameters
- */
- public int getDynamicParamCount() {
- return dynamicParamSqlNodes.size();
- }
-
- /**
- * Returns the type inferred for a dynamic parameter.
- *
- * @param index 0-based index of dynamic parameter
- * @return inferred type, never null
- */
- public RelDataType getDynamicParamType(int index) {
- SqlNode sqlNode = dynamicParamSqlNodes.get(index);
- if (sqlNode == null) {
- throw Util.needToImplement("dynamic param type inference");
- }
- return validator.getValidatedNodeType(sqlNode);
- }
-
- /**
- * Returns the current count of the number of dynamic parameters in an
- * EXPLAIN PLAN statement.
- *
- * @param increment if true, increment the count
- * @return the current count before the optional increment
- */
- public int getDynamicParamCountInExplain(boolean increment) {
- int retVal = nDynamicParamsInExplain;
- if (increment) {
- ++nDynamicParamsInExplain;
- }
- return retVal;
- }
-
- /**
- * @return mapping of non-correlated subqueries that have been converted to
- * the constants that they evaluate to
- */
- public Map<SqlNode, RexNode> getMapConvertedNonCorrSubqs() {
- return mapConvertedNonCorrSubqs;
- }
-
- /**
- * Adds to the current map of non-correlated converted subqueries the
- * elements from another map that contains non-correlated subqueries that
- * have been converted by another SqlToRelConverter.
- *
- * @param alreadyConvertedNonCorrSubqs the other map
- */
- public void addConvertedNonCorrSubqs(
- Map<SqlNode, RexNode> alreadyConvertedNonCorrSubqs) {
- mapConvertedNonCorrSubqs.putAll(alreadyConvertedNonCorrSubqs);
- }
-
- /**
- * Set a new DefaultValueFactory. To have any effect, this must be called
- * before any convert method.
- *
- * @param factory new DefaultValueFactory
- */
- public void setDefaultValueFactory(DefaultValueFactory factory) {
- defaultValueFactory = factory;
- }
-
- /**
- * Sets a new SubqueryConverter. To have any effect, this must be called
- * before any convert method.
- *
- * @param converter new SubqueryConverter
- */
- public void setSubqueryConverter(SubqueryConverter converter) {
- subqueryConverter = converter;
- }
-
- /**
- * Indicates that the current statement is part of an EXPLAIN PLAN statement
- *
- * @param nDynamicParams number of dynamic parameters in the statement
- */
- public void setIsExplain(int nDynamicParams) {
- isExplain = true;
- nDynamicParamsInExplain = nDynamicParams;
- }
-
- /**
- * Controls whether table access references are converted to physical rels
- * immediately. The optimizer doesn't like leaf rels to have
- * {@link Convention#NONE}. However, if we are doing further conversion
- * passes (e.g. {@link RelStructuredTypeFlattener}), then we may need to
- * defer conversion. To have any effect, this must be called before any
- * convert method.
- *
- * @param enabled true for immediate conversion (the default); false to
- * generate logical TableAccessRel instances
- */
- public void enableTableAccessConversion(boolean enabled) {
- shouldConvertTableAccess = enabled;
- }
-
- /**
- * Controls whether instances of {@link ValuesRel} are generated. These may
- * not be supported by all physical implementations. To have any effect,
- * this must be called before any convert method.
- *
- * @param enabled true to allow ValuesRel to be generated (the default);
- * false to force substitution of ProjectRel+OneRowRel instead
- */
- public void enableValuesRelCreation(boolean enabled) {
- shouldCreateValuesRel = enabled;
- }
-
- private void checkConvertedType(SqlNode query, RelNode result) {
- if (!query.isA(SqlKind.DML)) {
- // Verify that conversion from SQL to relational algebra did
- // not perturb any type information. (We can't do this if the
- // SQL statement is something like an INSERT which has no
- // validator type information associated with its result,
- // hence the namespace check above.)
- RelDataType convertedRowType = result.getRowType();
- if (!checkConvertedRowType(query, convertedRowType)) {
- RelDataType validatedRowType =
- validator.getValidatedNodeType(query);
- validatedRowType = uniquifyFields(validatedRowType);
- throw Util.newInternal(
- "Conversion to relational algebra failed to preserve "
- + "datatypes:\n"
- + "validated type:\n"
- + validatedRowType.getFullTypeString()
- + "\nconverted type:\n"
- + convertedRowType.getFullTypeString()
- + "\nrel:\n"
- + RelOptUtil.toString(result));
- }
- }
- }
-
- public RelNode flattenTypes(
- RelNode rootRel,
- boolean restructure) {
- RelStructuredTypeFlattener typeFlattener =
- new RelStructuredTypeFlattener(rexBuilder, createToRelContext());
- return typeFlattener.rewrite(rootRel, restructure);
- }
-
- /**
- * If subquery is correlated and decorrelation is enabled, performs
- * decorrelation.
- *
- * @param query Query
- * @param rootRel Root relational expression
- * @return New root relational expression after decorrelation
- */
- public RelNode decorrelate(SqlNode query, RelNode rootRel) {
- if (!enableDecorrelation()) {
- return rootRel;
- }
- final RelNode result = decorrelateQuery(rootRel);
- if (result != rootRel) {
- checkConvertedType(query, result);
- }
- return result;
- }
-
- /**
- * Walks over a tree of relational expressions, replacing each
- * {@link RelNode} with a 'slimmed down' relational expression that projects
- * only the fields required by its consumer.
- *
- * <p>This may make things easier for the optimizer, by removing crud that
- * would expand the search space, but is difficult for the optimizer itself
- * to do it, because optimizer rules must preserve the number and type of
- * fields. Hence, this transform that operates on the entire tree, similar
- * to the {@link RelStructuredTypeFlattener type-flattening transform}.
- *
- * <p>Currently this functionality is disabled in farrago/luciddb; the
- * default implementation of this method does nothing.
- *
- * @param rootRel Relational expression that is at the root of the tree
- * @return Trimmed relational expression
- */
- public RelNode trimUnusedFields(RelNode rootRel) {
- // Trim fields that are not used by their consumer.
- if (isTrimUnusedFields()) {
- final RelFieldTrimmer trimmer = newFieldTrimmer();
- rootRel = trimmer.trim(rootRel);
- boolean dumpPlan = SQL2REL_LOGGER.isLoggable(Level.FINE);
- if (dumpPlan) {
- SQL2REL_LOGGER.fine(
- RelOptUtil.dumpPlan(
- "Plan after trimming unused fields",
- rootRel,
- false,
- SqlExplainLevel.EXPPLAN_ATTRIBUTES));
- }
- }
- return rootRel;
- }
-
- /**
- * Creates a RelFieldTrimmer.
- *
- * @return Field trimmer
- */
- protected RelFieldTrimmer newFieldTrimmer() {
- return new RelFieldTrimmer(validator);
- }
-
- /**
- * Converts an unvalidated query's parse tree into a relational expression.
- *
- * @param query Query to convert
- * @param needsValidation Whether to validate the query before converting;
- * <code>false</code> if the query has already been
- * validated.
- * @param top Whether the query is top-level, say if its result
- * will become a JDBC result set; <code>false</code> if
- * the query will be part of a view.
- */
- public RelNode convertQuery(
- SqlNode query,
- final boolean needsValidation,
- final boolean top) {
- if (needsValidation) {
- query = validator.validate(query);
- }
-
- RelNode result = convertQueryRecursive(query, top, null);
- checkConvertedType(query, result);
-
- boolean dumpPlan = SQL2REL_LOGGER.isLoggable(Level.FINE);
- if (dumpPlan) {
- SQL2REL_LOGGER.fine(
- RelOptUtil.dumpPlan(
- "Plan after converting SqlNode to RelNode",
- result,
- false,
- SqlExplainLevel.EXPPLAN_ATTRIBUTES));
- }
-
- return result;
- }
-
- protected boolean checkConvertedRowType(
- SqlNode query,
- RelDataType convertedRowType) {
- RelDataType validatedRowType = validator.getValidatedNodeType(query);
- validatedRowType = uniquifyFields(validatedRowType);
-
- return RelOptUtil.equal(
- "validated row type",
- validatedRowType,
- "converted row type",
- convertedRowType,
- false);
- }
-
- protected RelDataType uniquifyFields(RelDataType rowType) {
- return validator.getTypeFactory().createStructType(
- RelOptUtil.getFieldTypeList(rowType),
- SqlValidatorUtil.uniquify(rowType.getFieldNames()));
- }
-
- /**
- * Converts a SELECT statement's parse tree into a relational expression.
- */
- public RelNode convertSelect(SqlSelect select) {
- final SqlValidatorScope selectScope = validator.getWhereScope(select);
- final Blackboard bb = createBlackboard(selectScope, null);
- convertSelectImpl(bb, select);
- return bb.root;
- }
-
- /**
- * Factory method for creating translation workspace.
- */
- protected Blackboard createBlackboard(
- SqlValidatorScope scope,
- Map<String, RexNode> nameToNodeMap) {
- return new Blackboard(scope, nameToNodeMap);
- }
-
- /**
- * Implementation of {@link #convertSelect(SqlSelect)}; derived class may
- * override.
- */
- protected void convertSelectImpl(
- final Blackboard bb,
- SqlSelect select) {
- convertFrom(
- bb,
- select.getFrom());
- convertWhere(
- bb,
- select.getWhere());
-
- List<SqlNode> orderExprList = new ArrayList<SqlNode>();
- List<RelFieldCollation> collationList =
- new ArrayList<RelFieldCollation>();
- gatherOrderExprs(
- bb,
- select,
- select.getOrderList(),
- orderExprList,
- collationList);
- final RelCollation collation =
- cluster.traitSetOf().canonize(RelCollationImpl.of(collationList));
-
- if (validator.isAggregate(select)) {
- convertAgg(
- bb,
- select,
- orderExprList);
- } else {
- convertSelectList(
- bb,
- select,
- orderExprList);
- }
-
- if (select.isDistinct()) {
- distinctify(bb, true);
- }
- convertOrder(
- select, bb, collation, orderExprList, select.getOffset(),
- select.getFetch());
- bb.setRoot(bb.root, true);
- }
-
- /**
- * Having translated 'SELECT ... FROM ... [GROUP BY ...] [HAVING ...]', adds
- * a relational expression to make the results unique.
- *
- * <p>If the SELECT clause contains duplicate expressions, adds {@link
- * ProjectRel}s so that we are grouping on the minimal set of keys. The
- * performance gain isn't huge, but it is difficult to detect these
- * duplicate expressions later.
- *
- * @param bb Blackboard
- * @param checkForDupExprs Check for duplicate expressions
- */
- private void distinctify(
- Blackboard bb,
- boolean checkForDupExprs) {
- // Look for duplicate expressions in the project.
- // Say we have 'select x, y, x, z'.
- // Then dups will be {[2, 0]}
- // and oldToNew will be {[0, 0], [1, 1], [2, 0], [3, 2]}
- RelNode rel = bb.root;
- if (checkForDupExprs && (rel instanceof ProjectRel)) {
- ProjectRel project = (ProjectRel) rel;
- final List<RexNode> projectExprs = project.getProjects();
- List<Integer> origins = new ArrayList<Integer>();
- int dupCount = 0;
- for (int i = 0; i < projectExprs.size(); i++) {
- int x = findExpr(projectExprs.get(i), projectExprs, i);
- if (x >= 0) {
- origins.add(x);
- ++dupCount;
- } else {
- origins.add(i);
- }
- }
- if (dupCount == 0) {
- distinctify(bb, false);
- return;
- }
-
- final Map<Integer, Integer> squished = Maps.newHashMap();
- final List<RelDataTypeField> fields = rel.getRowType().getFieldList();
- final List<Pair<RexNode, String>> newProjects = Lists.newArrayList();
- for (int i = 0; i < fields.size(); i++) {
- if (origins.get(i) == i) {
- squished.put(i, newProjects.size());
- newProjects.add(RexInputRef.of2(i, fields));
- }
- }
- rel =
- new ProjectRel(
- cluster,
- rel,
- Pair.left(newProjects),
- Pair.right(newProjects),
- ProjectRel.Flags.BOXED);
-
- bb.root = rel;
- distinctify(bb, false);
- rel = bb.root;
-
- // Create the expressions to reverse the mapping.
- // Project($0, $1, $0, $2).
- final List<Pair<RexNode, String>> undoProjects = Lists.newArrayList();
- for (int i = 0; i < fields.size(); i++) {
- final int origin = origins.get(i);
- RelDataTypeField field = fields.get(i);
- undoProjects.add(
- Pair.of(
- (RexNode) new RexInputRef(
- squished.get(origin), field.getType()),
- field.getName()));
- }
-
- rel =
- new ProjectRel(
- cluster,
- rel,
- Pair.left(undoProjects),
- Pair.right(undoProjects),
- ProjectRel.Flags.BOXED);
-
- bb.setRoot(
- rel,
- false);
-
- return;
- }
-
- // Usual case: all of the expressions in the SELECT clause are
- // different.
- rel =
- createAggregate(
- bb,
- BitSets.range(rel.getRowType().getFieldCount()),
- ImmutableList.<AggregateCall>of());
-
- bb.setRoot(
- rel,
- false);
- }
-
- private int findExpr(RexNode seek, List<RexNode> exprs, int count) {
- for (int i = 0; i < count; i++) {
- RexNode expr = exprs.get(i);
- if (expr.toString().equals(seek.toString())) {
- return i;
- }
- }
- return -1;
- }
-
- /**
- * Converts a query's ORDER BY clause, if any.
- *
- * @param select Query
- * @param bb Blackboard
- * @param collation Collation list
- * @param orderExprList Method populates this list with orderBy expressions
- * not present in selectList
- * @param offset Expression for number of rows to discard before
- * returning first row
- * @param fetch Expression for number of rows to fetch
- */
- protected void convertOrder(
- SqlSelect select,
- Blackboard bb,
- RelCollation collation,
- List<SqlNode> orderExprList,
- SqlNode offset,
- SqlNode fetch) {
- if (select.getOrderList() == null) {
- assert collation.getFieldCollations().isEmpty();
- if (offset == null && fetch == null) {
- return;
- }
- }
-
- // Create a sorter using the previously constructed collations.
- bb.setRoot(
- new SortRel(
- cluster,
- cluster.traitSetOf(Convention.NONE, collation),
- bb.root,
- collation,
- offset == null ? null : convertExpression(offset),
- fetch == null ? null : convertExpression(fetch)),
- false);
-
- // If extra expressions were added to the project list for sorting,
- // add another project to remove them.
- if (orderExprList.size() > 0) {
- List<RexNode> exprs = new ArrayList<RexNode>();
- final RelDataType rowType = bb.root.getRowType();
- final int fieldCount =
- rowType.getFieldCount() - orderExprList.size();
- for (int i = 0; i < fieldCount; i++) {
- exprs.add(rexBuilder.makeInputRef(bb.root, i));
- }
- bb.setRoot(
- new ProjectRel(
- cluster,
- cluster.traitSetOf(RelCollationImpl.PRESERVE),
- bb.root,
- exprs,
- cluster.getTypeFactory().createStructType(
- rowType.getFieldList().subList(0, fieldCount)),
- ProjectRelBase.Flags.BOXED),
- false);
- }
- }
-
- /**
- * Returns whether a given node contains a {@link SqlInOperator}.
- *
- * @param node a RexNode tree
- */
- private static boolean containsInOperator(
- SqlNode node) {
- try {
- SqlVisitor<Void> visitor =
- new SqlBasicVisitor<Void>() {
- public Void visit(SqlCall call) {
- if (call.getOperator() instanceof SqlInOperator) {
- throw new Util.FoundOne(call);
- }
- return super.visit(call);
- }
- };
- node.accept(visitor);
- return false;
- } catch (Util.FoundOne e) {
- Util.swallow(e, null);
- return true;
- }
- }
-
- /**
- * Push down all the NOT logical operators into any IN/NOT IN operators.
- *
- * @param sqlNode the root node from which to look for NOT operators
- * @return the transformed SqlNode representation with NOT pushed down.
- */
- private static SqlNode pushDownNotForIn(SqlNode sqlNode) {
- if ((sqlNode instanceof SqlCall) && containsInOperator(sqlNode)) {
- SqlCall sqlCall = (SqlCall) sqlNode;
- if ((sqlCall.getOperator() == SqlStdOperatorTable.AND)
- || (sqlCall.getOperator() == SqlStdOperatorTable.OR)) {
- SqlNode[] sqlOperands = ((SqlBasicCall) sqlCall).operands;
- for (int i = 0; i < sqlOperands.length; i++) {
- sqlOperands[i] = pushDownNotForIn(sqlOperands[i]);
- }
- return sqlNode;
- } else if (sqlCall.getOperator() == SqlStdOperatorTable.NOT) {
- SqlNode childNode = sqlCall.operand(0);
- assert childNode instanceof SqlCall;
- SqlBasicCall childSqlCall = (SqlBasicCall) childNode;
- if (childSqlCall.getOperator() == SqlStdOperatorTable.AND) {
- SqlNode[] andOperands = childSqlCall.getOperands();
- SqlNode[] orOperands = new SqlNode[andOperands.length];
- for (int i = 0; i < orOperands.length; i++) {
- orOperands[i] =
- SqlStdOperatorTable.NOT.createCall(
- SqlParserPos.ZERO,
- andOperands[i]);
- }
- for (int i = 0; i < orOperands.length; i++) {
- orOperands[i] = pushDownNotForIn(orOperands[i]);
- }
- return SqlStdOperatorTable.OR.createCall(SqlParserPos.ZERO,
- orOperands[0], orOperands[1]);
- } else if (childSqlCall.getOperator() == SqlStdOperatorTable.OR) {
- SqlNode[] orOperands = childSqlCall.getOperands();
- SqlNode[] andOperands = new SqlNode[orOperands.length];
- for (int i = 0; i < andOperands.length; i++) {
- andOperands[i] =
- SqlStdOperatorTable.NOT.createCall(
- SqlParserPos.ZERO,
- orOperands[i]);
- }
- for (int i = 0; i < andOperands.length; i++) {
- andOperands[i] = pushDownNotForIn(andOperands[i]);
- }
- return SqlStdOperatorTable.AND.createCall(SqlParserPos.ZERO,
- andOperands[0], andOperands[1]);
- } else if (childSqlCall.getOperator() == SqlStdOperatorTable.NOT) {
- SqlNode[] notOperands = childSqlCall.getOperands();
- assert notOperands.length == 1;
- return pushDownNotForIn(notOperands[0]);
- } else if (childSqlCall.getOperator() instanceof SqlInOperator) {
- SqlNode[] inOperands = childSqlCall.getOperands();
- SqlInOperator inOp =
- (SqlInOperator) childSqlCall.getOperator();
- if (inOp.isNotIn()) {
- return SqlStdOperatorTable.IN.createCall(
- SqlParserPos.ZERO,
- inOperands[0],
- inOperands[1]);
- } else {
- return SqlStdOperatorTable.NOT_IN.createCall(
- SqlParserPos.ZERO,
- inOperands[0],
- inOperands[1]);
- }
- } else {
- // childSqlCall is "leaf" node in a logical expression tree
- // (only considering AND, OR, NOT)
- return sqlNode;
- }
- } else {
- // sqlNode is "leaf" node in a logical expression tree
- // (only considering AND, OR, NOT)
- return sqlNode;
- }
- } else {
- // tree rooted at sqlNode does not contain inOperator
- return sqlNode;
- }
- }
-
- /**
- * Converts a WHERE clause.
- *
- * @param bb Blackboard
- * @param where WHERE clause, may be null
- */
- private void convertWhere(
- final Blackboard bb,
- final SqlNode where) {
- if (where == null) {
- return;
- }
- SqlNode newWhere = pushDownNotForIn(where);
- replaceSubqueries(bb, newWhere, RelOptUtil.Logic.UNKNOWN_AS_FALSE);
- final RexNode convertedWhere = bb.convertExpression(newWhere);
-
- // only allocate filter if the condition is not TRUE
- if (!convertedWhere.isAlwaysTrue()) {
- bb.setRoot(
- RelOptUtil.createFilter(bb.root, convertedWhere),
- false);
- }
- }
-
- private void replaceSubqueries(
- final Blackboard bb,
- final SqlNode expr,
- RelOptUtil.Logic logic) {
- findSubqueries(bb, expr, logic, false);
- for (SubQuery node : bb.subqueryList) {
- substituteSubquery(bb, node);
- }
- }
-
- private void substituteSubquery(Blackboard bb, SubQuery subQuery) {
- final RexNode expr = subQuery.expr;
- if (expr != null) {
- // Already done.
- return;
- }
-
- final SqlBasicCall call;
- final RelNode rel;
- final SqlNode query;
- final Pair<RelNode, Boolean> converted;
- switch (subQuery.node.getKind()) {
- case CURSOR:
- convertCursor(bb, subQuery);
- return;
-
- case MULTISET_QUERY_CONSTRUCTOR:
- case MULTISET_VALUE_CONSTRUCTOR:
- rel = convertMultisets(ImmutableList.of(subQuery.node), bb);
- subQuery.expr = bb.register(rel, JoinRelType.INNER);
- return;
-
- case IN:
- call = (SqlBasicCall) subQuery.node;
- final SqlNode[] operands = call.getOperands();
-
- SqlNode leftKeyNode = operands[0];
- query = operands[1];
-
- final List<RexNode> leftKeys;
- switch (leftKeyNode.getKind()) {
- case ROW:
- leftKeys = Lists.newArrayList();
- for (SqlNode sqlExpr : ((SqlBasicCall) leftKeyNode).getOperandList()) {
- leftKeys.add(bb.convertExpression(sqlExpr));
- }
- break;
- default:
- leftKeys = ImmutableList.of(bb.convertExpression(leftKeyNode));
- }
-
- final boolean isNotIn = ((SqlInOperator) call.getOperator()).isNotIn();
- if (query instanceof SqlNodeList) {
- SqlNodeList valueList = (SqlNodeList) query;
- if (!containsNullLiteral(valueList)
- && valueList.size() < getInSubqueryThreshold()) {
- // We're under the threshold, so convert to OR.
- subQuery.expr =
- convertInToOr(
- bb,
- leftKeys,
- valueList,
- isNotIn);
- return;
- }
-
- // Otherwise, let convertExists translate
- // values list into an inline table for the
- // reference to Q below.
- }
-
- // Project out the search columns from the left side
-
- // Q1:
- // "select from emp where emp.deptno in (select col1 from T)"
- //
- // is converted to
- //
- // "select from
- // emp inner join (select distinct col1 from T)) q
- // on emp.deptno = q.col1
- //
- // Q2:
- // "select from emp where emp.deptno not in (Q)"
- //
- // is converted to
- //
- // "select from
- // emp left outer join (select distinct col1, TRUE from T) q
- // on emp.deptno = q.col1
- // where emp.deptno <> null
- // and q.indicator <> TRUE"
- //
- final boolean outerJoin = bb.subqueryNeedsOuterJoin
- || isNotIn
- || subQuery.logic == RelOptUtil.Logic.TRUE_FALSE_UNKNOWN;
- converted =
- convertExists(query, RelOptUtil.SubqueryType.IN, subQuery.logic,
- outerJoin);
- if (converted.right) {
- // Generate
- // emp CROSS JOIN (SELECT COUNT(*) AS c,
- // COUNT(deptno) AS ck FROM dept)
- final RelDataType longType =
- typeFactory.createSqlType(SqlTypeName.BIGINT);
- final RelNode seek = converted.left.getInput(0); // fragile
- final int keyCount = leftKeys.size();
- final List<Integer> args = ImmutableIntList.range(0, keyCount);
- AggregateRel aggregate =
- new AggregateRel(cluster, seek, BitSets.of(),
- ImmutableList.of(
- new AggregateCall(SqlStdOperatorTable.COUNT, false,
- ImmutableList.<Integer>of(), longType, null),
- new AggregateCall(SqlStdOperatorTable.COUNT, false,
- args, longType, null)));
- JoinRel join =
- new JoinRel(cluster, bb.root, aggregate,
- rexBuilder.makeLiteral(true), JoinRelType.INNER,
- ImmutableSet.<String>of());
- bb.setRoot(join, false);
- }
- RexNode rex =
- bb.register(converted.left,
- outerJoin ? JoinRelType.LEFT : JoinRelType.INNER, leftKeys);
-
- subQuery.expr = translateIn(subQuery, bb.root, rex);
- if (isNotIn) {
- subQuery.expr =
- rexBuilder.makeCall(SqlStdOperatorTable.NOT, subQuery.expr);
- }
- return;
-
- case EXISTS:
- // "select from emp where exists (select a from T)"
- //
- // is converted to the following if the subquery is correlated:
- //
- // "select from emp left outer join (select AGG_TRUE() as indicator
- // from T group by corr_var) q where q.indicator is true"
- //
- // If there is no correlation, the expression is replaced with a
- // boolean indicating whether the subquery returned 0 or >= 1 row.
- call = (SqlBasicCall) subQuery.node;
- query = call.getOperands()[0];
- converted = convertExists(query, RelOptUtil.SubqueryType.EXISTS,
- subQuery.logic, true);
- assert !converted.right;
- if (convertNonCorrelatedSubQuery(subQuery, bb, converted.left, true)) {
- return;
- }
- subQuery.expr = bb.register(converted.left, JoinRelType.LEFT);
- return;
-
- case SCALAR_QUERY:
- // Convert the subquery. If it's non-correlated, convert it
- // to a constant expression.
- call = (SqlBasicCall) subQuery.node;
- query = call.getOperands()[0];
- converted = convertExists(query, RelOptUtil.SubqueryType.SCALAR,
- subQuery.logic, true);
- assert !converted.right;
- if (convertNonCorrelatedSubQuery(subQuery, bb, converted.left, false)) {
- return;
- }
- rel = convertToSingleValueSubq(query, converted.left);
- subQuery.expr = bb.register(rel, JoinRelType.LEFT);
- return;
-
- case SELECT:
- // This is used when converting multiset queries:
- //
- // select * from unnest(select multiset[deptno] from emps);
- //
- converted = convertExists(subQuery.node, RelOptUtil.SubqueryType.SCALAR,
- subQuery.logic, true);
- assert !converted.right;
- subQuery.expr = bb.register(converted.left, JoinRelType.LEFT);
- return;
-
- default:
- throw Util.newInternal("unexpected kind of subquery :" + subQuery.node);
- }
- }
-
- private RexNode translateIn(SubQuery subQuery, RelNode root,
- final RexNode rex) {
- switch (subQuery.logic) {
- case TRUE:
- return rexBuilder.makeLiteral(true);
-
- case UNKNOWN_AS_FALSE:
- assert rex instanceof RexRangeRef;
- final int fieldCount = rex.getType().getFieldCount();
- RexNode rexNode = rexBuilder.makeFieldAccess(rex, fieldCount - 1);
- rexNode = rexBuilder.makeCall(SqlStdOperatorTable.IS_TRUE, rexNode);
-
- // Then append the IS NOT NULL(leftKeysForIn).
- //
- // RexRangeRef contains the following fields:
- // leftKeysForIn,
- // rightKeysForIn (the original subquery select list),
- // nullIndicator
- //
- // The first two lists contain the same number of fields.
- final int k = (fieldCount - 1) / 2;
- for (int i = 0; i < k; i++) {
- rexNode =
- rexBuilder.makeCall(
- SqlStdOperatorTable.AND,
- rexNode,
- rexBuilder.makeCall(
- SqlStdOperatorTable.IS_NOT_NULL,
- rexBuilder.makeFieldAccess(rex, i)));
- }
- return rexNode;
-
- case TRUE_FALSE_UNKNOWN:
- case UNKNOWN_AS_TRUE:
- // select e.deptno,
- // case
- // when ct.c = 0 then false
- // when dt.i is not null then true
- // when e.deptno is null then null
- // when ct.ck < ct.c then null
- // else false
- // end
- // from e
- // cross join (select count(*) as c, count(deptno) as ck from v) as ct
- // left join (select distinct deptno, true as i from v) as dt
- // on e.deptno = dt.deptno
- final JoinRelBase join = (JoinRelBase) root;
- final ProjectRelBase left = (ProjectRelBase) join.getLeft();
- final RelNode leftLeft = ((JoinRelBase) left.getInput(0)).getLeft();
- final int leftLeftCount = leftLeft.getRowType().getFieldCount();
- final RelDataType nullableBooleanType =
- typeFactory.createTypeWithNullability(
- typeFactory.createSqlType(SqlTypeName.BOOLEAN), true);
- final RelDataType longType =
- typeFactory.createSqlType(SqlTypeName.BIGINT);
- final RexNode cRef = rexBuilder.makeInputRef(root, leftLeftCount);
- final RexNode ckRef = rexBuilder.makeInputRef(root, leftLeftCount + 1);
- final RexNode iRef =
- rexBuilder.makeInputRef(root, root.getRowType().getFieldCount() - 1);
-
- final RexLiteral zero =
- rexBuilder.makeExactLiteral(BigDecimal.ZERO, longType);
- final RexLiteral trueLiteral = rexBuilder.makeLiteral(true);
- final RexLiteral falseLiteral = rexBuilder.makeLiteral(false);
- final RexNode unknownLiteral =
- rexBuilder.makeNullLiteral(SqlTypeName.BOOLEAN);
-
- final ImmutableList.Builder<RexNode> args = ImmutableList.builder();
- args.add(rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, cRef, zero),
- falseLiteral,
- rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, iRef),
- trueLiteral);
- final JoinInfo joinInfo = join.analyzeCondition();
- for (int leftKey : joinInfo.leftKeys) {
- final RexNode kRef = rexBuilder.makeInputRef(root, leftKey);
- args.add(rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, kRef),
- unknownLiteral);
- }
- args.add(rexBuilder.makeCall(SqlStdOperatorTable.LESS_THAN, ckRef, cRef),
- unknownLiteral,
- falseLiteral);
-
- return rexBuilder.makeCall(
- nullableBooleanType,
- SqlStdOperatorTable.CASE,
- args.build());
-
- default:
- throw new AssertionError(subQuery.logic);
- }
- }
-
- private static boolean containsNullLiteral(SqlNodeList valueList) {
- for (SqlNode node : valueList.getList()) {
- if (node instanceof SqlLiteral) {
- SqlLiteral lit = (SqlLiteral) node;
- if (lit.getValue() == null) {
- return true;
- }
- }
- }
- return false;
- }
-
- /**
- * Determines if a subquery is non-correlated and if so, converts it to a
- * constant.
- *
- * @param subQuery the call that references the subquery
- * @param bb blackboard used to convert the subquery
- * @param converted RelNode tree corresponding to the subquery
- * @param isExists true if the subquery is part of an EXISTS expression
- * @return if the subquery can be converted to a constant
- */
- private boolean convertNonCorrelatedSubQuery(
- SubQuery subQuery,
- Blackboard bb,
- RelNode converted,
- boolean isExists) {
- SqlCall call = (SqlBasicCall) subQuery.node;
- if (subqueryConverter.canConvertSubquery()
- && isSubqNonCorrelated(converted, bb)) {
- // First check if the subquery has already been converted
- // because it's a nested subquery. If so, don't re-evaluate
- // it again.
- RexNode constExpr = mapConvertedNonCorrSubqs.get(call);
- if (constExpr == null) {
- constExpr =
- subqueryConverter.convertSubquery(
- call,
- this,
- isExists,
- isExplain);
- }
- if (constExpr != null) {
- subQuery.expr = constExpr;
- mapConvertedNonCorrSubqs.put(call, constExpr);
- return true;
- }
- }
- return false;
- }
-
- /**
- * Converts the RelNode tree for a select statement to a select that
- * produces a single value.
- *
- * @param query the query
- * @param plan the original RelNode tree corresponding to the statement
- * @return the converted RelNode tree
- */
- public RelNode convertToSingleValueSubq(
- SqlNode query,
- RelNode plan) {
- // Check whether query is guaranteed to produce a single value.
- if (query instanceof SqlSelect) {
- SqlSelect select = (SqlSelect) query;
- SqlNodeList selectList = select.getSelectList();
- SqlNodeList groupList = select.getGroup();
-
- if ((selectList.size() == 1)
- && ((groupList == null) || (groupList.size() == 0))) {
- SqlNode selectExpr = selectList.get(0);
- if (selectExpr instanceof SqlCall) {
- SqlCall selectExprCall = (SqlCall) selectExpr;
- if (selectExprCall.getOperator()
- instanceof SqlAggFunction) {
- return plan;
- }
- }
- }
- }
-
- // If not, project SingleValueAgg
- return RelOptUtil.createSingleValueAggRel(
- cluster,
- plan);
- }
-
- /**
- * Converts "x IN (1, 2, ...)" to "x=1 OR x=2 OR ...".
- *
- * @param leftKeys LHS
- * @param valuesList RHS
- * @param isNotIn is this a NOT IN operator
- * @return converted expression
- */
- private RexNode convertInToOr(
- final Blackboard bb,
- final List<RexNode> leftKeys,
- SqlNodeList valuesList,
- boolean isNotIn) {
- List<RexNode> comparisons = new ArrayList<RexNode>();
- for (SqlNode rightVals : valuesList) {
- RexNode rexComparison;
- if (leftKeys.size() == 1) {
- rexComparison =
- rexBuilder.makeCall(
- SqlStdOperatorTable.EQUALS,
- leftKeys.get(0),
- bb.convertExpression(rightVals));
- } else {
- assert rightVals instanceof SqlCall;
- final SqlBasicCall call = (SqlBasicCall) rightVals;
- assert (call.getOperator() instanceof SqlRowOperator)
- && call.getOperands().length == leftKeys.size();
- rexComparison =
- RexUtil.composeConjunction(
- rexBuilder,
- Iterables.transform(
- Pair.zip(leftKeys, call.getOperandList()),
- new Function<Pair<RexNode, SqlNode>, RexNode>() {
- public RexNode apply(Pair<RexNode, SqlNode> pair) {
- return rexBuilder.makeCall(SqlStdOperatorTable.EQUALS,
- pair.left, bb.convertExpression(pair.right));
- }
- }),
- false);
- }
- comparisons.add(rexComparison);
- }
-
- RexNode result =
- RexUtil.composeDisjunction(rexBuilder, comparisons, true);
- assert result != null;
-
- if (isNotIn) {
- result =
- rexBuilder.makeCall(
- SqlStdOperatorTable.NOT,
- result);
- }
-
- return result;
- }
-
- /**
- * Gets the list size threshold under which {@link #convertInToOr} is used.
- * Lists of this size or greater will instead be converted to use a join
- * against an inline table ({@link ValuesRel}) rather than a predicate. A
- * threshold of 0 forces usage of an inline table in all cases; a threshold
- * of Integer.MAX_VALUE forces usage of OR in all cases
- *
- * @return threshold, default 20
- */
- protected int getInSubqueryThreshold() {
- // OVERRIDE POINT
- return Integer.MAX_VALUE; // was 20
- }
-
- /**
- * Converts an EXISTS or IN predicate into a join. For EXISTS, the subquery
- * produces an indicator variable, and the result is a relational expression
- * which outer joins that indicator to the original query. After performing
- * the outer join, the condition will be TRUE if the EXISTS condition holds,
- * NULL otherwise.
- *
- * @param seek A query, for example 'select * from emp' or
- * 'values (1,2,3)' or '('Foo', 34)'.
- * @param subqueryType Whether sub-query is IN, EXISTS or scalar
- * @param logic Whether the answer needs to be in full 3-valued logic (TRUE,
- * FALSE, UNKNOWN) will be required, or whether we can accept an
- * approximation (say representing UNKNOWN as FALSE)
- * @param needsOuterJoin Whether an outer join is needed
- * @return join expression
- * @pre extraExpr == null || extraName != null
- */
- private Pair<RelNode, Boolean> convertExists(
- SqlNode seek,
- RelOptUtil.SubqueryType subqueryType,
- RelOptUtil.Logic logic,
- boolean needsOuterJoin) {
- final SqlValidatorScope seekScope =
- (seek instanceof SqlSelect)
- ? validator.getSelectScope((SqlSelect) seek)
- : null;
- final Blackboard seekBb = createBlackboard(seekScope, null);
- RelNode seekRel = convertQueryOrInList(seekBb, seek);
-
- return RelOptUtil.createExistsPlan(seekRel, subqueryType, logic,
- needsOuterJoin);
- }
-
- private RelNode convertQueryOrInList(
- Blackboard bb,
- SqlNode seek) {
- // NOTE: Once we start accepting single-row queries as row constructors,
- // there will be an ambiguity here for a case like X IN ((SELECT Y FROM
- // Z)). The SQL standard resolves the ambiguity by saying that a lone
- // select should be interpreted as a table expression, not a row
- // expression. The semantic difference is that a table expression can
- // return multiple rows.
- if (seek instanceof SqlNodeList) {
- return convertRowValues(
- bb,
- seek,
- ((SqlNodeList) seek).getList(),
- false,
- null);
- } else {
- return convertQueryRecursive(seek, false, null);
- }
- }
-
- private RelNode convertRowValues(
- Blackboard bb,
- SqlNode rowList,
- Collection<SqlNode> rows,
- boolean allowLiteralsOnly,
- RelDataType targetRowType) {
- // NOTE jvs 30-Apr-2006: We combine all rows consisting entirely of
- // literals into a single ValuesRel; this gives the optimizer a smaller
- // input tree. For everything else (computed expressions, row
- // subqueries), we union each row in as a projection on top of a
- // OneRowRel.
-
- final List<List<RexLiteral>> tupleList =
- new ArrayList<List<RexLiteral>>();
- final RelDataType rowType;
- if (targetRowType != null) {
- rowType = targetRowType;
- } else {
- rowType =
- SqlTypeUtil.promoteToRowType(
- typeFactory,
- validator.getValidatedNodeType(rowList),
- null);
- }
-
- List<RelNode> unionInputs = new ArrayList<RelNode>();
- for (SqlNode node : rows) {
- SqlBasicCall call;
- if (isRowConstructor(node)) {
- call = (SqlBasicCall) node;
- List<RexLiteral> tuple = new ArrayList<RexLiteral>();
- for (SqlNode operand : call.operands) {
- RexLiteral rexLiteral =
- convertLiteralInValuesList(
- operand,
- bb,
- rowType,
- tuple.size());
- if ((rexLiteral == null) && allowLiteralsOnly) {
- return null;
- }
- if ((rexLiteral == null) || !shouldCreateValuesRel) {
- // fallback to convertRowConstructor
- tuple = null;
- break;
- }
- tuple.add(rexLiteral);
- }
- if (tuple != null) {
- tupleList.add(tuple);
- continue;
- }
- } else {
- RexLiteral rexLiteral =
- convertLiteralInValuesList(
- node,
- bb,
- rowType,
- 0);
- if ((rexLiteral != null) && shouldCreateValuesRel) {
- tupleList.add(
- Collections.singletonList(rexLiteral));
- continue;
- } else {
- if ((rexLiteral == null) && allowLiteralsOnly) {
- return null;
- }
- }
-
- // convert "1" to "row(1)"
- call =
- (SqlBasicCall) SqlStdOperatorTable.ROW.createCall(
- SqlParserPos.ZERO,
- node);
- }
- unionInputs.add(convertRowConstructor(bb, call));
- }
- ValuesRel valuesRel =
- new ValuesRel(
- cluster,
- rowType,
- tupleList);
- RelNode resultRel;
- if (unionInputs.isEmpty()) {
- resultRel = valuesRel;
- } else {
- if (!tupleList.isEmpty()) {
- unionInputs.add(valuesRel);
- }
- UnionRel unionRel =
- new UnionRel(
- cluster,
- unionInputs,
- true);
- resultRel = unionRel;
- }
- leaves.add(resultRel);
- return resultRel;
- }
-
- private RexLiteral convertLiteralInValuesList(
- SqlNode sqlNode,
- Blackboard bb,
- RelDataType rowType,
- int iField) {
- if (!(sqlNode instanceof SqlLiteral)) {
- return null;
- }
- RelDataTypeField field = rowType.getFieldList().get(iField);
- RelDataType type = field.getType();
- if (type.isStruct()) {
- // null literals for weird stuff like UDT's need
- // special handling during type flattening, so
- // don't use ValuesRel for those
- return null;
- }
-
- RexNode literalExpr =
- exprConverter.convertLiteral(
- bb,
- (SqlLiteral) sqlNode);
-
- if (!(literalExpr instanceof RexLiteral)) {
- assert literalExpr.isA(SqlKind.CAST);
- RexNode child = ((RexCall) literalExpr).getOperands().get(0);
- assert RexLiteral.isNullLiteral(child);
-
- // NOTE jvs 22-Nov-2006: we preserve type info
- // in ValuesRel digest, so it's OK to lose it here
- return (RexLiteral) child;
- }
-
- RexLiteral literal = (RexLiteral) literalExpr;
-
- Comparable value = literal.getValue();
-
- if (SqlTypeUtil.isExactNumeric(type)) {
- BigDecimal roundedValue =
- NumberUtil.rescaleBigDecimal(
- (BigDecimal) value,
- type.getScale());
- return rexBuilder.makeExactLiteral(
- roundedValue,
- type);
- }
-
- if ((value instanceof NlsString)
- && (type.getSqlTypeName() == SqlTypeName.CHAR)) {
- // pad fixed character type
- NlsString unpadded = (NlsString) value;
- return rexBuilder.makeCharLiteral(
- new NlsString(
- Util.rpad(unpadded.getValue(), type.getPrecision()),
- unpadded.getCharsetName(),
- unpadded.getCollation()));
- }
- return literal;
- }
-
- private boolean isRowConstructor(SqlNode node) {
- if (!(node.getKind() == SqlKind.ROW)) {
- return false;
- }
- SqlCall call = (SqlCall) node;
- return call.getOperator().getName().equalsIgnoreCase("row");
- }
-
- /**
- * Builds a list of all <code>IN</code> or <code>EXISTS</code> operators
- * inside SQL parse tree. Does not traverse inside queries.
- *
- * @param bb blackboard
- * @param node the SQL parse tree
- * @param logic Whether the answer needs to be in full 3-valued logic (TRUE,
- * FALSE, UNKNOWN) will be required, or whether we can accept
- * an approximation (say representing UNKNOWN as FALSE)
- * @param registerOnlyScalarSubqueries if set to true and the parse tree
- * corresponds to a variation of a select
- * node, only register it if it's a scalar
- * subquery
- */
-private void findSubqueries(
- Blackboard bb,
- SqlNode node,
- RelOptUtil.Logic logic,
- boolean registerOnlyScalarSubqueries) {
- final SqlKind kind = node.getKind();
- switch (kind) {
- case EXISTS:
- case SELECT:
- case MULTISET_QUERY_CONSTRUCTOR:
- case MULTISET_VALUE_CONSTRUCTOR:
- case CURSOR:
- case SCALAR_QUERY:
- if (!registerOnlyScalarSubqueries
- || (kind == SqlKind.SCALAR_QUERY)) {
- bb.registerSubquery(node, RelOptUtil.Logic.TRUE_FALSE);
- }
- return;
- case IN:
- if (((SqlCall) node).getOperator() == SqlStdOperatorTable.NOT_IN) {
- logic = logic.negate();
- }
- break;
- case NOT:
- logic = logic.negate();
- break;
- }
- if (node instanceof SqlCall) {
- if (kind == SqlKind.OR
- || kind == SqlKind.NOT) {
- // It's always correct to outer join subquery with
- // containing query; however, when predicates involve Or
- // or NOT, outer join might be necessary.
- bb.subqueryNeedsOuterJoin = true;
- }
- for (SqlNode operand : ((SqlCall) node).getOperandList()) {
- if (operand != null) {
- // In the case of an IN expression, locate scalar
- // subqueries so we can convert them to constants
- findSubqueries(
- bb,
- operand,
- logic,
- kind == SqlKind.IN || registerOnlyScalarSubqueries);
- }
- }
- } else if (node instanceof SqlNodeList) {
- for (SqlNode child : (SqlNodeList) node) {
- findSubqueries(
- bb,
- child,
- logic,
- kind == SqlKind.IN || registerOnlyScalarSubqueries);
- }
- }
-
- // Now that we've located any scalar subqueries inside the IN
- // expression, register the IN expression itself. We need to
- // register the scalar subqueries first so they can be converted
- // before the IN expression is converted.
- if (kind == SqlKind.IN) {
- if (logic == RelOptUtil.Logic.TRUE_FALSE_UNKNOWN
- && !validator.getValidatedNodeType(node).isNullable()) {
- logic = RelOptUtil.Logic.UNKNOWN_AS_FALSE;
- }
- // TODO: This conversion is only valid in the WHERE clause
- if (logic == RelOptUtil.Logic.UNKNOWN_AS_FALSE
- && !bb.subqueryNeedsOuterJoin) {
- logic = RelOptUtil.Logic.TRUE;
- }
- bb.registerSubquery(node, logic);
- }
- }
-
- /**
- * Converts an expression from {@link SqlNode} to {@link RexNode} format.
- *
- * @param node Expression to translate
- * @return Converted expression
- */
- public RexNode convertExpression(
- SqlNode node) {
- Map<String, RelDataType> nameToTypeMap = Collections.emptyMap();
- Blackboard bb =
- createBlackboard(
- new ParameterScope((SqlValidatorImpl) validator, nameToTypeMap),
- null);
- return bb.convertExpression(node);
- }
-
- /**
- * Converts an expression from {@link SqlNode} to {@link RexNode} format,
- * mapping identifier references to predefined expressions.
- *
- * @param node Expression to translate
- * @param nameToNodeMap map from String to {@link RexNode}; when an
- * {@link SqlIdentifier} is encountered, it is used as a
- * key and translated to the corresponding value from
- * this map
- * @return Converted expression
- */
- public RexNode convertExpression(
- SqlNode node,
- Map<String, RexNode> nameToNodeMap) {
- final Map<String, RelDataType> nameToTypeMap =
- new HashMap<String, RelDataType>();
- for (Map.Entry<String, RexNode> entry : nameToNodeMap.entrySet()) {
- nameToTypeMap.put(entry.getKey(), entry.getValue().getType());
- }
- Blackboard bb =
- createBlackboard(
- new ParameterScope((SqlValidatorImpl) validator, nameToTypeMap),
- nameToNodeMap);
- return bb.convertExpression(node);
- }
-
- /**
- * Converts a non-standard expression.
- *
- * <p>This method is an extension-point that derived classes can override. If
- * this method returns a null result, the normal expression translation
- * process will proceed. The default implementation always returns null.
- *
- * @param node Expression
- * @param bb Blackboard
- * @return null to proceed with the usual expression translation process
- */
- protected RexNode convertExtendedExpression(
- SqlNode node,
- Blackboard bb) {
- return null;
- }
-
- private RexNode convertOver(Blackboard bb, SqlNode node) {
- SqlCall call = (SqlCall) node;
- SqlCall aggCall = call.operand(0);
- SqlNode windowOrRef = call.operand(1);
- final SqlWindow window =
- validator.resolveWindow(windowOrRef, bb.scope, true);
- final SqlNodeList partitionList = window.getPartitionList();
- final ImmutableList.Builder<RexNode> partitionKeys =
- ImmutableList.builder();
- for (SqlNode partition : partitionList) {
- partitionKeys.add(bb.convertExpression(partition));
- }
- RexNode lowerBound = bb.convertExpression(window.getLowerBound());
- RexNode upperBound = bb.convertExpression(window.getUpperBound());
- SqlNodeList orderList = window.getOrderList();
- if ((orderList.size() == 0) && !window.isRows()) {
- // A logical range requires an ORDER BY clause. Use the implicit
- // ordering of this relation. There must be one, otherwise it would
- // have failed validation.
- orderList = bb.scope.getOrderList();
- if (orderList == null) {
- throw new AssertionError(
- "Relation should have sort key for implicit ORDER BY");
- }
- }
- final ImmutableList.Builder<RexFieldCollation> orderKeys =
- ImmutableList.builder();
- final Set<SqlKind> flags = EnumSet.noneOf(SqlKind.class);
- for (SqlNode order : orderList) {
- flags.clear();
- RexNode e = bb.convertSortExpression(order, flags);
- orderKeys.add(new RexFieldCollation(e, flags));
- }
- try {
- Util.permAssert(bb.window == null, "already in window agg mode");
- bb.window = window;
- RexNode rexAgg = exprConverter.convertCall(bb, aggCall);
- rexAgg =
- rexBuilder.ensureType(
- validator.getValidatedNodeType(call), rexAgg, false);
-
- // Walk over the tree and apply 'over' to all agg functions. This is
- // necessary because the returned expression is not necessarily a call
- // to an agg function. For example, AVG(x) becomes SUM(x) / COUNT(x).
- final RexShuttle visitor =
- new HistogramShuttle(
- partitionKeys.build(), orderKeys.build(),
- RexWindowBound.create(window.getLowerBound(), lowerBound),
- RexWindowBound.create(window.getUpperBound(), upperBound),
- window);
- return rexAgg.accept(visitor);
- } finally {
- bb.window = null;
- }
- }
-
- /**
- * Converts a FROM clause into a relational expression.
- *
- * @param bb Scope within which to resolve identifiers
- * @param from FROM clause of a query. Examples include:
- *
- * <ul>
- * <li>a single table ("SALES.EMP"),
- * <li>an aliased table ("EMP AS E"),
- * <li>a list of tables ("EMP, DEPT"),
- * <li>an ANSI Join expression ("EMP JOIN DEPT ON EMP.DEPTNO =
- * DEPT.DEPTNO"),
- * <li>a VALUES clause ("VALUES ('Fred', 20)"),
- * <li>a query ("(SELECT * FROM EMP WHERE GENDER = 'F')"),
- * <li>or any combination of the above.
- * </ul>
- */
- protected void convertFrom(
- Blackboard bb,
- SqlNode from) {
- SqlCall call;
- final SqlNode[] operands;
- switch (from.getKind()) {
- case AS:
- operands = ((SqlBasicCall) from).getOperands();
- convertFrom(bb, operands[0]);
- return;
-
- case WITH_ITEM:
- convertFrom(bb, ((SqlWithItem) from).query);
- return;
-
- case WITH:
- convertFrom(bb, ((SqlWith) from).body);
- return;
-
- case TABLESAMPLE:
- operands = ((SqlBasicCall) from).getOperands();
- SqlSampleSpec sampleSpec = SqlLiteral.sampleValue(operands[1]);
- if (sampleSpec instanceof SqlSampleSpec.SqlSubstitutionSampleSpec) {
- String sampleName =
- ((SqlSampleSpec.SqlSubstitutionSampleSpec) sampleSpec)
- .getName();
- datasetStack.push(sampleName);
- convertFrom(bb, operands[0]);
- datasetStack.pop();
- } else if (sampleSpec instanceof SqlSampleSpec.SqlTableSampleSpec) {
- SqlSampleSpec.SqlTableSampleSpec tableSampleSpec =
- (SqlSampleSpec.SqlTableSampleSpec) sampleSpec;
- convertFrom(bb, operands[0]);
- RelOptSamplingParameters params =
- new RelOptSamplingParameters(
- tableSampleSpec.isBernoulli(),
- tableSampleSpec.getSamplePercentage(),
- tableSampleSpec.isRepeatable(),
- tableSampleSpec.getRepeatableSeed());
- bb.setRoot(new SamplingRel(cluster, bb.root, params), false);
- } else {
- throw Util.newInternal(
- "unknown TABLESAMPLE type: " + sampleSpec);
- }
- return;
-
- case IDENTIFIER:
- final SqlValidatorNamespace fromNamespace =
- validator.getNamespace(from).resolve();
- if (fromNamespace.getNode() != null) {
- convertFrom(bb, fromNamespace.getNode());
- return;
- }
- final String datasetName =
- datasetStack.isEmpty() ? null : datasetStack.peek();
- boolean[] usedDataset = {false};
- RelOptTable table =
- SqlValidatorUtil.getRelOptTable(
- fromNamespace,
- catalogReader,
- datasetName,
- usedDataset);
- final RelNode tableRel;
- if (shouldConvertTableAccess) {
- tableRel = toRel(table);
- } else {
- tableRel = new TableAccessRel(cluster, table);
- }
- bb.setRoot(tableRel, true);
- if (usedDataset[0]) {
- bb.setDataset(datasetName);
- }
- return;
-
- case JOIN:
- final SqlJoin join = (SqlJoin) from;
- final Blackboard fromBlackboard =
- createBlackboard(validator.getJoinScope(from), null);
- SqlNode left = join.getLeft();
- SqlNode right = join.getRight();
- final boolean isNatural = join.isNatural();
- final JoinType joinType = join.getJoinType();
- final Blackboard leftBlackboard =
- createBlackboard(
- Util.first(validator.getJoinScope(left),
- ((DelegatingScope) bb.scope).getParent()), null);
- final Blackboard rightBlackboard =
- createBlackboard(
- Util.first(validator.getJoinScope(right),
- ((DelegatingScope) bb.scope).getParent()), null);
- convertFrom(leftBlackboard, left);
- RelNode leftRel = leftBlackboard.root;
- convertFrom(rightBlackboard, right);
- RelNode rightRel = rightBlackboard.root;
- JoinRelType convertedJoinType = convertJoinType(joinType);
- RexNode conditionExp;
- if (isNatural) {
- final List<String> columnList =
- SqlValidatorUtil.deriveNaturalJoinColumnList(
- validator.getNamespace(left).getRowType(),
- validator.getNamespace(right).getRowType());
- conditionExp = convertUsing(leftRel, rightRel, columnList);
- } else {
- conditionExp =
- convertJoinCondition(
- fromBlackboard,
- join.getCondition(),
- join.getConditionType(),
- leftRel,
- rightRel);
- }
-
- final RelNode joinRel =
- createJoin(
- fromBlackboard,
- leftRel,
- rightRel,
- conditionExp,
- convertedJoinType);
- bb.setRoot(joinRel, false);
- return;
-
- case SELECT:
- case INTERSECT:
- case EXCEPT:
- case UNION:
- final RelNode rel = convertQueryRecursive(from, false, null);
- bb.setRoot(rel, true);
- return;
-
- case VALUES:
- convertValuesImpl(bb, (SqlCall) from, null);
- return;
-
- case UNNEST:
- final SqlNode node = ((SqlCall) from).operand(0);
- replaceSubqueries(bb, node, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
- final RelNode childRel =
- RelOptUtil.createProject(
- (null != bb.root) ? bb.root : new OneRowRel(cluster),
- Collections.singletonList(bb.convertExpression(node)),
- Collections.singletonList(validator.deriveAlias(node, 0)),
- true);
-
- UncollectRel uncollectRel =
- new UncollectRel(cluster, cluster.traitSetOf(Convention.NONE),
- childRel);
- bb.setRoot(uncollectRel, true);
- return;
-
- case COLLECTION_TABLE:
- call = (SqlCall) from;
-
- // Dig out real call; TABLE() wrapper is just syntactic.
- assert call.getOperandList().size() == 1;
- call = call.operand(0);
- convertCollectionTable(bb, call);
- return;
-
- default:
- throw Util.newInternal("not a join operator " + from);
- }
- }
-
- protected void convertCollectionTable(
- Blackboard bb,
- SqlCall call) {
- final SqlOperator operator = call.getOperator();
- if (operator == SqlStdOperatorTable.TABLESAMPLE) {
- final String sampleName =
- SqlLiteral.stringValue(call.operand(0));
- datasetStack.push(sampleName);
- SqlCall cursorCall = call.operand(1);
- SqlNode query = cursorCall.operand(0);
- RelNode converted = convertQuery(query, false, false);
- bb.setRoot(converted, false);
- datasetStack.pop();
- return;
- }
- replaceSubqueries(bb, call, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
-
- // Expand table macro if possible. It's more efficient than
- // TableFunctionRel.
- if (operator instanceof SqlUserDefinedTableMacro) {
- final SqlUserDefinedTableMacro udf =
- (SqlUserDefinedTableMacro) operator;
- final TranslatableTable table = udf.getTable(typeFactory,
- call.getOperandList());
- final RelDataType rowType = table.getRowType(typeFactory);
- RelOptTable relOptTable = RelOptTableImpl.create(null, rowType, table);
- RelNode converted = toRel(relOptTable);
- bb.setRoot(converted, true);
- return;
- }
-
- Type elementType;
- if (operator instanceof SqlUserDefinedTableFunction) {
- SqlUserDefinedTableFunction udtf = (SqlUserDefinedTableFunction) operator;
- elementType = udtf.getElementType(typeFactory, call.getOperandList());
- } else {
- elementType = null;
- }
-
- RexNode rexCall = bb.convertExpression(call);
- final List<RelNode> inputs = bb.retrieveCursors();
- Set<RelColumnMapping> columnMappings =
- getColumnMappings(operator);
- TableFunctionRel callRel =
- new TableFunctionRel(
- cluster,
- inputs,
- rexCall,
- elementType,
- validator.getValidatedNodeType(call),
- columnMappings);
- bb.setRoot(callRel, true);
- afterTableFunction(bb, call, callRel);
- }
-
- protected void afterTableFunction(
- SqlToRelConverter.Blackboard bb,
- SqlCall call,
- TableFunctionRel callRel) {
- }
-
- private Set<RelColumnMapping> getColumnMappings(SqlOperator op) {
- SqlReturnTypeInference rti = op.getReturnTypeInference();
- if (rti == null) {
- return null;
- }
- if (rti instanceof TableFunctionReturnTypeInference) {
- TableFunctionReturnTypeInference tfrti =
- (TableFunctionReturnTypeInference) rti;
- return tfrti.getColumnMappings();
- } else {
- return null;
- }
- }
-
- protected RelNode createJoin(
- Blackboard bb,
- RelNode leftRel,
- RelNode rightRel,
- RexNode joinCond,
- JoinRelType joinType) {
- assert joinCond != null;
-
- Set<String> correlatedVariables = RelOptUtil.getVariablesUsed(rightRel);
- if (correlatedVariables.size() > 0) {
- final List<Correlation> correlations = Lists.newArrayList();
-
- for (String correlName : correlatedVariables) {
- DeferredLookup lookup = mapCorrelToDeferred.get(correlName);
- RexFieldAccess fieldAccess = lookup.getFieldAccess(correlName);
- String originalRelName = lookup.getOriginalRelName();
- String originalFieldName = fieldAccess.getField().getName();
-
- int[] nsIndexes = {-1};
- final SqlValidatorScope[] ancestorScopes = {null};
- SqlValidatorNamespace foundNs =
- lookup.bb.scope.resolve(
- originalRelName,
- ancestorScopes,
- nsIndexes);
-
- assert foundNs != null;
- assert nsIndexes.length == 1;
-
- int childNamespaceIndex = nsIndexes[0];
-
- SqlValidatorScope ancestorScope = ancestorScopes[0];
- boolean correlInCurrentScope = ancestorScope == bb.scope;
-
- if (correlInCurrentScope) {
- int namespaceOffset = 0;
- if (childNamespaceIndex > 0) {
- // If not the first child, need to figure out the width
- // of output types from all the preceding namespaces
- assert ancestorScope instanceof ListScope;
- List<SqlValidatorNamespace> children =
- ((ListScope) ancestorScope).getChildren();
-
- for (int i = 0; i < childNamespaceIndex; i++) {
- SqlValidatorNamespace child = children.get(i);
- namespaceOffset +=
- child.getRowType().getFieldCount();
- }
- }
-
- RelDataTypeField field =
- catalogReader.field(foundNs.getRowType(), originalFieldName);
- int pos = namespaceOffset + field.getIndex();
-
- assert field.getType()
- == lookup.getFieldAccess(correlName).getField().getType();
-
- assert pos != -1;
-
- if (bb.mapRootRelToFieldProjection.containsKey(bb.root)) {
- // bb.root is an aggregate and only projects group by
- // keys.
- Map<Integer, Integer> exprProjection =
- bb.mapRootRelToFieldProjection.get(bb.root);
-
- // subquery can reference group by keys projected from
- // the root of the outer relation.
- if (exprProjection.containsKey(pos)) {
- pos = exprProjection.get(pos);
- } else {
- // correl not grouped
- throw Util.newInternal(
- "Identifier '" + originalRelName + "."
- + originalFieldName + "' is not a group expr");
- }
- }
-
- Correlation newCorVar =
- new Correlation(
- getCorrelOrdinal(correlName),
- pos);
-
- correlations.add(newCorVar);
- }
- }
-
- if (!correlations.isEmpty()) {
- return new CorrelatorRel(
- rightRel.getCluster(),
- leftRel,
- rightRel,
- joinCond,
- correlations,
- joinType);
- }
- }
-
- final List<RexNode> extraLeftExprs = new ArrayList<RexNode>();
- final List<RexNode> extraRightExprs = new ArrayList<RexNode>();
- final int leftCount = leftRel.getRowType().getFieldCount();
- final int rightCount = rightRel.getRowType().getFieldCount();
- if (!containsGet(joinCond)) {
- joinCond = pushDownJoinConditions(
- joinCond, leftCount, rightCount, extraLeftExprs, extraRightExprs);
- }
- if (!extraLeftExprs.isEmpty()) {
- final List<RelDataTypeField> fields =
- leftRel.getRowType().getFieldList();
- leftRel = RelOptUtil.createProject(
- leftRel,
- new AbstractList<Pair<RexNode, String>>() {
- @Override
- public int size() {
- return leftCount + extraLeftExprs.size();
- }
-
- @Override
- public Pair<RexNode, String> get(int index) {
- if (index < leftCount) {
- RelDataTypeField field = fields.get(index);
- return Pair.<RexNode, String>of(
- new RexInputRef(index, field.getType()),
- field.getName());
- } else {
- return Pair.<RexNode, String>of(
- extraLeftExprs.get(index - leftCount), null);
- }
- }
- },
- true);
- }
- if (!extraRightExprs.isEmpty()) {
- final List<RelDataTypeField> fields =
- rightRel.getRowType().getFieldList();
- final int newLeftCount = leftCount + extraLeftExprs.size();
- rightRel = RelOptUtil.createProject(
- rightRel,
- new AbstractList<Pair<RexNode, String>>() {
- @Override
- public int size() {
- return rightCount + extraRightExprs.size();
- }
-
- @Override
- public Pair<RexNode, String> get(int index) {
- if (index < rightCount) {
- RelDataTypeField field = fields.get(index);
- return Pair.<RexNode, String>of(
- new RexInputRef(index, field.getType()),
- field.getName());
- } else {
- return Pair.of(
- RexUtil.shift(
- extraRightExprs.get(index - rightCount),
- -newLeftCount),
- null);
- }
- }
- },
- true);
- }
- RelNode join = createJoin(
- leftRel,
- rightRel,
- joinCond,
- joinType,
- ImmutableSet.<String>of());
- if (!extraLeftExprs.isEmpty() || !extraRightExprs.isEmpty()) {
- Mappings.TargetMapping mapping =
- Mappings.createShiftMapping(
- leftCount + extraLeftExprs.size()
- + rightCount + extraRightExprs.size(),
- 0, 0, leftCount,
- leftCount, leftCount + extraLeftExprs.size(), rightCount);
- return RelOptUtil.project(join, mapping);
- }
- return join;
- }
-
- private static boolean containsGet(RexNode node) {
- try {
- node.accept(
- new RexVisitorImpl<Void>(true) {
- @Override public Void visitCall(RexCall call) {
- if (call.getOperator() == RexBuilder.GET_OPERATOR) {
- throw Util.FoundOne.NULL;
- }
- return super.visitCall(call);
- }
- });
- return false;
- } catch (Util.FoundOne e) {
- return true;
- }
- }
-
- /**
- * Pushes down parts of a join condition. For example, given
- * "emp JOIN dept ON emp.deptno + 1 = dept.deptno", adds a project above
- * "emp" that computes the expression
- * "emp.deptno + 1". The resulting join condition is a simple combination
- * of AND, equals, and input fields.
- */
- private RexNode pushDownJoinConditions(
- RexNode node,
- int leftCount,
- int rightCount,
- List<RexNode> extraLeftExprs,
- List<RexNode> extraRightExprs) {
- switch (node.getKind()) {
- case AND:
- case OR:
- case EQUALS:
- RexCall call = (RexCall) node;
- List<RexNode> list = new ArrayList<RexNode>();
- List<RexNode> operands = Lists.newArrayList(call.getOperands());
- for (int i = 0; i < operands.size(); i++) {
- RexNode operand = operands.get(i);
- final int left2 = leftCount + extraLeftExprs.size();
- final int right2 = rightCount + extraRightExprs.size();
- final RexNode e =
- pushDownJoinConditions(
- operand,
- leftCount,
- rightCount,
- extraLeftExprs,
- extraRightExprs);
- final List<RexNode> remainingOperands = Util.skip(operands, i + 1);
- final int left3 = leftCount + extraLeftExprs.size();
- final int right3 = rightCount + extraRightExprs.size();
- fix(remainingOperands, left2, left3);
- fix(list, left2, left3);
- list.add(e);
- }
- if (!list.equals(call.getOperands())) {
- return call.clone(call.getType(), list);
- }
- return call;
- case INPUT_REF:
- case LITERAL:
- return node;
- default:
- BitSet bits = RelOptUtil.InputFinder.bits(node);
- final int mid = leftCount + extraLeftExprs.size();
- switch (Side.of(bits, mid)) {
- case LEFT:
- fix(extraRightExprs, mid, mid + 1);
- extraLeftExprs.add(node);
- return new RexInputRef(mid, node.getType());
- case RIGHT:
- final int index2 = mid + rightCount + extraRightExprs.size();
- extraRightExprs.add(node);
- return new RexInputRef(index2, node.getType());
- case BOTH:
- case EMPTY:
- default:
- return node;
- }
- }
- }
-
- private void fix(List<RexNode> operands, int before, int after) {
- if (before == after) {
- return;
- }
- for (int i = 0; i < operands.size(); i++) {
- RexNode node = operands.get(i);
- operands.set(i, RexUtil.shift(node, before, after - before));
- }
- }
-
- /**
- * Categorizes whether a bit set contains bits left and right of a
- * line.
- */
- enum Side {
- LEFT, RIGHT, BOTH, EMPTY;
-
- static Side of(BitSet bitSet, int middle) {
- final int firstBit = bitSet.nextSetBit(0);
- if (firstBit < 0) {
- return EMPTY;
- }
- if (firstBit >= middle) {
- return RIGHT;
- }
- if (bitSet.nextSetBit(middle) < 0) {
- return LEFT;
- }
- return BOTH;
- }
- }
-
- /**
- * Determines whether a subquery is non-correlated. Note that a
- * non-correlated subquery can contain correlated references, provided those
- * references do not reference select statements that are parents of the
- * subquery.
- *
- * @param subq the subquery
- * @param bb blackboard used while converting the subquery, i.e., the
- * blackboard of the parent query of this subquery
- * @return true if the subquery is non-correlated.
- */
- private boolean isSubqNonCorrelated(RelNode subq, Blackboard bb) {
- Set<String> correlatedVariables = RelOptUtil.getVariablesUsed(subq);
- for (String correlName : correlatedVariables) {
- DeferredLookup lookup = mapCorrelToDeferred.get(correlName);
- String originalRelName = lookup.getOriginalRelName();
-
- int[] nsIndexes = {-1};
- final SqlValidatorScope[] ancestorScopes = {null};
- SqlValidatorNamespace foundNs =
- lookup.bb.scope.resolve(
- originalRelName,
- ancestorScopes,
- nsIndexes);
-
- assert foundNs != null;
- assert nsIndexes.length == 1;
-
- SqlValidatorScope ancestorScope = ancestorScopes[0];
-
- // If the correlated reference is in a scope that's "above" the
- // subquery, then this is a correlated subquery.
- SqlValidatorScope parentScope = bb.scope;
- do {
- if (ancestorScope == parentScope) {
- return false;
- }
- if (parentScope instanceof DelegatingScope) {
- parentScope = ((DelegatingScope) parentScope).getParent();
- } else {
- break;
- }
- } while (parentScope != null);
- }
- return true;
- }
-
- /**
- * Returns a list of fields to be prefixed to each relational expression.
- *
- * @return List of system fields
- */
- protected List<RelDataTypeField> getSystemFields() {
- return Collections.emptyList();
- }
-
- private RexNode convertJoinCondition(
- Blackboard bb,
- SqlNode condition,
- JoinConditionType conditionType,
- RelNode leftRel,
- RelNode rightRel) {
- if (condition == null) {
- return rexBuilder.makeLiteral(true);
- }
- bb.setRoot(ImmutableList.of(leftRel, rightRel));
- replaceSubqueries(bb, condition, RelOptUtil.Logic.UNKNOWN_AS_FALSE);
- switch (conditionType) {
- case ON:
- bb.setRoot(ImmutableList.of(leftRel, rightRel));
- return bb.convertExpression(condition);
- case USING:
- SqlNodeList list = (SqlNodeList) condition;
- List<String> nameList = new ArrayList<String>();
- for (SqlNode columnName : list) {
- final SqlIdentifier id = (SqlIdentifier) columnName;
- String name = id.getSimple();
- nameList.add(name);
- }
- return convertUsing(leftRel, rightRel, nameList);
- default:
- throw Util.unexpected(conditionType);
- }
- }
-
- /**
- * Returns an expression for matching columns of a USING clause or inferred
- * from NATURAL JOIN. "a JOIN b USING (x, y)" becomes "a.x = b.x AND a.y =
- * b.y". Returns null if the column list is empty.
- *
- * @param leftRel Left input to the join
- * @param rightRel Right input to the join
- * @param nameList List of column names to join on
- * @return Expression to match columns from name list, or true if name list
- * is empty
- */
- private RexNode convertUsing(
- RelNode leftRel,
- RelNode rightRel,
- List<String> nameList) {
- final List<RexNode> list = Lists.newArrayList();
- for (String name : nameList) {
- final RelDataType leftRowType = leftRel.getRowType();
- RelDataTypeField leftField = catalogReader.field(leftRowType, name);
- RexNode left =
- rexBuilder.makeInputRef(
- leftField.getType(),
- leftField.getIndex());
- final RelDataType rightRowType = rightRel.getRowType();
- RelDataTypeField rightField =
- catalogReader.field(rightRowType, name);
- RexNode right =
- rexBuilder.makeInputRef(
- rightField.getType(),
- leftRowType.getFieldList().size() + rightField.getIndex());
- RexNode equalsCall =
- rexBuilder.makeCall(
- SqlStdOperatorTable.EQUALS,
- left,
- right);
- list.add(equalsCall);
- }
- return RexUtil.composeConjunction(rexBuilder, list, false);
- }
-
- private static JoinRelType convertJoinType(JoinType joinType) {
- switch (joinType) {
- case COMMA:
- case INNER:
- case CROSS:
- return JoinRelType.INNER;
- case FULL:
- return JoinRelType.FULL;
- case LEFT:
- return JoinRelType.LEFT;
- case RIGHT:
- return JoinRelType.RIGHT;
- default:
- throw Util.unexpected(joinType);
- }
- }
-
- /**
- * Converts the SELECT, GROUP BY and HAVING clauses of an aggregate query.
- *
- * <p>This method extracts SELECT, GROUP BY and HAVING clauses, and creates
- * an {@link AggConverter}, then delegates to {@link #createAggImpl}.
- * Derived class may override this method to change any of those clauses or
- * specify a different {@link AggConverter}.
- *
- * @param bb Scope within which to resolve identifiers
- * @param select Query
- * @param orderExprList Additional expressions needed to implement ORDER BY
- */
- protected void convertAgg(
- Blackboard bb,
- SqlSelect select,
- List<SqlNode> orderExprList) {
- assert bb.root != null : "precondition: child != null";
- SqlNodeList groupList = select.getGroup();
- SqlNodeList selectList = select.getSelectList();
- SqlNode having = select.getHaving();
-
- final AggConverter aggConverter = new AggConverter(bb, select);
- createAggImpl(
- bb,
- aggConverter,
- selectList,
- groupList,
- having,
- orderExprList);
- }
-
- protected final void createAggImpl(
- Blackboard bb,
- AggConverter aggConverter,
- SqlNodeList selectList,
- SqlNodeList groupList,
- SqlNode having,
- List<SqlNode> orderExprList) {
- SqlNodeList aggList = new SqlNodeList(SqlParserPos.ZERO);
-
- for (SqlNode selectNode : selectList) {
- if (validator.isAggregate(selectNode)) {
- aggList.add(selectNode);
- }
- }
-
- // first replace the subqueries inside the aggregates
- // because they will provide input rows to the aggregates.
- replaceSubqueries(bb, aggList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
-
- // If group-by clause is missing, pretend that it has zero elements.
- if (groupList == null) {
- groupList = SqlNodeList.EMPTY;
- }
-
- // register the group exprs
-
- // build a map to remember the projections from the top scope to the
- // output of the current root.
- //
- // Currently farrago allows expressions, not just column references in
- // group by list. This is not SQL 2003 compliant.
-
- Map<Integer, Integer> groupExprProjection =
- new HashMap<Integer, Integer>();
-
- int i = -1;
- for (SqlNode groupExpr : groupList) {
- ++i;
- final SqlNode expandedGroupExpr =
- validator.expand(groupExpr, bb.scope);
- aggConverter.addGroupExpr(expandedGroupExpr);
-
- if (expandedGroupExpr instanceof SqlIdentifier) {
- // SQL 2003 does not allow expressions of column references
- SqlIdentifier expr = (SqlIdentifier) expandedGroupExpr;
-
- // column references should be fully qualified.
- assert expr.names.size() == 2;
- String originalRelName = expr.names.get(0);
- String originalFieldName = expr.names.get(1);
-
- int[] nsIndexes = {-1};
- final SqlValidatorScope[] ancestorScopes = {null};
- SqlValidatorNamespace foundNs =
- bb.scope.resolve(
- originalRelName,
- ancestorScopes,
- nsIndexes);
-
- assert foundNs != null;
- assert nsIndexes.length == 1;
- int childNamespaceIndex = nsIndexes[0];
-
- int namespaceOffset = 0;
-
- if (childNamespaceIndex > 0) {
- // If not the first child, need to figure out the width of
- // output types from all the preceding namespaces
- assert ancestorScopes[0] instanceof ListScope;
- List<SqlValidatorNamespace> children =
- ((ListScope) ancestorScopes[0]).getChildren();
-
- for (int j = 0; j < childNamespaceIndex; j++) {
- namespaceOffset +=
- children.get(j).getRowType().getFieldCount();
- }
- }
-
- RelDataTypeField field =
- catalogReader.field(foundNs.getRowType(), originalFieldName);
- int origPos = namespaceOffset + field.getIndex();
-
- groupExprProjection.put(origPos, i);
- }
- }
-
- RexNode havingExpr = null;
- List<RexNode> selectExprs = new ArrayList<RexNode>();
- List<String> selectNames = new ArrayList<String>();
-
- try {
- Util.permAssert(bb.agg == null, "already in agg mode");
- bb.agg = aggConverter;
-
- // convert the select and having expressions, so that the
- // agg converter knows which aggregations are required
-
- selectList.accept(aggConverter);
- for (SqlNode expr : orderExprList) {
- expr.accept(aggConverter);
- }
- if (having != null) {
- having.accept(aggConverter);
- }
-
- // compute inputs to the aggregator
- List<RexNode> preExprs = aggConverter.getPreExprs();
- List<String> preNames = aggConverter.getPreNames();
-
- if (preExprs.size() == 0) {
- // Special case for COUNT(*), where we can end up with no inputs
- // at all. The rest of the system doesn't like 0-tuples, so we
- // select a dummy constant here.
- preExprs =
- Collections.singletonList(
- (RexNode) rexBuilder.makeExactLiteral(BigDecimal.ZERO));
- preNames = Collections.singletonList(null);
- }
-
- final RelNode inputRel = bb.root;
-
- // Project the expressions required by agg and having.
- bb.setRoot(
- RelOptUtil.createProject(
- inputRel,
- preExprs,
- preNames,
- true),
- false);
- bb.mapRootRelToFieldProjection.put(bb.root, groupExprProjection);
-
- // REVIEW jvs 31-Oct-2007: doesn't the declaration of
- // monotonicity here assume sort-based aggregation at
- // the physical level?
-
- // Tell bb which of group columns are sorted.
- bb.columnMonotonicities.clear();
- for (SqlNode groupItem : groupList) {
- bb.columnMonotonicities.add(
- bb.scope.getMonotonicity(groupItem));
- }
-
- // Add the aggregator
- bb.setRoot(
- createAggregate(
- bb,
- BitSets.range(aggConverter.groupExprs.size()),
- aggConverter.getAggCalls()),
- false);
-
- bb.mapRootRelToFieldProjection.put(bb.root, groupExprProjection);
-
- // Replace subqueries in having here and modify having to use
- // the replaced expressions
- if (having != null) {
- SqlNode newHaving = pushDownNotForIn(having);
- replaceSubqueries(bb, newHaving, RelOptUtil.Logic.UNKNOWN_AS_FALSE);
- havingExpr = bb.convertExpression(newHaving);
- if (havingExpr.isAlwaysTrue()) {
- havingExpr = null;
- }
- }
-
- // Now convert the other subqueries in the select list.
- // This needs to be done separately from the subquery inside
- // any aggregate in the select list, and after the aggregate rel
- // is allocated.
- replaceSubqueries(bb, selectList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
-
- // Now subqueries in the entire select list have been converted.
- // Convert the select expressions to get the final list to be
- // projected.
- int k = 0;
-
- // For select expressions, use the field names previously assigned
- // by the validator. If we derive afresh, we might generate names
- // like "EXPR$2" that don't match the names generated by the
- // validator. This is especially the case when there are system
- // fields; system fields appear in the relnode's rowtype but do not
- // (yet) appear in the validator type.
- final SelectScope selectScope =
- SqlValidatorUtil.getEnclosingSelectScope(bb.scope);
- final SqlValidatorNamespace selectNamespace =
- validator.getNamespace(selectScope.getNode());
- final List<String> names =
- selectNamespace.getRowType().getFieldNames();
- int sysFieldCount = selectList.size() - names.size();
- for (SqlNode expr : selectList) {
- selectExprs.add(bb.convertExpression(expr));
- selectNames.add(
- k < sysFieldCount
- ? validator.deriveAlias(expr, k++)
- : names.get(k++ - sysFieldCount));
- }
-
- for (SqlNode expr : orderExprList) {
- selectExprs.add(bb.convertExpression(expr));
- selectNames.add(validator.deriveAlias(expr, k++));
- }
- } finally {
- bb.agg = null;
- }
-
- // implement HAVING (we have already checked that it is non-trivial)
- if (havingExpr != null) {
- bb.setRoot(RelOptUtil.createFilter(bb.root, havingExpr), false);
- }
-
- // implement the SELECT list
- bb.setRoot(
- RelOptUtil.createProject(
- bb.root,
- selectExprs,
- selectNames,
- true),
- false);
-
- // Tell bb which of group columns are sorted.
- bb.columnMonotonicities.clear();
- for (SqlNode selectItem : selectList) {
- bb.columnMonotonicities.add(
- bb.scope.getMonotonicity(selectItem));
- }
- }
-
- /**
- * Creates an AggregateRel.
- *
- * <p>In case the aggregate rel changes the order in which it projects
- * fields, the <code>groupExprProjection</code> parameter is provided, and
- * the implementation of this method may modify it.
- *
- * <p>The <code>sortedCount</code> parameter is the number of expressions
- * known to be monotonic. These expressions must be on the leading edge of
- * the grouping keys. The default implementation of this method ignores this
- * parameter.
- *
- * @param bb Blackboard
- * @param groupSet Bit set of ordinals of grouping columns
- * @param aggCalls Array of calls to aggregate functions
- * @return AggregateRel
- */
- protected RelNode createAggregate(
- Blackboard bb,
- BitSet groupSet,
- List<AggregateCall> aggCalls) {
- return new AggregateRel(
- cluster,
- bb.root,
- groupSet,
- aggCalls);
- }
-
- public RexDynamicParam convertDynamicParam(
- final SqlDynamicParam dynamicParam) {
- // REVIEW jvs 8-Jan-2005: dynamic params may be encountered out of
- // order. Should probably cross-check with the count from the parser
- // at the end and make sure they all got filled in. Why doesn't List
- // have a resize() method?!? Make this a utility.
- while (dynamicParam.getIndex() >= dynamicParamSqlNodes.size()) {
- dynamicParamSqlNodes.add(null);
- }
-
- dynamicParamSqlNodes.set(
- dynamicParam.getIndex(),
- dynamicParam);
- return rexBuilder.makeDynamicParam(
- getDynamicParamType(dynamicParam.getIndex()),
- dynamicParam.getIndex());
- }
-
- /**
- * Creates a list of collations required to implement the ORDER BY clause,
- * if there is one. Populates <code>extraOrderExprs</code> with any sort
- * expressions which are not in the select clause.
- *
- * @param bb Scope within which to resolve identifiers
- * @param select Select clause. Never null, because we invent a
- * dummy SELECT if ORDER BY is applied to a set
- * operation (UNION etc.)
- * @param orderList Order by clause, may be null
- * @param extraOrderExprs Sort expressions which are not in the select
- * clause (output)
- * @param collationList List of collations (output)
- */
- protected void gatherOrderExprs(
- Blackboard bb,
- SqlSelect select,
- SqlNodeList orderList,
- List<SqlNode> extraOrderExprs,
- List<RelFieldCollation> collationList) {
- // TODO: add validation rules to SqlValidator also
- assert bb.root != null : "precondition: child != null";
- assert select != null;
- if (orderList == null) {
- return;
- }
- for (SqlNode orderItem : orderList) {
- collationList.add(
- convertOrderItem(
- select,
- orderItem,
- extraOrderExprs,
- RelFieldCollation.Direction.ASCENDING,
- RelFieldCollation.NullDirection.UNSPECIFIED));
- }
- }
-
- protected RelFieldCollation convertOrderItem(
- SqlSelect select,
- SqlNode orderItem, List<SqlNode> extraExprs,
- RelFieldCollation.Direction direction,
- RelFieldCollation.NullDirection nullDirection) {
- assert select != null;
- // Handle DESC keyword, e.g. 'select a, b from t order by a desc'.
- switch (orderItem.getKind()) {
- case DESCENDING:
- return convertOrderItem(
- select,
- ((SqlCall) orderItem).operand(0),
- extraExprs,
- RelFieldCollation.Direction.DESCENDING,
- nullDirection);
- case NULLS_FIRST:
- return convertOrderItem(
- select,
- ((SqlCall) orderItem).operand(0),
- extraExprs,
- direction,
- RelFieldCollation.NullDirection.FIRST);
- case NULLS_LAST:
- return convertOrderItem(
- select,
- ((SqlCall) orderItem).operand(0),
- extraExprs,
- direction,
- RelFieldCollation.NullDirection.LAST);
- }
-
- SqlNode converted = validator.expandOrderExpr(select, orderItem);
-
- // Scan the select list and order exprs for an identical expression.
- final SelectScope selectScope = validator.getRawSelectScope(select);
- int ordinal = -1;
- for (SqlNode selectItem : selectScope.getExpandedSelectList()) {
- ++ordinal;
- if (converted.equalsDeep(stripAs(selectItem), false)) {
- return new RelFieldCollation(
- ordinal, direction, nullDirection);
- }
- }
-
- for (SqlNode extraExpr : extraExprs) {
- ++ordinal;
- if (converted.equalsDeep(extraExpr, false)) {
- return new RelFieldCollation(
- ordinal, direction, nullDirection);
- }
- }
-
- // TODO: handle collation sequence
- // TODO: flag expressions as non-standard
-
- extraExprs.add(converted);
- return new RelFieldCollation(ordinal + 1, direction, nullDirection);
- }
-
- protected boolean enableDecorrelation() {
- // disable subquery decorrelation when needed.
- // e.g. if outer joins are not supported.
- return decorrelationEnabled;
- }
-
- protected RelNode decorrelateQuery(RelNode rootRel) {
- return RelDecorrelator.decorrelateQuery(rootRel);
- }
-
- /**
- * Sets whether to trim unused fields as part of the conversion process.
- *
- * @param trim Whether to trim unused fields
- */
- public void setTrimUnusedFields(boolean trim) {
- this.trimUnusedFields = trim;
- }
-
- /**
- * Returns whether to trim unused fields as part of the conversion process.
- *
- * @return Whether to trim unused fields
- */
- public boolean isTrimUnusedFields() {
- // OVERRIDE POINT
- return false; // was `trimUnusedFields`
- }
-
- /**
- * Recursively converts a query to a relational expression.
- *
- * @param query Query
- * @param top Whether this query is the top-level query of the
- * statement
- * @param targetRowType Target row type, or null
- * @return Relational expression
- */
- protected RelNode convertQueryRecursive(
- SqlNode query,
- boolean top,
- RelDataType targetRowType) {
- switch (query.getKind()) {
- case SELECT:
- return convertSelect((SqlSelect) query);
- case INSERT:
- return convertInsert((SqlInsert) query);
- case DELETE:
- return convertDelete((SqlDelete) query);
- case UPDATE:
- return convertUpdate((SqlUpdate) query);
- case MERGE:
- return convertMerge((SqlMerge) query);
- case UNION:
- case INTERSECT:
- case EXCEPT:
- return convertSetOp((SqlCall) query);
- case WITH:
- return convertWith((SqlWith) query);
- case VALUES:
- return convertValues((SqlCall) query, targetRowType);
- default:
- throw Util.newInternal("not a query: " + query);
- }
- }
-
- /**
- * Converts a set operation (UNION, INTERSECT, MINUS) into relational
- * expressions.
- *
- * @param call Call to set operator
- * @return Relational expression
- */
- protected RelNode convertSetOp(SqlCall call) {
- final RelNode left = convertQueryRecursive(call.operand(0), false, null);
- final RelNode right = convertQueryRecursive(call.operand(1), false, null);
- boolean all = false;
- if (call.getOperator() instanceof SqlSetOperator) {
- all = ((SqlSetOperator) (call.getOperator())).isAll();
- }
- switch (call.getKind()) {
- case UNION:
- return new UnionRel(
- cluster,
- ImmutableList.of(left, right),
- all);
-
- case INTERSECT:
- // TODO: all
- if (!all) {
- return new IntersectRel(
- cluster,
- ImmutableList.of(left, right),
- all);
- } else {
- throw Util.newInternal(
- "set operator INTERSECT ALL not suported");
- }
-
- case EXCEPT:
- // TODO: all
- if (!all) {
- return new MinusRel(
- cluster,
- ImmutableList.of(left, right),
- all);
- } else {
- throw Util.newInternal(
- "set operator EXCEPT ALL not suported");
- }
-
- default:
- throw Util.unexpected(call.getKind());
- }
- }
-
- protected RelNode convertInsert(SqlInsert call) {
- RelOptTable targetTable = getTargetTable(call);
-
- final RelDataType targetRowType =
- validator.getValidatedNodeType(call);
- assert targetRowType != null;
- RelNode sourceRel =
- convertQueryRecursive(
- call.getSource(),
- false,
- targetRowType);
- RelNode massagedRel = convertColumnList(call, sourceRel);
-
- final ModifiableTable modifiableTable =
- targetTable.unwrap(ModifiableTable.class);
- if (modifiableTable != null) {
- return modifiableTable.toModificationRel(
- cluster,
- targetTable,
- catalogReader,
- massagedRel,
- TableModificationRel.Operation.INSERT,
- null,
- false);
- }
- return new TableModificationRel(
- cluster,
- targetTable,
- catalogReader,
- massagedRel,
- TableModificationRel.Operation.INSERT,
- null,
- false);
- }
-
- private RelOptTable.ToRelContext createToRelContext() {
- return new RelOptTable.ToRelContext() {
- public RelOptCluster getCluster() {
- return cluster;
- }
-
- public RelNode expandView(
- RelDataType rowType,
- String queryString,
- List<String> schemaPath) {
- return viewExpander.expandView(rowType, queryString, schemaPath);
- }
- };
- }
-
- public RelNode toRel(RelOptTable table) {
- return table.toRel(createToRelContext());
- }
-
- protected RelOptTable getTargetTable(SqlNode call) {
- SqlValidatorNamespace targetNs = validator.getNamespace(call).resolve();
- return SqlValidatorUtil.getRelOptTable(targetNs, catalogReader, null, null);
- }
-
- /**
- * Creates a source for an INSERT statement.
- *
- * <p>If the column list is not specified, source expressions match target
- * columns in order.
- *
- * <p>If the column list is specified, Source expressions are mapped to
- * target columns by name via targetColumnList, and may not cover the entire
- * target table. So, we'll make up a full row, using a combination of
- * default values and the source expressions provided.
- *
- * @param call Insert expression
- * @param sourceRel Source relational expression
- * @return Converted INSERT statement
- */
- protected RelNode convertColumnList(
- SqlInsert call,
- RelNode sourceRel) {
- RelDataType sourceRowType = sourceRel.getRowType();
- final RexNode sourceRef =
- rexBuilder.makeRangeReference(sourceRowType, 0, false);
- final List<String> targetColumnNames = new ArrayList<String>();
- final List<RexNode> columnExprs = new ArrayList<RexNode>();
- collectInsertTargets(call, sourceRef, targetColumnNames, columnExprs);
-
- final RelOptTable targetTable = getTargetTable(call);
- final RelDataType targetRowType = targetTable.getRowType();
- final List<RelDataTypeField> targetFields =
- targetRowType.getFieldList();
- final List<RexNode> sourceExps =
- new ArrayList<RexNode>(
- Collections.<RexNode>nCopies(targetFields.size(), null));
- final List<String> fieldNames =
- new ArrayList<String>(
- Collections.<String>nCopies(targetFields.size(), null));
-
- // Walk the name list and place the associated value in the
- // expression list according to the ordinal value returned from
- // the table construct, leaving nulls in the list for columns
- // that are not referenced.
- for (Pair<String, RexNode> p : Pair.zip(targetColumnNames, columnExprs)) {
- RelDataTypeField field = catalogReader.field(targetRowType, p.left);
- assert field != null : "column " + p.left + " not found";
- sourceExps.set(field.getIndex(), p.right);
- }
-
- // Walk the expression list and get default values for any columns
- // that were not supplied in the statement. Get field names too.
- for (int i = 0; i < targetFields.size(); ++i) {
- final RelDataTypeField field = targetFields.get(i);
- final String fieldName = field.getName();
- fieldNames.set(i, fieldName);
- if (sourceExps.get(i) != null) {
- if (defaultValueFactory.isGeneratedAlways(targetTable, i)) {
- throw RESOURCE.insertIntoAlwaysGenerated(fieldName).ex();
- }
- continue;
- }
- sourceExps.set(
- i, defaultValueFactory.newColumnDefaultValue(targetTable, i));
-
- // bare nulls are dangerous in the wrong hands
- sourceExps.set(
- i,
- castNullLiteralIfNeeded(
- sourceExps.get(i), field.getType()));
- }
-
- return RelOptUtil.createProject(sourceRel, sourceExps, fieldNames, true);
- }
-
- private RexNode castNullLiteralIfNeeded(RexNode node, RelDataType type) {
- if (!RexLiteral.isNullLiteral(node)) {
- return node;
- }
- return rexBuilder.makeCast(type, node);
- }
-
- /**
- * Given an INSERT statement, collects the list of names to be populated and
- * the expressions to put in them.
- *
- * @param call Insert statement
- * @param sourceRef Expression representing a row from the source
- * relational expression
- * @param targetColumnNames List of target column names, to be populated
- * @param columnExprs List of expressions, to be populated
- */
- protected void collectInsertTargets(
- SqlInsert call,
- final RexNode sourceRef,
- final List<String> targetColumnNames,
- List<RexNode> columnExprs) {
- final RelOptTable targetTable = getTargetTable(call);
- final RelDataType targetRowType = targetTable.getRowType();
- SqlNodeList targetColumnList = call.getTargetColumnList();
- if (targetColumnList == null) {
- targetColumnNames.addAll(targetRowType.getFieldNames());
- } else {
- for (int i = 0; i < targetColumnList.size(); i++) {
- SqlIdentifier id = (SqlIdentifier) targetColumnList.get(i);
- targetColumnNames.add(id.getSimple());
- }
- }
-
- for (int i = 0; i < targetColumnNames.size(); i++) {
- final RexNode expr = rexBuilder.makeFieldAccess(sourceRef, i);
- columnExprs.add(expr);
- }
- }
-
- private RelNode convertDelete(SqlDelete call) {
- RelOptTable targetTable = getTargetTable(call);
- RelNode sourceRel = convertSelect(call.getSourceSelect());
- return new TableModificationRel(
- cluster,
- targetTable,
- catalogReader,
- sourceRel,
- TableModificationRel.Operation.DELETE,
- null,
- false);
- }
-
- private RelNode convertUpdate(SqlUpdate call) {
- RelOptTable targetTable = getTargetTable(call);
-
- // convert update column list from SqlIdentifier to String
- List<String> targetColumnNameList = new ArrayList<String>();
- for (SqlNode node : call.getTargetColumnList()) {
- SqlIdentifier id = (SqlIdentifier) node;
- String name = id.getSimple();
- targetColumnNameList.add(name);
- }
-
- RelNode sourceRel = convertSelect(call.getSourceSelect());
-
- return new TableModificationRel(
- cluster,
- targetTable,
- catalogReader,
- sourceRel,
- TableModificationRel.Operation.UPDATE,
- targetColumnNameList,
- false);
- }
-
- private RelNode convertMerge(SqlMerge call) {
- RelOptTable targetTable = getTargetTable(call);
-
- // convert update column list from SqlIdentifier to String
- List<String> targetColumnNameList = new ArrayList<String>();
- SqlUpdate updateCall = call.getUpdateCall();
- if (updateCall != null) {
- for (SqlNode targetColumn : updateCall.getTargetColumnList()) {
- SqlIdentifier id = (SqlIdentifier) targetColumn;
- String name = id.getSimple();
- targetColumnNameList.add(name);
- }
- }
-
- // replace the projection of the source select with a
- // projection that contains the following:
- // 1) the expressions corresponding to the new insert row (if there is
- // an insert)
- // 2) all columns from the target table (if there is an update)
- // 3) the set expressions in the update call (if there is an update)
-
- // first, convert the merge's source select to construct the columns
- // from the target table and the set expressions in the update call
- RelNode mergeSourceRel = convertSelect(call.getSourceSelect());
-
- // then, convert the insert statement so we can get the insert
- // values expressions
- SqlInsert insertCall = call.getInsertCall();
- int nLevel1Exprs = 0;
- List<RexNode> level1InsertExprs = null;
- List<RexNode> level2InsertExprs = null;
- if (insertCall != null) {
- RelNode insertRel = convertInsert(insertCall);
-
- // if there are 2 level of projections in the insert source, combine
- // them into a single project; level1 refers to the topmost project;
- // the level1 projection contains references to the level2
- // expressions, except in the case where no target expression was
- // provided, in which case, the expression is the default value for
- // the column; or if the expressions directly map to the source
- // table
- level1InsertExprs =
- ((ProjectRel) insertRel.getInput(0)).getProjects();
- if (insertRel.getInput(0).getInput(0) instanceof ProjectRel) {
- level2InsertExprs =
- ((ProjectRel) insertRel.getInput(0).getInput(0))
- .getProjects();
- }
- nLevel1Exprs = level1InsertExprs.size();
- }
-
- JoinRel joinRel = (JoinRel) mergeSourceRel.getInput(0);
- int nSourceFields = joinRel.getLeft().getRowType().getFieldCount();
- List<RexNode> projects = new ArrayList<RexNode>();
- for (int level1Idx = 0; level1Idx < nLevel1Exprs; level1Idx++) {
- if ((level2InsertExprs != null)
- && (level1InsertExprs.get(level1Idx) instanceof RexInputRef)) {
- int level2Idx =
- ((RexInputRef) level1InsertExprs.get(level1Idx)).getIndex();
- projects.add(level2InsertExprs.get(level2Idx));
- } else {
- projects.add(level1InsertExprs.get(level1Idx));
- }
- }
- if (updateCall != null) {
- final ProjectRel project = (ProjectRel) mergeSourceRel;
- projects.addAll(
- Util.skip(project.getProjects(), nSourceFields));
- }
-
- RelNode massagedRel =
- RelOptUtil.createProject(joinRel, projects, null, true);
-
- return new TableModificationRel(
- cluster,
- targetTable,
- catalogReader,
- massagedRel,
- TableModificationRel.Operation.MERGE,
- targetColumnNameList,
- false);
- }
-
- /**
- * Converts an identifier into an expression in a given scope. For example,
- * the "empno" in "select empno from emp join dept" becomes "emp.empno".
- */
- private RexNode convertIdentifier(
- Blackboard bb,
- SqlIdentifier identifier) {
- // first check for reserved identifiers like CURRENT_USER
- final SqlCall call = SqlUtil.makeCall(opTab, identifier);
- if (call != null) {
- return bb.convertExpression(call);
- }
-
- if (bb.agg != null) {
- throw Util.newInternal("Identifier '" + identifier
- + "' is not a group expr");
- }
-
- SqlValidatorNamespace namespace = null;
- if (bb.scope != null) {
- identifier = bb.scope.fullyQualify(identifier);
- namespace = bb.scope.resolve(identifier.names.get(0), null, null);
- }
- RexNode e = bb.lookupExp(identifier.names.get(0));
- final String correlationName;
- if (e instanceof RexCorrelVariable) {
- correlationName = ((RexCorrelVariable) e).getName();
- } else {
- correlationName = null;
- }
-
- for (String name : Util.skip(identifier.names)) {
- if (namespace != null) {
- name = namespace.translate(name);
- namespace = null;
- }
- final boolean caseSensitive = true; // name already fully-qualified
- e = rexBuilder.makeFieldAccess(e, name, caseSensitive);
- }
- if (e instanceof RexInputRef) {
- // adjust the type to account for nulls introduced by outer joins
- e = adjustInputRef(bb, (RexInputRef) e);
- }
-
- if (null != correlationName) {
- // REVIEW: make mapCorrelateVariableToRexNode map to RexFieldAccess
- assert e instanceof RexFieldAccess;
- final RexNode prev =
- bb.mapCorrelateVariableToRexNode.put(correlationName, e);
- assert prev == null;
- }
- return e;
- }
-
- /**
- * Adjusts the type of a reference to an input field to account for nulls
- * introduced by outer joins; and adjusts the offset to match the physical
- * implementation.
- *
- * @param bb Blackboard
- * @param inputRef Input ref
- * @return Adjusted input ref
- */
- protected RexNode adjustInputRef(
- Blackboard bb,
- RexInputRef inputRef) {
- RelDataTypeField field = bb.getRootField(inputRef);
- if (field != null) {
- return rexBuilder.makeInputRef(
- field.getType(),
- inputRef.getIndex());
- }
- return inputRef;
- }
-
- /**
- * Converts a row constructor into a relational expression.
- *
- * @param bb Blackboard
- * @param rowConstructor Row constructor expression
- * @return Relational expression which returns a single row.
- * @pre isRowConstructor(rowConstructor)
- */
- private RelNode convertRowConstructor(
- Blackboard bb,
- SqlCall rowConstructor) {
- assert isRowConstructor(rowConstructor) : rowConstructor;
- final List<SqlNode> operands = rowConstructor.getOperandList();
- return convertMultisets(operands, bb);
- }
-
- private RelNode convertCursor(Blackboard bb, SubQuery subQuery) {
- final SqlCall cursorCall = (SqlCall) subQuery.node;
- assert cursorCall.operandCount() == 1;
- SqlNode query = cursorCall.operand(0);
- RelNode converted = convertQuery(query, false, false);
- int iCursor = bb.cursors.size();
- bb.cursors.add(converted);
- subQuery.expr =
- new RexInputRef(
- iCursor,
- converted.getRowType());
- return converted;
- }
-
- private RelNode convertMultisets(final List<SqlNode> operands,
- Blackboard bb) {
- // NOTE: Wael 2/04/05: this implementation is not the most efficient in
- // terms of planning since it generates XOs that can be reduced.
- List<Object> joinList = new ArrayList<Object>();
- List<SqlNode> lastList = new ArrayList<SqlNode>();
- for (int i = 0; i < operands.size(); i++) {
- SqlNode operand = operands.get(i);
- if (!(operand instanceof SqlCall)) {
- lastList.add(operand);
- continue;
- }
-
- final SqlCall call = (SqlCall) operand;
- final SqlOperator op = call.getOperator();
- if ((op != SqlStdOperatorTable.MULTISET_VALUE)
- && (op != SqlStdOperatorTable.MULTISET_QUERY)) {
- lastList.add(operand);
- continue;
- }
- final RelNode input;
- if (op == SqlStdOperatorTable.MULTISET_VALUE) {
- final SqlNodeList list =
- new SqlNodeList(call.getOperandList(), call.getParserPosition());
-// assert bb.scope instanceof SelectScope : bb.scope;
- CollectNamespace nss =
- (CollectNamespace) validator.getNamespace(call);
- Blackboard usedBb;
- if (null != nss) {
- usedBb = createBlackboard(nss.getScope(), null);
- } else {
- usedBb =
- createBlackboard(
- new ListScope(bb.scope) {
- public SqlNode getNode() {
- return call;
- }
- },
- null);
- }
- RelDataType multisetType = validator.getValidatedNodeType(call);
- validator.setValidatedNodeType(
- list,
- multisetType.getComponentType());
- input = convertQueryOrInList(usedBb, list);
- } else {
- input = convertQuery(call.operand(0), false, true);
- }
-
- if (lastList.size() > 0) {
- joinList.add(lastList);
- }
- lastList = new ArrayList<SqlNode>();
- CollectRel collectRel =
- new CollectRel(
- cluster,
- cluster.traitSetOf(Convention.NONE),
- input,
- validator.deriveAlias(call, i));
- joinList.add(collectRel);
- }
-
- if (joinList.size() == 0) {
- joinList.add(lastList);
- }
-
- for (int i = 0; i < joinList.size(); i++) {
- Object o = joinList.get(i);
- if (o instanceof List) {
- List<SqlNode> projectList = (List<SqlNode>) o;
- final List<RexNode> selectList = new ArrayList<RexNode>();
- final List<String> fieldNameList = new ArrayList<String>();
- for (int j = 0; j < projectList.size(); j++) {
- SqlNode operand = projectList.get(j);
- selectList.add(bb.convertExpression(operand));
-
- // REVIEW angel 5-June-2005: Use deriveAliasFromOrdinal
- // instead of deriveAlias to match field names from
- // SqlRowOperator. Otherwise, get error Type
- // 'RecordType(INTEGER EMPNO)' has no field 'EXPR$0' when
- // doing select * from unnest( select multiset[empno]
- // from sales.emps);
-
- fieldNameList.add(SqlUtil.deriveAliasFromOrdinal(j));
- }
-
- RelNode projRel =
- RelOptUtil.createProject(
- new OneRowRel(cluster),
- selectList,
- fieldNameList);
-
- joinList.set(i, projRel);
- }
- }
-
- RelNode ret = (RelNode) joinList.get(0);
- for (int i = 1; i < joinList.size(); i++) {
- RelNode relNode = (RelNode) joinList.get(i);
- ret =
- createJoin(
- ret,
- relNode,
- rexBuilder.makeLiteral(true),
- JoinRelType.INNER,
- ImmutableSet.<String>of());
- }
- return ret;
- }
-
- /**
- * Factory method that creates a join.
- * A subclass can override to use a different kind of join.
- *
- * @param left Left input
- * @param right Right input
- * @param condition Join condition
- * @param joinType Join type
- * @param variablesStopped Set of names of variables which are set by the
- * LHS and used by the RHS and are not available to
- * nodes above this JoinRel in the tree
- * @return A relational expression representing a join
- */
- protected RelNode createJoin(
- RelNode left,
- RelNode right,
- RexNode condition,
- JoinRelType joinType,
- Set<String> variablesStopped) {
- return new JoinRel(
- cluster,
- left,
- right,
- condition,
- joinType,
- variablesStopped);
- }
-
- private void convertSelectList(
- Blackboard bb,
- SqlSelect select,
- List<SqlNode> orderList) {
- SqlNodeList selectList = select.getSelectList();
- selectList = validator.expandStar(selectList, select, false);
-
- replaceSubqueries(bb, selectList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
-
- List<String> fieldNames = new ArrayList<String>();
- List<RexNode> exprs = new ArrayList<RexNode>();
- Collection<String> aliases = new TreeSet<String>();
-
- // Project any system fields. (Must be done before regular select items,
- // because offsets may be affected.)
- final List<SqlMonotonicity> columnMonotonicityList =
- new ArrayList<SqlMonotonicity>();
- extraSelectItems(
- bb,
- select,
- exprs,
- fieldNames,
- aliases,
- columnMonotonicityList);
-
- // Project select clause.
- int i = -1;
- for (SqlNode expr : selectList) {
- ++i;
- exprs.add(bb.convertExpression(expr));
- fieldNames.add(deriveAlias(expr, aliases, i));
- }
-
- // Project extra fields for sorting.
- for (SqlNode expr : orderList) {
- ++i;
- SqlNode expr2 = validator.expandOrderExpr(select, expr);
- exprs.add(bb.convertExpression(expr2));
- fieldNames.add(deriveAlias(expr, aliases, i));
- }
-
- fieldNames = SqlValidatorUtil.uniquify(fieldNames);
-
- RelNode inputRel = bb.root;
- bb.setRoot(
- RelOptUtil.createProject(bb.root, exprs, fieldNames),
- false);
-
- assert bb.columnMonotonicities.isEmpty();
- bb.columnMonotonicities.addAll(columnMonotonicityList);
- for (SqlNode selectItem : selectList) {
- bb.columnMonotonicities.add(
- selectItem.getMonotonicity(bb.scope));
- }
- }
-
- /**
- * Adds extra select items. The default implementation adds nothing; derived
- * classes may add columns to exprList, nameList, aliasList and
- * columnMonotonicityList.
- *
- * @param bb Blackboard
- * @param select Select statement being translated
- * @param exprList List of expressions in select clause
- * @param nameList List of names, one per column
- * @param aliasList Collection of aliases that have been used
- * already
- * @param columnMonotonicityList List of monotonicity, one per column
- */
- protected void extraSelectItems(
- Blackboard bb,
- SqlSelect select,
- List<RexNode> exprList,
- List<String> nameList,
- Collection<String> aliasList,
- List<SqlMonotonicity> columnMonotonicityList) {
- }
-
- private String deriveAlias(
- final SqlNode node,
- Collection<String> aliases,
- final int ordinal) {
- String alias = validator.deriveAlias(node, ordinal);
- if ((alias == null) || aliases.contains(alias)) {
- String aliasBase = (alias == null) ? "EXPR$" : alias;
- for (int j = 0;; j++) {
- alias = aliasBase + j;
- if (!aliases.contains(alias)) {
- break;
- }
- }
- }
- aliases.add(alias);
- return alias;
- }
-
- /**
- * Converts a WITH sub-query into a relational expression.
- */
- public RelNode convertWith(SqlWith with) {
- return convertQuery(with.body, false, false);
- }
-
- /**
- * Converts a SELECT statement's parse tree into a relational expression.
- */
- public RelNode convertValues(
- SqlCall values,
- RelDataType targetRowType) {
- final SqlValidatorScope scope = validator.getOverScope(values);
- assert scope != null;
- final Blackboard bb = createBlackboard(scope, null);
- convertValuesImpl(bb, values, targetRowType);
- return bb.root;
- }
-
- /**
- * Converts a values clause (as in "INSERT INTO T(x,y) VALUES (1,2)") into a
- * relational expression.
- *
- * @param bb Blackboard
- * @param values Call to SQL VALUES operator
- * @param targetRowType Target row type
- */
- private void convertValuesImpl(
- Blackboard bb,
- SqlCall values,
- RelDataType targetRowType) {
- // Attempt direct conversion to ValuesRel; if that fails, deal with
- // fancy stuff like subqueries below.
- RelNode valuesRel =
- convertRowValues(
- bb,
- values,
- values.getOperandList(),
- true,
- targetRowType);
- if (valuesRel != null) {
- bb.setRoot(valuesRel, true);
- return;
- }
-
- List<RelNode> unionRels = new ArrayList<RelNode>();
- for (SqlNode rowConstructor1 : values.getOperandList()) {
- SqlCall rowConstructor = (SqlCall) rowConstructor1;
- Blackboard tmpBb = createBlackboard(bb.scope, null);
- replaceSubqueries(tmpBb, rowConstructor,
- RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
- List<Pair<RexNode, String>> exps =
- new ArrayList<Pair<RexNode, String>>();
- for (Ord<SqlNode> operand : Ord.zip(rowConstructor.getOperandList())) {
- exps.add(
- Pair.of(
- tmpBb.convertExpression(operand.e),
- validator.deriveAlias(operand.e, operand.i)));
- }
- RelNode in =
- (null == tmpBb.root)
- ? new OneRowRel(cluster)
- : tmpBb.root;
- unionRels.add(
- RelOptUtil.createProject(
- in,
- Pair.left(exps),
- Pair.right(exps),
- true));
- }
-
- if (unionRels.size() == 0) {
- throw Util.newInternal("empty values clause");
- } else if (unionRels.size() == 1) {
- bb.setRoot(
- unionRels.get(0),
- true);
- } else {
- bb.setRoot(
- new UnionRel(
- cluster,
- unionRels,
- true),
- true);
- }
-
- // REVIEW jvs 22-Jan-2004: should I add
- // mapScopeToLux.put(validator.getScope(values),bb.root);
- // ?
- }
-
- private String createCorrel() {
- int n = nextCorrel++;
- return CORREL_PREFIX + n;
- }
-
- private int getCorrelOrdinal(String correlName) {
- assert correlName.startsWith(CORREL_PREFIX);
- return Integer.parseInt(correlName.substring(CORREL_PREFIX.length()));
- }
-
- //~ Inner Classes ----------------------------------------------------------
-
- /**
- * Workspace for translating an individual SELECT statement (or sub-SELECT).
- */
- protected class Blackboard implements SqlRexContext, SqlVisitor<RexNode> {
- /**
- * Collection of {@link RelNode} objects which correspond to a SELECT
- * statement.
- */
- public final SqlValidatorScope scope;
- private final Map<String, RexNode> nameToNodeMap;
- public RelNode root;
- private List<RelNode> inputs;
- private final Map<String, RexNode> mapCorrelateVariableToRexNode =
- new HashMap<String, RexNode>();
-
- List<RelNode> cursors;
-
- /**
- * List of <code>IN</code> and <code>EXISTS</code> nodes inside this
- * <code>SELECT</code> statement (but not inside sub-queries).
- */
- private final Set<SubQuery> subqueryList = Sets.newLinkedHashSet();
-
- private final Map<SqlNode, SubQuery> subqueryMap =
- Util.asIndexMap(subqueryList, FN);
-
- private boolean subqueryNeedsOuterJoin;
-
- /**
- * Workspace for building aggregates.
- */
- AggConverter agg;
-
- /**
- * When converting window aggregate, we need to know if the window is
- * guaranteed to be non-empty.
- */
- SqlWindow window;
-
- /**
- * Project the groupby expressions out of the root of this sub-select.
- * Subqueries can reference group by expressions projected from the
- * "right" to the subquery.
- */
- private final Map<RelNode, Map<Integer, Integer>>
- mapRootRelToFieldProjection =
- new HashMap<RelNode, Map<Integer, Integer>>();
-
- private final List<SqlMonotonicity> columnMonotonicities =
- new ArrayList<SqlMonotonicity>();
-
- private final List<RelDataTypeField> systemFieldList =
- new ArrayList<RelDataTypeField>();
-
- /**
- * Creates a Blackboard.
- *
- * @param scope Name-resolution scope for expressions validated
- * within this query. Can be null if this Blackboard is
- * for a leaf node, say
- * @param nameToNodeMap Map which translates the expression to map a
- * given parameter into, if translating expressions;
- * null otherwise
- */
- protected Blackboard(
- SqlValidatorScope scope,
- Map<String, RexNode> nameToNodeMap) {
- this.scope = scope;
- this.nameToNodeMap = nameToNodeMap;
- this.cursors = new ArrayList<RelNode>();
- subqueryNeedsOuterJoin = false;
- }
-
- public RexNode register(
- RelNode rel,
- JoinRelType joinType) {
- return register(rel, joinType, null);
- }
-
- /**
- * Registers a relational expression.
- *
- * @param rel Relational expression
- * @param joinType Join type
- * @param leftKeys LHS of IN clause, or null for expressions
- * other than IN
- * @return Expression with which to refer to the row (or partial row)
- * coming from this relational expression's side of the join
- */
- public RexNode register(
- RelNode rel,
- JoinRelType joinType,
- List<RexNode> leftKeys) {
- assert joinType != null;
- if (root == null) {
- assert leftKeys == null;
- setRoot(rel, false);
- return rexBuilder.makeRangeReference(
- root.getRowType(),
- 0,
- false);
- }
-
- final RexNode joinCond;
- final int origLeftInputCount = root.getRowType().getFieldCount();
- if (leftKeys != null) {
- List<RexNode> newLeftInputExpr = Lists.newArrayList();
- for (int i = 0; i < origLeftInputCount; i++) {
- newLeftInputExpr.add(rexBuilder.makeInputRef(root, i));
- }
-
- final List<Integer> leftJoinKeys = Lists.newArrayList();
- for (RexNode leftKey : leftKeys) {
- newLeftInputExpr.add(leftKey);
- leftJoinKeys.add(origLeftInputCount + leftJoinKeys.size());
- }
-
- ProjectRel newLeftInput =
- (ProjectRel) RelOptUtil.createProject(
- root,
- newLeftInputExpr,
- null,
- true);
-
- // maintain the group by mapping in the new ProjectRel
- if (mapRootRelToFieldProjection.containsKey(root)) {
- mapRootRelToFieldProjection.put(
- newLeftInput,
- mapRootRelToFieldProjection.get(root));
- }
-
- setRoot(newLeftInput, false);
-
- // right fields appear after the LHS fields.
- final int rightOffset = root.getRowType().getFieldCount()
- - newLeftInput.getRowType().getFieldCount();
- final List<Integer> rightKeys =
- Util.range(rightOffset, rightOffset + leftJoinKeys.size());
-
- joinCond =
- RelOptUtil.createEquiJoinCondition(newLeftInput, leftJoinKeys,
- rel, rightKeys, rexBuilder);
- } else {
- joinCond = rexBuilder.makeLiteral(true);
- }
-
- int leftFieldCount = root.getRowType().getFieldCount();
- final RelNode join =
- createJoin(
- this,
- root,
- rel,
- joinCond,
- joinType);
-
- setRoot(join, false);
-
- if (leftKeys != null
- && joinType == JoinRelType.LEFT) {
- final int leftKeyCount = leftKeys.size();
- int rightFieldLength = rel.getRowType().getFieldCount();
- assert leftKeyCount == rightFieldLength - 1;
-
- final int rexRangeRefLength = leftKeyCount + rightFieldLength;
- RelDataType returnType =
- typeFactory.createStructType(
- new AbstractList<Map.Entry<String, RelDataType>>() {
- public Map.Entry<String, RelDataType> get(
- int index) {
- return join.getRowType().getFieldList()
- .get(origLeftInputCount + index);
- }
-
- public int size() {
- return rexRangeRefLength;
- }
- });
-
- return rexBuilder.makeRangeReference(
- returnType,
- origLeftInputCount,
- false);
- } else {
- return rexBuilder.makeRangeReference(
- rel.getRowType(),
- leftFieldCount,
- joinType.generatesNullsOnRight());
- }
- }
-
- /**
- * Sets a new root relational expression, as the translation process
- * backs its way further up the tree.
- *
- * @param root New root relational expression
- * @param leaf Whether the relational expression is a leaf, that is,
- * derived from an atomic relational expression such as a table
- * name in the from clause, or the projection on top of a
- * select-subquery. In particular, relational expressions
- * derived from JOIN operators are not leaves, but set
- * expressions are.
- */
- public void setRoot(RelNode root, boolean leaf) {
- setRoot(
- Collections.singletonList(root), root, root instanceof JoinRel);
- if (leaf) {
- leaves.add(root);
- }
- this.columnMonotonicities.clear();
- }
-
- private void setRoot(
- List<RelNode> inputs,
- RelNode root,
- boolean hasSystemFields) {
- this.inputs = inputs;
- this.root = root;
- this.systemFieldList.clear();
- if (hasSystemFields) {
- this.systemFieldList.addAll(getSystemFields());
- }
- }
-
- /**
- * Notifies this Blackboard that the root just set using {@link
- * #setRoot(RelNode, boolean)} was derived using dataset substitution.
- *
- * <p>The default implementation is not interested in such
- * notifications, and does nothing.
- *
- * @param datasetName Dataset name
- */
- public void setDataset(String datasetName) {
- }
-
- void setRoot(List<RelNode> inputs) {
- setRoot(inputs, null, false);
- }
-
- /**
- * Returns an expression with which to reference a from-list item.
- *
- * @param name the alias of the from item
- * @return a {@link RexFieldAccess} or {@link RexRangeRef}, or null if
- * not found
- */
- RexNode lookupExp(String name) {
- if (nameToNodeMap != null) {
- RexNode node = nameToNodeMap.get(name);
- if (node == null) {
- throw Util.newInternal(
- "Unknown identifier '" + name
- + "' encountered while expanding expression" + node);
- }
- return node;
- }
- int[] offsets = {-1};
- final SqlValidatorScope[] ancestorScopes = {null};
- SqlValidatorNamespace foundNs =
- scope.resolve(name, ancestorScopes, offsets);
- if (foundNs == null) {
- return null;
- }
-
- // Found in current query's from list. Find which from item.
- // We assume that the order of the from clause items has been
- // preserved.
- SqlValidatorScope ancestorScope = ancestorScopes[0];
- boolean isParent = ancestorScope != scope;
- if ((inputs != null) && !isParent) {
- int offset = offsets[0];
- final LookupContext rels =
- new LookupContext(this, inputs, systemFieldList.size());
- return lookup(offset, rels);
- } else {
- // We're referencing a relational expression which has not been
- // converted yet. This occurs when from items are correlated,
- // e.g. "select from emp as emp join emp.getDepts() as dept".
- // Create a temporary expression.
- assert isParent;
- DeferredLookup lookup = new DeferredLookup(this, name);
- String correlName = createCorrel();
- mapCorrelToDeferred.put(correlName, lookup);
- final RelDataType rowType = foundNs.getRowType();
- return rexBuilder.makeCorrel(rowType, correlName);
- }
- }
-
- /**
- * Creates an expression with which to reference the expression whose
- * offset in its from-list is {@code offset}.
- */
- RexNode lookup(
- int offset,
- LookupContext lookupContext) {
- Pair<RelNode, Integer> pair = lookupContext.findRel(offset);
- return rexBuilder.makeRangeReference(
- pair.left.getRowType(),
- pair.right,
- false);
- }
-
- RelDataTypeField getRootField(RexInputRef inputRef) {
- int fieldOffset = inputRef.getIndex();
- for (RelNode input : inputs) {
- RelDataType rowType = input.getRowType();
- if (rowType == null) {
- // TODO: remove this once leastRestrictive
- // is correctly implemented
- return null;
- }
- if (fieldOffset < rowType.getFieldCount()) {
- return rowType.getFieldList().get(fieldOffset);
- }
- fieldOffset -= rowType.getFieldCount();
- }
- throw new AssertionError();
- }
-
- public void flatten(
- List<RelNode> rels,
- int systemFieldCount,
- int[] start,
- List<Pair<RelNode, Integer>> relOffsetList) {
- for (RelNode rel : rels) {
- if (leaves.contains(rel)) {
- relOffsetList.add(
- Pair.of(rel, start[0]));
- start[0] += rel.getRowType().getFieldCount();
- } else {
- if (rel instanceof JoinRel
- || rel instanceof AggregateRel) {
- start[0] += systemFieldCount;
- }
- flatten(
- rel.getInputs(),
- systemFieldCount,
- start,
- relOffsetList);
- }
- }
- }
-
- void registerSubquery(SqlNode node, RelOptUtil.Logic logic) {
- subqueryList.add(new SubQuery(node, logic));
- }
-
- ImmutableList<RelNode> retrieveCursors() {
- try {
- return ImmutableList.copyOf(cursors);
- } finally {
- cursors.clear();
- }
- }
-
- // implement SqlRexContext
- public RexNode convertExpression(SqlNode expr) {
- // If we're in aggregation mode and this is an expression in the
- // GROUP BY clause, return a reference to the field.
- if (agg != null) {
- final SqlNode expandedGroupExpr = validator.expand(expr, scope);
- RexNode rex = agg.lookupGroupExpr(expandedGroupExpr);
- if (rex != null) {
- return rex;
- }
- if (expr instanceof SqlCall) {
- rex = agg.lookupAggregates((SqlCall) expr);
- if (rex != null) {
- return rex;
- }
- }
- }
-
- // Allow the derived class chance to override the standard
- // behavior for special kinds of expressions.
- RexNode rex = convertExtendedExpression(expr, this);
- if (rex != null) {
- return rex;
- }
-
- boolean needTruthTest;
-
- // Sub-queries and OVER expressions are not like ordinary
- // expressions.
- final SqlKind kind = expr.getKind();
- final SubQuery subQuery;
- switch (kind) {
- case CURSOR:
- case SELECT:
- case EXISTS:
- case SCALAR_QUERY:
- subQuery = subqueryMap.get(expr);
- assert subQuery != null;
- rex = subQuery.expr;
- assert rex != null : "rex != null";
-
- if (kind == SqlKind.CURSOR) {
- // cursor reference is pre-baked
- return rex;
- }
- if (((kind == SqlKind.SCALAR_QUERY)
- || (kind == SqlKind.EXISTS))
- && isConvertedSubq(rex)) {
- // scalar subquery or EXISTS has been converted to a
- // constant
- return rex;
- }
-
- RexNode fieldAccess;
- needTruthTest = false;
-
- // The indicator column is the last field of the subquery.
- fieldAccess =
- rexBuilder.makeFieldAccess(
- rex,
- rex.getType().getFieldCount() - 1);
-
- // The indicator column will be nullable if it comes from
- // the null-generating side of the join. For EXISTS, add an
- // "IS TRUE" check so that the result is "BOOLEAN NOT NULL".
- if (fieldAccess.getType().isNullable()) {
- if (kind == SqlKind.EXISTS) {
- needTruthTest = true;
- }
- }
-
- if (needTruthTest) {
- fieldAccess =
- rexBuilder.makeCall(
- SqlStdOperatorTable.IS_NOT_NULL,
- fieldAccess);
- }
- return fieldAccess;
-
- case IN:
- subQuery = subqueryMap.get(expr);
- assert subQuery != null;
- assert subQuery.expr != null : "expr != null";
- return subQuery.expr;
-
- case OVER:
- return convertOver(this, expr);
-
- default:
- // fall through
- }
-
- // Apply standard conversions.
- rex = expr.accept(this);
- Util.permAssert(rex != null, "conversion result not null");
- return rex;
- }
-
- /**
- * Converts an item in an ORDER BY clause, extracting DESC, NULLS LAST
- * and NULLS FIRST flags first.
- */
- public RexNode convertSortExpression(SqlNode expr, Set<SqlKind> flags) {
- switch (expr.getKind()) {
- case DESCENDING:
- case NULLS_LAST:
- case NULLS_FIRST:
- flags.add(expr.getKind());
- final SqlNode operand = ((SqlCall) expr).operand(0);
- return convertSortExpression(operand, flags);
- default:
- return convertExpression(expr);
- }
- }
-
- /**
- * Determines whether a RexNode corresponds to a subquery that's been
- * converted to a constant.
- *
- * @param rex the expression to be examined
- * @return true if the expression is a dynamic parameter, a literal, or
- * a literal that is being cast
- */
- private boolean isConvertedSubq(RexNode rex) {
- if ((rex instanceof RexLiteral)
- || (rex instanceof RexDynamicParam)) {
- return true;
- }
- if (rex instanceof RexCall) {
- RexCall call = (RexCall) rex;
- if (call.getOperator() == SqlStdOperatorTable.CAST) {
- RexNode operand = call.getOperands().get(0);
- if (operand instanceof RexLiteral) {
- return true;
- }
- }
- }
- return false;
- }
-
- // implement SqlRexContext
- public int getGroupCount() {
- if (agg != null) {
- return agg.groupExprs.size();
- }
- if (window != null) {
- return window.isAlwaysNonEmpty() ? 1 : 0;
- }
- return -1;
- }
-
- // implement SqlRexContext
- public RexBuilder getRexBuilder() {
- return rexBuilder;
- }
-
- // implement SqlRexContext
- public RexRangeRef getSubqueryExpr(SqlCall call) {
- final SubQuery subQuery = subqueryMap.get(call);
- assert subQuery != null;
- return (RexRangeRef) subQuery.expr;
- }
-
- // implement SqlRexContext
- public RelDataTypeFactory getTypeFactory() {
- return typeFactory;
- }
-
- // implement SqlRexContext
- public DefaultValueFactory getDefaultValueFactory() {
- return defaultValueFactory;
- }
-
- // implement SqlRexContext
- public SqlValidator getValidator() {
- return validator;
- }
-
- // implement SqlRexContext
- public RexNode convertLiteral(SqlLiteral literal) {
- return exprConverter.convertLiteral(this, literal);
- }
-
- public RexNode convertInterval(SqlIntervalQualifier intervalQualifier) {
- return exprConverter.convertInterval(this, intervalQualifier);
- }
-
- // implement SqlVisitor
- public RexNode visit(SqlLiteral literal) {
- return exprConverter.convertLiteral(this, literal);
- }
-
- // implement SqlVisitor
- public RexNode visit(SqlCall call) {
- if (agg != null) {
- final SqlOperator op = call.getOperator();
- if (op.isAggregator()) {
- return agg.lookupAggregates(call);
- }
- }
- return exprConverter.convertCall(this, call);
- }
-
- // implement SqlVisitor
- public RexNode visit(SqlNodeList nodeList) {
- throw new UnsupportedOperationException();
- }
-
- // implement SqlVisitor
- public RexNode visit(SqlIdentifier id) {
- return convertIdentifier(this, id);
- }
-
- // implement SqlVisitor
- public RexNode visit(SqlDataTypeSpec type) {
- throw new UnsupportedOperationException();
- }
-
- // implement SqlVisitor
- public RexNode visit(SqlDynamicParam param) {
- return convertDynamicParam(param);
- }
-
- // implement SqlVisitor
- public RexNode visit(SqlIntervalQualifier intervalQualifier) {
- return convertInterval(intervalQualifier);
- }
-
- public List<SqlMonotonicity> getColumnMonotonicities() {
- return columnMonotonicities;
- }
- }
-
- private static class DeferredLookup {
- Blackboard bb;
- String originalRelName;
-
- DeferredLookup(
- Blackboard bb,
- String originalRelName) {
- this.bb = bb;
- this.originalRelName = originalRelName;
- }
-
- public RexFieldAccess getFieldAccess(String name) {
- return (RexFieldAccess) bb.mapCorrelateVariableToRexNode.get(name);
- }
-
- public String getOriginalRelName() {
- return originalRelName;
- }
- }
-
- /**
- * An implementation of DefaultValueFactory which always supplies NULL.
- */
- class NullDefaultValueFactory implements DefaultValueFactory {
- public boolean isGeneratedAlways(
- RelOptTable table,
- int iColumn) {
- return false;
- }
-
- public RexNode newColumnDefaultValue(
- RelOptTable table,
- int iColumn) {
- return rexBuilder.constantNull();
- }
-
- public RexNode newAttributeInitializer(
- RelDataType type,
- SqlFunction constructor,
- int iAttribute,
- List<RexNode> constructorArgs) {
- return rexBuilder.constantNull();
- }
- }
-
- /**
- * A default implementation of SubqueryConverter that does no conversion.
- */
- private class NoOpSubqueryConverter implements SubqueryConverter {
- // implement SubqueryConverter
- public boolean canConvertSubquery() {
- return false;
- }
-
- // implement SubqueryConverter
- public RexNode convertSubquery(
- SqlCall subquery,
- SqlToRelConverter parentConverter,
- boolean isExists,
- boolean isExplain) {
- throw new IllegalArgumentException();
- }
- }
-
- /**
- * Converts expressions to aggregates.
- *
- * <p>Consider the expression SELECT deptno, SUM(2 * sal) FROM emp GROUP BY
- * deptno Then
- *
- * <ul>
- * <li>groupExprs = {SqlIdentifier(deptno)}</li>
- * <li>convertedInputExprs = {RexInputRef(deptno), 2 *
- * RefInputRef(sal)}</li>
- * <li>inputRefs = {RefInputRef(#0), RexInputRef(#1)}</li>
- * <li>aggCalls = {AggCall(SUM, {1})}</li>
- * </ul>
- */
- protected class AggConverter implements SqlVisitor<Void> {
- private final Blackboard bb;
-
- private final Map<String, String> nameMap =
- new HashMap<String, String>();
-
- /**
- * The group-by expressions, in {@link SqlNode} format.
- */
- private final SqlNodeList groupExprs =
- new SqlNodeList(SqlParserPos.ZERO);
-
- /**
- * Input expressions for the group columns and aggregates, in {@link
- * RexNode} format. The first elements of the list correspond to the
- * elements in {@link #groupExprs}; the remaining elements are for
- * aggregates.
- */
- private final List<RexNode> convertedInputExprs =
- new ArrayList<RexNode>();
-
- /**
- * Names of {@link #convertedInputExprs}, where the expressions are
- * simple mappings to input fields.
- */
- private final List<String> convertedInputExprNames =
- new ArrayList<String>();
-
- private final List<RexInputRef> inputRefs =
- new ArrayList<RexInputRef>();
- private final List<AggregateCall> aggCalls =
- new ArrayList<AggregateCall>();
- private final Map<SqlNode, RexNode> aggMapping =
- new HashMap<SqlNode, RexNode>();
- private final Map<AggregateCall, RexNode> aggCallMapping =
- new HashMap<AggregateCall, RexNode>();
-
- /**
- * Creates an AggConverter.
- *
- * <p>The <code>select</code> parameter provides enough context to name
- * aggregate calls which are top-level select list items.
- *
- * @param bb Blackboard
- * @param select Query being translated; provides context to give
- */
- public AggConverter(Blackboard bb, SqlSelect select) {
- this.bb = bb;
-
- // Collect all expressions used in the select list so that aggregate
- // calls can be named correctly.
- final SqlNodeList selectList = select.getSelectList();
- for (int i = 0; i < selectList.size(); i++) {
- SqlNode selectItem = selectList.get(i);
- String name = null;
- if (SqlUtil.isCallTo(
- selectItem,
- SqlStdOperatorTable.AS)) {
- final SqlCall call = (SqlCall) selectItem;
- selectItem = call.operand(0);
- name = call.operand(1).toString();
- }
- if (name == null) {
- name = validator.deriveAlias(selectItem, i);
- }
- nameMap.put(selectItem.toString(), name);
- }
- }
-
- public void addGroupExpr(SqlNode expr) {
- RexNode convExpr = bb.convertExpression(expr);
- final RexNode rex = lookupGroupExpr(expr);
- if (rex != null) {
- return; // don't add duplicates, in e.g. "GROUP BY x, y, x"
- }
- groupExprs.add(expr);
- String name = nameMap.get(expr.toString());
- addExpr(convExpr, name);
- final RelDataType type = convExpr.getType();
- inputRefs.add(rexBuilder.makeInputRef(type, inputRefs.size()));
- }
-
- /**
- * Adds an expression, deducing an appropriate name if possible.
- *
- * @param expr Expression
- * @param name Suggested name
- */
- private void addExpr(RexNode expr, String name) {
- convertedInputExprs.add(expr);
- if ((name == null) && (expr instanceof RexInputRef)) {
- final int i = ((RexInputRef) expr).getIndex();
- name = bb.root.getRowType().getFieldList().get(i).getName();
- }
- if (convertedInputExprNames.contains(name)) {
- // In case like 'SELECT ... GROUP BY x, y, x', don't add
- // name 'x' twice.
- name = null;
- }
- convertedInputExprNames.add(name);
- }
-
- // implement SqlVisitor
- public Void visit(SqlIdentifier id) {
- return null;
- }
-
- // implement SqlVisitor
- public Void visit(SqlNodeList nodeList) {
- for (int i = 0; i < nodeList.size(); i++) {
- nodeList.get(i).accept(this);
- }
- return null;
- }
-
- // implement SqlVisitor
- public Void visit(SqlLiteral lit) {
- return null;
- }
-
- // implement SqlVisitor
- public Void visit(SqlDataTypeSpec type) {
- return null;
- }
-
- // implement SqlVisitor
- public Void visit(SqlDynamicParam param) {
- return null;
- }
-
- // implement SqlVisitor
- public Void visit(SqlIntervalQualifier intervalQualifier) {
- return null;
- }
-
- public Void visit(SqlCall call) {
- if (call.getOperator().isAggregator()) {
- assert bb.agg == this;
- List<Integer> args = new ArrayList<Integer>();
- List<RelDataType> argTypes =
- call.getOperator() instanceof SqlCountAggFunction
- ? new ArrayList<RelDataType>(call.getOperandList().size())
- : null;
- try {
- // switch out of agg mode
- bb.agg = null;
- for (SqlNode operand : call.getOperandList()) {
- RexNode convertedExpr;
-
- // special case for COUNT(*): delete the *
- if (operand instanceof SqlIdentifier) {
- SqlIdentifier id = (SqlIdentifier) operand;
- if (id.isStar()) {
- assert call.operandCount() == 1;
- assert args.isEmpty();
- break;
- }
- }
- convertedExpr = bb.convertExpression(operand);
- assert convertedExpr != null;
- if (argTypes != null) {
- argTypes.add(convertedExpr.getType());
- }
- args.add(lookupOrCreateGroupExpr(convertedExpr));
- }
- } finally {
- // switch back into agg mode
- bb.agg = this;
- }
-
- final Aggregation aggregation =
- (Aggregation) call.getOperator();
- RelDataType type = validator.deriveType(bb.scope, call);
- boolean distinct = false;
- SqlLiteral quantifier = call.getFunctionQuantifier();
- if ((null != quantifier)
- && (quantifier.getValue() == SqlSelectKeyword.DISTINCT)) {
- distinct = true;
- }
- final AggregateCall aggCall =
- new AggregateCall(
- aggregation,
- distinct,
- args,
- type,
- nameMap.get(call.toString()));
- RexNode rex =
- rexBuilder.addAggCall(
- aggCall,
- groupExprs.size(),
- aggCalls,
- aggCallMapping,
- argTypes);
- aggMapping.put(call, rex);
- } else if (call instanceof SqlSelect) {
- // rchen 2006-10-17:
- // for now do not detect aggregates in subqueries.
- return null;
- } else {
- for (SqlNode operand : call.getOperandList()) {
- // Operands are occasionally null, e.g. switched CASE arg 0.
- if (operand != null) {
- operand.accept(this);
- }
- }
- }
- return null;
- }
-
- private int lookupOrCreateGroupExpr(RexNode expr) {
- for (int i = 0; i < convertedInputExprs.size(); i++) {
- RexNode convertedInputExpr = convertedInputExprs.get(i);
- if (expr.toString().equals(convertedInputExpr.toString())) {
- return i;
- }
- }
-
- // not found -- add it
- int index = convertedInputExprs.size();
- addExpr(expr, null);
- return index;
- }
-
- /**
- * If an expression is structurally identical to one of the group-by
- * expressions, returns a reference to the expression, otherwise returns
- * null.
- */
- public RexNode lookupGroupExpr(SqlNode expr) {
- for (int i = 0; i < groupExprs.size(); i++) {
- SqlNode groupExpr = groupExprs.get(i);
- if (expr.equalsDeep(groupExpr, false)) {
- return inputRefs.get(i);
- }
- }
- return null;
- }
-
- public RexNode lookupAggregates(SqlCall call) {
- // assert call.getOperator().isAggregator();
- assert bb.agg == this;
-
- return aggMapping.get(call);
- }
-
- public List<RexNode> getPreExprs() {
- return convertedInputExprs;
- }
-
- public List<String> getPreNames() {
- return convertedInputExprNames;
- }
-
- public List<AggregateCall> getAggCalls() {
- return aggCalls;
- }
-
- public RelDataTypeFactory getTypeFactory() {
- return typeFactory;
- }
- }
-
- /**
- * Context to find a relational expression to a field offset.
- */
- private static class LookupContext {
- private final List<Pair<RelNode, Integer>> relOffsetList =
- new ArrayList<Pair<RelNode, Integer>>();
-
- /**
- * Creates a LookupContext with multiple input relational expressions.
- *
- * @param bb Context for translating this subquery
- * @param rels Relational expressions
- * @param systemFieldCount Number of system fields
- */
- LookupContext(Blackboard bb, List<RelNode> rels, int systemFieldCount) {
- bb.flatten(rels, systemFieldCount, new int[]{0}, relOffsetList);
- }
-
- /**
- * Returns the relational expression with a given offset, and the
- * ordinal in the combined row of its first field.
- *
- * <p>For example, in {@code Emp JOIN Dept}, findRel(1) returns the
- * relational expression for {@code Dept} and offset 6 (because
- * {@code Emp} has 6 fields, therefore the first field of {@code Dept}
- * is field 6.
- *
- * @param offset Offset of relational expression in FROM clause
- * @return Relational expression and the ordinal of its first field
- */
- Pair<RelNode, Integer> findRel(int offset) {
- return relOffsetList.get(offset);
- }
- }
-
- /**
- * Shuttle which walks over a tree of {@link RexNode}s and applies 'over' to
- * all agg functions.
- *
- * <p>This is necessary because the returned expression is not necessarily a
- * call to an agg function. For example,
- *
- * <blockquote><code>AVG(x)</code></blockquote>
- *
- * becomes
- *
- * <blockquote><code>SUM(x) / COUNT(x)</code></blockquote>
- *
- * <p>Any aggregate functions are converted to calls to the internal <code>
- * $Histogram</code> aggregation function and accessors such as <code>
- * $HistogramMin</code>; for example,
- *
- * <blockquote><code>MIN(x), MAX(x)</code></blockquote>
- *
- * are converted to
- *
- * <blockquote><code>$HistogramMin($Histogram(x)),
- * $HistogramMax($Histogram(x))</code></blockquote>
- *
- * Common sub-expression elmination will ensure that only one histogram is
- * computed.
- */
- private class HistogramShuttle extends RexShuttle {
- /**
- * Whether to convert calls to MIN(x) to HISTOGRAM_MIN(HISTOGRAM(x)).
- * Histograms allow rolling computation, but require more space.
- */
- static final boolean ENABLE_HISTOGRAM_AGG = false;
-
- private final List<RexNode> partitionKeys;
- private final ImmutableList<RexFieldCollation> orderKeys;
- private final RexWindowBound lowerBound;
- private final RexWindowBound upperBound;
- private final SqlWindow window;
-
- HistogramShuttle(
- List<RexNode> partitionKeys,
- ImmutableList<RexFieldCollation> orderKeys,
- RexWindowBound lowerBound, RexWindowBound upperBound,
- SqlWindow window) {
- this.partitionKeys = partitionKeys;
- this.orderKeys = orderKeys;
- this.lowerBound = lowerBound;
- this.upperBound = upperBound;
- this.window = window;
- }
-
- public RexNode visitCall(RexCall call) {
- final SqlOperator op = call.getOperator();
- if (!(op instanceof SqlAggFunction)) {
- return super.visitCall(call);
- }
- final SqlAggFunction aggOp = (SqlAggFunction) op;
- final RelDataType type = call.getType();
- List<RexNode> exprs = call.getOperands();
-
- SqlFunction histogramOp = !ENABLE_HISTOGRAM_AGG
- ? null
- : getHistogramOp(aggOp);
-
- if (histogramOp != null) {
- final RelDataType histogramType = computeHistogramType(type);
-
- // For DECIMAL, since it's already represented as a bigint we
- // want to do a reinterpretCast instead of a cast to avoid
- // losing any precision.
- boolean reinterpretCast =
- type.getSqlTypeName() == SqlTypeName.DECIMAL;
-
- // Replace original expression with CAST of not one
- // of the supported types
- if (histogramType != type) {
- exprs = new ArrayList<RexNode>(exprs);
- exprs.set(
- 0,
- reinterpretCast
- ? rexBuilder.makeReinterpretCast(histogramType, exprs.get(0),
- rexBuilder.makeLiteral(false))
- : rexBuilder.makeCast(histogramType, exprs.get(0)));
- }
-
- RexCallBinding bind =
- new RexCallBinding(
- rexBuilder.getTypeFactory(),
- SqlStdOperatorTable.HISTOGRAM_AGG,
- exprs);
-
- RexNode over =
- rexBuilder.makeOver(
- SqlStdOperatorTable.HISTOGRAM_AGG
- .inferReturnType(bind),
- SqlStdOperatorTable.HISTOGRAM_AGG,
- exprs,
- partitionKeys,
- orderKeys,
- lowerBound,
- upperBound,
- window.isRows(),
- window.isAllowPartial(),
- false);
-
- RexNode histogramCall =
- rexBuilder.makeCall(
- histogramType,
- histogramOp,
- ImmutableList.of(over));
-
- // If needed, post Cast result back to original
- // type.
- if (histogramType != type) {
- if (reinterpretCast) {
- histogramCall =
- rexBuilder.makeReinterpretCast(
- type,
- histogramCall,
- rexBuilder.makeLiteral(false));
- } else {
- histogramCall =
- rexBuilder.makeCast(type, histogramCall);
- }
- }
-
- return histogramCall;
- } else {
- boolean needSum0 = aggOp == SqlStdOperatorTable.SUM
- && type.isNullable();
- SqlAggFunction aggOpToUse =
- needSum0 ? SqlStdOperatorTable.SUM0
- : aggOp;
- return rexBuilder.makeOver(
- type,
- aggOpToUse,
- exprs,
- partitionKeys,
- orderKeys,
- lowerBound,
- upperBound,
- window.isRows(),
- window.isAllowPartial(),
- needSum0);
- }
- }
-
- /**
- * Returns the histogram operator corresponding to a given aggregate
- * function.
- *
- * <p>For example, <code>getHistogramOp({@link
- * SqlStdOperatorTable#MIN}}</code> returns {@link
- * SqlStdOperatorTable#HISTOGRAM_MIN}.
- *
- * @param aggFunction An aggregate function
- * @return Its histogram function, or null
- */
- SqlFunction getHistogramOp(SqlAggFunction aggFunction) {
- if (aggFunction == SqlStdOperatorTable.MIN) {
- return SqlStdOperatorTable.HISTOGRAM_MIN;
- } else if (aggFunction == SqlStdOperatorTable.MAX) {
- return SqlStdOperatorTable.HISTOGRAM_MAX;
- } else if (aggFunction == SqlStdOperatorTable.FIRST_VALUE) {
- return SqlStdOperatorTable.HISTOGRAM_FIRST_VALUE;
- } else if (aggFunction == SqlStdOperatorTable.LAST_VALUE) {
- return SqlStdOperatorTable.HISTOGRAM_LAST_VALUE;
- } else {
- return null;
- }
- }
-
- /**
- * Returns the type for a histogram function. It is either the actual
- * type or an an approximation to it.
- */
- private RelDataType computeHistogramType(RelDataType type) {
- if (SqlTypeUtil.isExactNumeric(type)
- && type.getSqlTypeName() != SqlTypeName.BIGINT) {
- return typeFactory.createSqlType(SqlTypeName.BIGINT);
- } else if (SqlTypeUtil.isApproximateNumeric(type)
- && type.getSqlTypeName() != SqlTypeName.DOUBLE) {
- return typeFactory.createSqlType(SqlTypeName.DOUBLE);
- } else {
- return type;
- }
- }
- }
-
- /** A sub-query, whether it needs to be translated using 2- or 3-valued
- * logic. */
- private static class SubQuery {
- final SqlNode node;
- final RelOptUtil.Logic logic;
- RexNode expr;
-
- private SubQuery(SqlNode node, RelOptUtil.Logic logic) {
- this.node = node;
- this.logic = logic;
- }
- }
-}
-
-// End SqlToRelConverter.java
diff --git a/bin/check-env.sh b/bin/check-env.sh
new file mode 100755
index 0000000000..d3cd709f6c
--- /dev/null
+++ b/bin/check-env.sh
@@ -0,0 +1,55 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#by source
+
+if [ -z "$KYLIN_HOME" ]
+then
+ echo 'please make sure KYLIN_HOME has been set'
+ exit 1
+else
+ echo "KYLIN_HOME is set to ${KYLIN_HOME}"
+fi
+
+if [ -z "$(command -v hbase version)" ]
+then
+ echo "Please make sure the user has the privilege to run hbase shell"
+ exit 1
+fi
+
+if [ -z "$(command -v hive --version)" ]
+then
+ echo "Please make sure the user has the privilege to run hive shell"
+ exit 1
+fi
+
+if [ -z "$(command -v hadoop version)" ]
+then
+ echo "Please make sure the user has the privilege to run hadoop shell"
+ exit 1
+fi
+
+WORKING_DIR=`sh $KYLIN_HOME/bin/get-properties.sh kylin.hdfs.working.dir`
+hadoop fs -mkdir -p $WORKING_DIR
+
+if [ $? != 0 ]
+then
+ echo "failed to create $WORKING_DIR, Please make sure the user has right to access $WORKING_DIR"
+ exit 1
+fi
diff --git a/bin/find-hbase-dependency.sh b/bin/find-hbase-dependency.sh
new file mode 100755
index 0000000000..03ab2a1685
--- /dev/null
+++ b/bin/find-hbase-dependency.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+hbase_classpath=`hbase classpath`
+arr=(`echo $hbase_classpath | cut -d ":" --output-delimiter=" " -f 1-`)
+hbase_common_path=
+for data in ${arr[@]}
+do
+ result=`echo $data | grep -e 'hbase-common[a-z0-9A-Z\.-]*jar' | grep -v tests`
+ if [ $result ]
+ then
+ hbase_common_path=$data
+ fi
+done
+
+if [ -z "$hbase_common_path" ]
+then
+ echo "hbase-common lib not found"
+ exit 1
+fi
+
+hbase_dependency=${hbase_common_path}
+echo "hbase dependency: $hbase_dependency"
+export hbase_dependency
diff --git a/bin/find-hive-dependency.sh b/bin/find-hive-dependency.sh
new file mode 100755
index 0000000000..5994dda001
--- /dev/null
+++ b/bin/find-hive-dependency.sh
@@ -0,0 +1,70 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+hive_env=`hive -e set | grep 'env:CLASSPATH'`
+
+hive_classpath=`echo $hive_env | grep 'env:CLASSPATH' | awk -F '=' '{print $2}'`
+arr=(`echo $hive_classpath | cut -d ":" --output-delimiter=" " -f 1-`)
+hive_conf_path=
+hive_exec_path=
+for data in ${arr[@]}
+do
+ result=`echo $data | grep -e 'hive-exec[a-z0-9A-Z\.-]*jar'`
+ if [ $result ]
+ then
+ hive_exec_path=$data
+ fi
+ result=`echo $data | grep -e 'hive[^/]*/conf'`
+ if [ $result ]
+ then
+ hive_conf_path=$data
+ fi
+done
+
+# in some versions of hive hcatalog is not in hive's classpath, find it separately
+if [ -z "$HCAT_HOME" ]
+then
+ echo "HCAT_HOME not found, try to find hcatalog path from hadoop home"
+ hadoop_home=`echo $hive_exec_path | awk -F '/hive.*/lib/' '{print $1}'`
+ if [ -d "${hadoop_home}/hive-hcatalog" ]; then
+ hcatalog_home=${hadoop_home}/hive-hcatalog
+ elif [ -d "${hadoop_home}/hive/hcatalog" ]; then
+ hcatalog_home=${hadoop_home}/hive/hcatalog
+ else
+ echo "Couldn't locate hcatalog installation, please make sure it is installed and set HCAT_HOME to the path."
+ exit 1
+ fi
+else
+ echo "HCAT_HOME is set to: $HCAT_HOME, use it to find hcatalog path:"
+ hcatalog_home=${HCAT_HOME}
+fi
+
+hcatalog=`find -L ${hcatalog_home} -name "hive-hcatalog-core[0-9\.-]*jar" 2>&1 | grep -m 1 -v 'Permission denied'`
+
+if [ -z "$hcatalog" ]
+then
+ echo "hcatalog lib not found"
+ exit 1
+fi
+
+
+hive_lib=`find -L "$(dirname $hive_exec_path)" -name '*.jar' ! -name '*calcite*' -printf '%p:' | sed 's/:$//'`
+hive_dependency=${hive_conf_path}:${hive_lib}:${hcatalog}
+echo "hive dependency: $hive_dependency"
+export hive_dependency
diff --git a/bin/get-properties.sh b/bin/get-properties.sh
new file mode 100755
index 0000000000..4c93b3bbb7
--- /dev/null
+++ b/bin/get-properties.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+if [ $# != 1 ]
+then
+ echo 'invalid input'
+ exit -1
+fi
+
+result=
+for i in `cat ${KYLIN_HOME}/conf/kylin.properties | grep -w "$1" | grep -v '^#' |awk -F '=' '{print $2}' | cut -c 1-`
+do
+ :
+ result=$i
+done
+echo $result
\ No newline at end of file
diff --git a/bin/health-check.sh b/bin/health-check.sh
new file mode 100755
index 0000000000..3e18631844
--- /dev/null
+++ b/bin/health-check.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+ALERT="your@email.com"
+
+OUTPUT=$(
+ curl --max-time 20 -# \
+ --data '{"sql":"select count(*) from test_kylin_fact","offset":0,"limit":50000,"acceptPartial":true,"project":"default"}' \
+ -H "Authorization:Basic QURNSU46S1lMSU4=" \
+ -H "Content-Type:application/json;charset=UTF-8" \
+ http://localhost:7070/kylin/api/query \
+)
+
+# ----------------------------------------------------------------------------
+
+date
+
+if [[ $OUTPUT == *"results"* ]]; then
+ echo "Good."
+else
+ echo "Bad."
+ TS_FILE=/tmp/kylin_healthmon_ts
+ LAST_TS=`stat -c%Y $TS_FILE 2>/dev/null`
+ CURR_TS=`date +%s`
+ echo last: $LAST_TS
+ echo curr: $CURR_TS
+ if (( ${LAST_TS:-"0"} < $CURR_TS - 3600 )); then
+ echo "Sending mail..."
+ echo "Kylin Prod health check failed as of $(date)." | mail -s "KYLIN PROD DOWN" $ALERT
+ if [ "$?" == "0" ]; then
+ touch $TS_FILE
+ fi
+ fi
+fi
diff --git a/bin/kylin.sh b/bin/kylin.sh
new file mode 100755
index 0000000000..8ca3c9111d
--- /dev/null
+++ b/bin/kylin.sh
@@ -0,0 +1,135 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+dir=$(dirname ${0})
+source ${dir}/check-env.sh
+mkdir -p ${KYLIN_HOME}/logs
+
+# start command
+if [ $1 == "start" ]
+then
+
+ if [ -f "${KYLIN_HOME}/pid" ]
+ then
+ PID=`cat $KYLIN_HOME/pid`
+ if ps -p $PID > /dev/null
+ then
+ echo "Kylin is running, stop it first"
+ exit 1
+ fi
+ fi
+
+ tomcat_root=${dir}/../tomcat
+ export tomcat_root
+
+
+ #The location of all hadoop/hbase configurations are difficult to get.
+ #Plus, some of the system properties are secretly set in hadoop/hbase shell command.
+ #For example, in hdp 2.2, there is a system property called hdp.version,
+ #which we cannot get until running hbase or hadoop shell command.
+ #
+ #To save all these troubles, we use hbase runjar to start tomcat.
+ #In this way we no longer need to explicitly configure hadoop/hbase related classpath for tomcat,
+ #hbase command will do all the dirty tasks for us:
+
+
+
+ useSandbox=`sh ${KYLIN_HOME}/bin/get-properties.sh kylin.sandbox`
+ spring_profile="default"
+ if [ "$useSandbox" = "true" ]
+ then spring_profile="sandbox"
+ fi
+
+ #retrive $hive_dependency and $hbase_dependency
+ source ${dir}/find-hive-dependency.sh
+ source ${dir}/find-hbase-dependency.sh
+ #retrive $KYLIN_EXTRA_START_OPTS
+ if [ -f "${dir}/setenv.sh" ]
+ then source ${dir}/setenv.sh
+ fi
+
+ export HBASE_CLASSPATH_PREFIX=${tomcat_root}/bin/bootstrap.jar:${tomcat_root}/bin/tomcat-juli.jar:${tomcat_root}/lib/*:$HBASE_CLASSPATH_PREFIX
+ export HBASE_CLASSPATH=$hive_dependency:${HBASE_CLASSPATH}
+
+ #debug if encounter NoClassDefError
+ #hbase classpath
+
+ hbase ${KYLIN_EXTRA_START_OPTS} \
+ -Djava.util.logging.config.file=${tomcat_root}/conf/logging.properties \
+ -Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager \
+ -Dorg.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=true \
+ -Dorg.apache.catalina.connector.CoyoteAdapter.ALLOW_BACKSLASH=true \
+ -Djava.endorsed.dirs=${tomcat_root}/endorsed \
+ -Dcatalina.base=${tomcat_root} \
+ -Dcatalina.home=${tomcat_root} \
+ -Djava.io.tmpdir=${tomcat_root}/temp \
+ -Dkylin.hive.dependency=${hive_dependency} \
+ -Dkylin.hbase.dependency=${hbase_dependency} \
+ -Dspring.profiles.active=${spring_profile} \
+ org.apache.hadoop.util.RunJar ${tomcat_root}/bin/bootstrap.jar org.apache.catalina.startup.Bootstrap start >> ${tomcat_root}/logs/kylin.log 2>&1 & echo $! > ${KYLIN_HOME}/pid &
+ echo "A new Kylin instance is started by $USER, stop it using \"kylin.sh stop\""
+ if [ "$useSandbox" = "true" ]
+ then echo "Please visit http://<your_sandbox_ip>:7070/kylin to play with the cubes! (Useranme: ADMIN, Password: KYLIN)"
+ else
+ echo "Please visit http://<ip>:7070/kylin"
+ fi
+ echo "You can check the log at ${tomcat_root}/logs/kylin.log"
+ exit 0
+
+# stop command
+elif [ $1 == "stop" ]
+then
+ if [ -f "${KYLIN_HOME}/pid" ]
+ then
+ PID=`cat $KYLIN_HOME/pid`
+ if ps -p $PID > /dev/null
+ then
+ echo "stopping Kylin:$PID"
+ kill $PID
+ rm ${KYLIN_HOME}/pid
+ exit 0
+ else
+ echo "Kylin is not running, please check"
+ exit 1
+ fi
+
+ else
+ echo "Kylin is not running, please check"
+ exit 1
+ fi
+
+# tool command
+elif [[ $1 = org.apache.kylin.* ]]
+then
+ #retrive $hive_dependency and $hbase_dependency
+ source ${dir}/find-hive-dependency.sh
+ source ${dir}/find-hbase-dependency.sh
+ #retrive $KYLIN_EXTRA_START_OPTS
+ if [ -f "${dir}/setenv-tool.sh" ]
+ then source ${dir}/setenv-tool.sh
+ fi
+
+ export HBASE_CLASSPATH=${KYLIN_HOME}/lib/*:$hive_dependency:${HBASE_CLASSPATH}
+
+ exec hbase -Dlog4j.configuration=kylin-log4j.properties "$@"
+
+else
+ echo "usage: kylin.sh start or kylin.sh stop"
+ exit 1
+fi
diff --git a/bin/metastore.sh b/bin/metastore.sh
new file mode 100755
index 0000000000..39593d4d85
--- /dev/null
+++ b/bin/metastore.sh
@@ -0,0 +1,67 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This script is for production metadata store manipulation
+# It is desinged to run in hadoop CLI, both in sandbox or in real hadoop environment
+#
+# If you're a developper of Kylin and want to download sandbox's metadata into your dev machine,
+# take a look at SandboxMetastoreCLI
+
+
+
+dir=$(dirname ${0})
+source ${dir}/check-env.sh
+
+if [ $1 == "backup" ]
+then
+
+ mkdir -p ${KYLIN_HOME}/meta_backups
+
+ _now=$(date +"%Y_%m_%d_%H_%M_%S")
+ _file="${KYLIN_HOME}/meta_backups/meta_${_now}"
+ echo "Starting backup to ${_file}"
+ mkdir -p ${_file}
+
+ ${KYLIN_HOME}/bin/kylin.sh org.apache.kylin.common.persistence.ResourceTool download ${_file}
+ echo "metadata store backed up to ${_file}"
+
+elif [ $1 == "restore" ]
+then
+
+ _file=$2
+ echo "Starting restoring $_file"
+ ${KYLIN_HOME}/bin/kylin.sh org.apache.kylin.common.persistence.ResourceTool upload $_file
+
+elif [ $1 == "reset" ]
+then
+
+ ${KYLIN_HOME}/bin/kylin.sh org.apache.kylin.common.persistence.ResourceTool reset
+
+elif [ $1 == "clean" ]
+then
+
+ ${KYLIN_HOME}/bin/kylin.sh org.apache.kylin.job.hadoop.cube.MetadataCleanupJob "${@:2}"
+
+else
+ echo "usage: metastore.sh backup"
+ echo " metastore.sh reset"
+ echo " metastore.sh restore PATH_TO_LOCAL_META"
+ echo " metastore.sh clean [--delete true]"
+ exit 1
+fi
diff --git a/bin/performance-monitor.sh b/bin/performance-monitor.sh
new file mode 100755
index 0000000000..d8421e31f9
--- /dev/null
+++ b/bin/performance-monitor.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+ if [ -z "$KYLIN_HOME" ]
+ then
+ echo 'please make sure KYLIN_HOME has been set'
+ exit 1
+ else
+ echo "KYLIN_HOME is set to ${KYLIN_HOME}"
+ fi
+
+ dir=$(dirname ${0})
+
+ #retrive $hive_dependency and $hbase_dependency
+ source ${dir}/find-hive-dependency.sh
+ source ${dir}/find-hbase-dependency.sh
+
+ export HBASE_CLASSPATH=$hive_dependency:${HBASE_CLASSPATH}:$KYLIN_HOME/conf
+ _monitorjar=`ls ${KYLIN_HOME}/lib |grep kylin-monitor`
+
+ hbase \
+ org.apache.hadoop.util.RunJar $KYLIN_HOME/lib/${_monitorjar} org.apache.kylin.monitor.Client
+ exit 0
diff --git a/bin/sample.sh b/bin/sample.sh
new file mode 100755
index 0000000000..e02ef51cc4
--- /dev/null
+++ b/bin/sample.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+dir=$(dirname ${0})
+source ${dir}/check-env.sh
+job_jar=`find -L ${KYLIN_HOME}/lib/ -name kylin-job*.jar`
+echo "Going to create sample tables in hive..."
+cd ${KYLIN_HOME}/sample_cube/data
+hive -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; }
+
+echo "Sample hive tables are created successfully; Going to create sample cube..."
+cd ${KYLIN_HOME}
+hbase org.apache.hadoop.util.RunJar ${job_jar} org.apache.kylin.common.persistence.ResourceTool upload ${KYLIN_HOME}/sample_cube/metadata || { exit 1; }
+echo "Sample cube is created successfully in project 'learn_kylin'; Restart Kylin server or reload the metadata from web UI to see the change."
diff --git a/bin/setenv.sh b/bin/setenv.sh
new file mode 100755
index 0000000000..f8c177afec
--- /dev/null
+++ b/bin/setenv.sh
@@ -0,0 +1,54 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# uncomment following to for it to take effect
+export KYLIN_JVM_SETTINGS="-Xms1024M -Xmx4096M -XX:MaxPermSize=128M"
+# export KYLIN_JVM_SETTINGS="-Xms16g -Xmx16g -XX:MaxPermSize=512m -XX:NewSize=3g -XX:MaxNewSize=3g -XX:SurvivorRatio=4 -XX:+CMSClassUnloadingEnabled -XX:+CMSParallelRemarkEnabled -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode -XX:CMSInitiatingOccupancyFraction=70 -XX:+DisableExplicitGC"
+
+# uncomment following to for it to take effect
+# export KYLIN_DEBUG_SETTINGS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -javaagent:${KYLIN_HOME}/lib/CustomAgent.jar -Dcom.ebay.lnp.rmiagent.port=8080"
+
+# uncomment following to for it to take effect
+# export KYLIN_LD_LIBRARY_SETTINGS="-Djava.library.path=/apache/hadoop/lib/native/Linux-amd64-64"
+
+export KYLIN_EXTRA_START_OPTS=""
+
+if [ ! -z "${KYLIN_JVM_SETTINGS}" ]
+then
+ echo "KYLIN_JVM_SETTINGS is ${KYLIN_JVM_SETTINGS}"
+ KYLIN_EXTRA_START_OPTS="${KYLIN_JVM_SETTINGS} ${KYLIN_EXTRA_START_OPTS}"
+else
+ echo "KYLIN_JVM_SETTINGS is not set, using default jvm settings"
+fi
+
+if [ ! -z "${KYLIN_DEBUG_SETTINGS}" ]
+then
+ echo "KYLIN_DEBUG_SETTINGS is ${KYLIN_DEBUG_SETTINGS}"
+ KYLIN_EXTRA_START_OPTS="${KYLIN_DEBUG_SETTINGS} ${KYLIN_EXTRA_START_OPTS}"
+else
+ echo "KYLIN_DEBUG_SETTINGS is not set, will not enable remote debuging"
+fi
+
+if [ ! -z "${KYLIN_LD_LIBRARY_SETTINGS}" ]
+then
+ echo "KYLIN_LD_LIBRARY_SETTINGS is ${KYLIN_LD_LIBRARY_SETTINGS}"
+ KYLIN_EXTRA_START_OPTS="${KYLIN_LD_LIBRARY_SETTINGS} ${KYLIN_EXTRA_START_OPTS}"
+else
+ echo "KYLIN_LD_LIBRARY_SETTINGS is not set, lzo compression at MR and hbase might not work"
+fi
diff --git a/common/pom.xml b/common/pom.xml
index 32a46c632a..dc8188c01b 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -1,3 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
@@ -6,9 +25,9 @@
<name>Kylin:Common</name>
<parent>
- <groupId>com.kylinolap</groupId>
+ <groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>0.6.3-SNAPSHOT</version>
+ <version>1.2</version>
</parent>
<properties>
@@ -78,11 +97,54 @@
<scope>provided</scope>
</dependency>
<dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <scope>provided</scope>
</dependency>
- </dependencies>
+ <!-- ES dependencies -->
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-testing-util</artifactId>
+ <version>${hbase-hadoop2.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet.jsp</groupId>
+ <artifactId>jsp-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.reflections</groupId>
+ <artifactId>reflections</artifactId>
+ <version>0.9.9-RC1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
+ <version>1.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive.hcatalog</groupId>
+ <artifactId>hive-hcatalog-core</artifactId>
+ <version>${hive-hcatalog.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
</project>
diff --git a/common/src/main/java/com/kylinolap/common/KylinConfig.java b/common/src/main/java/com/kylinolap/common/KylinConfig.java
deleted file mode 100644
index 34b348ddc6..0000000000
--- a/common/src/main/java/com/kylinolap/common/KylinConfig.java
+++ /dev/null
@@ -1,630 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.common;
-
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-
-import org.apache.commons.configuration.ConfigurationException;
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.kylinolap.common.restclient.RestClient;
-import com.kylinolap.common.util.CliCommandExecutor;
-
-/**
- * @author yangli9
- */
-public class KylinConfig {
-
- public static final String KYLIN_STORAGE_URL = "kylin.storage.url";
-
- public static final String PROP_SCAN_THRESHOLD = "PROP_SCAN_THRESHOLD";
-
- public static final String KYLIN_METADATA_URL = "kylin.metadata.url";
-
- public static final String KYLIN_REST_SERVERS = "kylin.rest.servers";
-
- public static final String KYLIN_REST_TIMEZONE = "kylin.rest.timezone";
- /**
- * The dir containing scripts for kylin. For example: /usr/lib/kylin/bin
- */
- public static final String KYLIN_SCRIPT_DIR = "kylin.script.dir";
- /**
- * The script file name for generating table metadat from hive. For example:
- * generateTable.sh
- */
- public static final String KYLIN_SCRIPT_GEN_TABLE_META = "kylin.script.genTableMeta";
-
- public static final String KYLIN_JOB_CONCURRENT_MAX_LIMIT = "kylin.job.concurrent.max.limit";
-
- public static final String KYLIN_JOB_YARN_APP_REST_CHECK_STATUS_URL = "kylin.job.yarn.app.rest.check.status.url";
-
- public static final String KYLIN_JOB_YARN_APP_REST_CHECK_INTERVAL_SECONDS = "kylin.job.yarn.app.rest.check.interval.seconds";
-
- public static final String KYLIN_TMP_HDFS_DIR = "kylin.tmp.hdfs.dir";
-
- public static final String HIVE_TABLE_LOCATION_PREFIX = "hive.table.location.";
-
- public static final String KYLIN_JOB_REMOTE_CLI_PASSWORD = "kylin.job.remote.cli.password";
-
- public static final String KYLIN_JOB_REMOTE_CLI_USERNAME = "kylin.job.remote.cli.username";
-
- public static final String KYLIN_JOB_REMOTE_CLI_HOSTNAME = "kylin.job.remote.cli.hostname";
-
- public static final String KYLIN_JOB_REMOTE_CLI_WORKING_DIR = "kylin.job.remote.cli.working.dir";
-
- public static final String KYLIN_JOB_CMD_EXTRA_ARGS = "kylin.job.cmd.extra.args";
- /**
- * Toggle to indicate whether to use hive for table flattening. Default
- * true.
- */
- public static final String KYLIN_JOB_HIVE_FLATTEN = "kylin.job.hive.flatten";
-
- public static final String KYLIN_JOB_RUN_AS_REMOTE_CMD = "kylin.job.run.as.remote.cmd";
-
- public static final String KYLIN_JOB_MAPREDUCE_DEFAULT_REDUCE_COUNT_RATIO = "kylin.job.mapreduce.default.reduce.count.ratio";
-
- public static final String KYLIN_JOB_MAPREDUCE_DEFAULT_REDUCE_INPUT_MB = "kylin.job.mapreduce.default.reduce.input.mb";
-
- public static final String KYLIN_JOB_MAPREDUCE_MAX_REDUCER_NUMBER = "kylin.job.mapreduce.max.reducer.number";
-
- public static final String KYLIN_JOB_JAR = "kylin.job.jar";
-
- public static final String COPROCESSOR_LOCAL_JAR = "kylin.coprocessor.local.jar";
- public static final String COPROCESSOR_SCAN_BITS_THRESHOLD = "kylin.coprocessor.scan.bits.threshold";
-
- public static final String KYLIN_JOB_JAR_LOCAL = "kylin.job.jar.local";
-
- public static final String KYLIN_JOB_LOG_DIR = "kylin.job.log.dir";
-
- public static final String KYLIN_HDFS_WORKING_DIR = "kylin.hdfs.working.dir";
-
- public static final String HIVE_PASSWORD = "hive.password";
-
- public static final String HIVE_USER = "hive.user";
-
- public static final String HIVE_URL = "hive.url";
- /**
- * Key string to point to the kylin conf directory
- */
- public static final String KYLIN_CONF = "KYLIN_CONF";
- /**
- * Key string to specify the kylin evn: prod, dev, qa
- */
- public static final String KYLIN_ENV = "KYLIN_ENV";
- /**
- * Default Kylin conf path
- */
- public static final String KYLIN_CONF_DEFAULT = "/etc/kylin";
- /**
- * Kylin properties file
- */
- public static final String KYLIN_CONF_PROPERTIES_FILE = "kylin.properties";
-
- public static final String MAIL_ENABLED = "mail.enabled";
-
- public static final String MAIL_HOST = "mail.host";
-
- public static final String MAIL_USERNAME = "mail.username";
-
- public static final String MAIL_PASSWORD = "mail.password";
-
- public static final String MAIL_SENDER = "mail.sender";
-
- private static final Logger logger = LoggerFactory.getLogger(KylinConfig.class);
-
- // static cached instances
- private static KylinConfig ENV_INSTANCE = null;
-
- public static KylinConfig getInstanceFromEnv() {
- if (ENV_INSTANCE == null) {
- try {
- KylinConfig config = loadKylinConfig();
- ENV_INSTANCE = config;
- } catch (IllegalArgumentException e) {
- throw new IllegalStateException("Failed to find KylinConfig ", e);
- }
- }
- return ENV_INSTANCE;
- }
-
- public static void destoryInstance() {
- ENV_INSTANCE = null;
- }
-
-
- /**
- * This method only for test case. You can get a KylinConfig instance by
- * path "/a/b/c", where "/a/b/c/kylin.properties" exists. By default, the
- * getInstanceFromEnv() should be called.
- *
- * @param confPath
- * @return
- * @deprecated
- */
- public static KylinConfig getInstanceForTest(String confPath) {
- File file = new File(confPath);
- if (!file.exists() || !file.isDirectory()) {
- throw new IllegalArgumentException(confPath + " is not a valid path");
- }
-
- String env = System.getProperty(KYLIN_CONF);
- System.setProperty(KYLIN_CONF, confPath);
- KylinConfig config = getInstanceFromEnv();
- if (env == null) {
- System.clearProperty(KYLIN_CONF);
- } else {
- System.setProperty(KYLIN_CONF, env);
- }
- return config;
- }
-
- public static enum UriType {
- PROPERTIES_FILE, REST_ADDR, LOCAL_FOLDER
- }
-
- private static UriType decideUriType(String metaUri) {
-
- try {
- File file = new File(metaUri);
- if (file.exists()) {
- if (file.isDirectory()) {
- return UriType.LOCAL_FOLDER;
- } else if (file.isFile()) {
- if (file.getName().equalsIgnoreCase(KYLIN_CONF_PROPERTIES_FILE)) {
- return UriType.PROPERTIES_FILE;
- } else {
- throw new IllegalStateException("Metadata uri : " + metaUri + " is a local file but not kylin.properties");
- }
- }
- } else {
- if (RestClient.matchFullRestPattern(metaUri))
- return UriType.REST_ADDR;
- else
- throw new IllegalStateException("Metadata uri : " + metaUri + " is not a valid REST URI address");
- }
- } catch (Exception e) {
- logger.info(e.getLocalizedMessage());
- throw new IllegalStateException("Metadata uri : " + metaUri + " is not recognized");
- }
-
- return null;
- }
-
- public static KylinConfig createInstanceFromUri(String uri) {
- /**
- * --hbase: 1. PROPERTIES_FILE: path to kylin.properties 2. REST_ADDR:
- * rest service resource, format: user:password@host:port --local: 1.
- * LOCAL_FOLDER: path to resource folder
- */
- UriType uriType = decideUriType(uri);
- logger.info("The URI " + uri + " is recognized as " + uriType);
-
- if (uriType == UriType.LOCAL_FOLDER) {
- KylinConfig config = new KylinConfig();
- config.setMetadataUrl(uri);
- return config;
- } else if (uriType == UriType.PROPERTIES_FILE) {
- KylinConfig config;
- try {
- config = new KylinConfig();
- InputStream is = new FileInputStream(uri);
- config.reloadKylinConfig(is);
- is.close();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- return config;
- } else {// rest_addr
- try {
- KylinConfig config = new KylinConfig();
- RestClient client = new RestClient(uri);
- String propertyText = client.getKylinProperties();
- InputStream is = IOUtils.toInputStream(propertyText);
- config.reloadKylinConfig(is);
- is.close();
- return config;
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- public static KylinConfig getKylinConfigFromInputStream(InputStream is) {
- KylinConfig config = new KylinConfig();
- config.reloadKylinConfig(is);
- return config;
- }
-
- // ============================================================================
-
- /**
- * Find config from environment. The Search process: 1. Check the
- * $KYLIN_CONF/kylin.properties 2. Check the /etc/kylin/kylin.properties 3.
- * Check the kylin.properties in classpath
- *
- * @return
- */
- private static KylinConfig loadKylinConfig() {
- InputStream is = getKylinPropertiesAsInputSteam();
- if (is == null) {
- throw new IllegalArgumentException("Failed to load kylin config");
- }
- KylinConfig config = new KylinConfig();
- config.reloadKylinConfig(is);
- return config;
- }
-
- private PropertiesConfiguration kylinConfig = new PropertiesConfiguration();
-
- public CliCommandExecutor getCliCommandExecutor() throws IOException {
- CliCommandExecutor exec = new CliCommandExecutor();
- if (getRunAsRemoteCommand()) {
- exec.setRunAtRemote(getRemoteHadoopCliHostname(), getRemoteHadoopCliUsername(), getRemoteHadoopCliPassword());
- }
- return exec;
- }
-
- // ============================================================================
-
- public String getStorageUrl() {
- return getOptional(KYLIN_STORAGE_URL);
- }
-
- public String getZookeeperString() {
- // storage url format is: hbase:zookeeperHostname:port:/znode
-
- String zookeeperString = "";
- String storageString = this.getStorageUrl();
- if (!storageString.startsWith("hbase:")) {
- return "";
- }
-
- int cut = storageString.indexOf("/");
- if (cut != -1) {
- zookeeperString = storageString.substring(6, cut - 1);
- } else {
- zookeeperString = storageString.substring(6);
- }
- return zookeeperString;
- }
-
- public String getHiveUrl() {
- return getOptional(HIVE_URL, "");
- }
-
- public String getHiveUser() {
- return getOptional(HIVE_USER, "");
- }
-
- public String getHivePassword() {
- return getOptional(HIVE_PASSWORD, "");
- }
-
- public String getHdfsWorkingDirectory() {
- return getRequired(KYLIN_HDFS_WORKING_DIR);
- }
-
- public String getKylinJobLogDir() {
- return getOptional(KYLIN_JOB_LOG_DIR, "/tmp/kylin/logs");
- }
-
- public String getKylinJobJarPath() {
- return getRequired(KYLIN_JOB_JAR);
- }
-
- public void overrideKylinJobJarPath(String path) {
- kylinConfig.setProperty(KYLIN_JOB_JAR, path);
- }
-
- public String getCoprocessorLocalJar() {
- return getRequired(COPROCESSOR_LOCAL_JAR);
- }
-
- public void overrideCoprocessorLocalJar(String path) {
- kylinConfig.setProperty(COPROCESSOR_LOCAL_JAR, path);
- }
-
- public int getCoprocessorScanBitsThreshold() {
- return Integer.parseInt(getOptional(COPROCESSOR_SCAN_BITS_THRESHOLD, "32"));
- }
-
- public double getDefaultHadoopJobReducerInputMB() {
- return Double.parseDouble(getOptional(KYLIN_JOB_MAPREDUCE_DEFAULT_REDUCE_INPUT_MB, "500"));
- }
-
- public double getDefaultHadoopJobReducerCountRatio() {
- return Double.parseDouble(getOptional(KYLIN_JOB_MAPREDUCE_DEFAULT_REDUCE_COUNT_RATIO, "1.0"));
- }
-
- public int getHadoopJobMaxReducerNumber() {
- return Integer.parseInt(getOptional(KYLIN_JOB_MAPREDUCE_MAX_REDUCER_NUMBER, "5000"));
- }
-
- public boolean getRunAsRemoteCommand() {
- return Boolean.parseBoolean(getOptional(KYLIN_JOB_RUN_AS_REMOTE_CMD));
- }
-
- public String getRemoteHadoopCliHostname() {
- return getOptional(KYLIN_JOB_REMOTE_CLI_HOSTNAME);
- }
-
- public String getRemoteHadoopCliUsername() {
- return getOptional(KYLIN_JOB_REMOTE_CLI_USERNAME);
- }
-
- public String getRemoteHadoopCliPassword() {
- return getOptional(KYLIN_JOB_REMOTE_CLI_PASSWORD);
- }
-
- public String getCliWorkingDir() {
- return getOptional(KYLIN_JOB_REMOTE_CLI_WORKING_DIR);
- }
-
- public String getMapReduceCmdExtraArgs() {
- return getOptional(KYLIN_JOB_CMD_EXTRA_ARGS);
- }
-
- public boolean getFlatTableByHive() {
- return Boolean.parseBoolean(getOptional(KYLIN_JOB_HIVE_FLATTEN, "true"));
- }
-
- public String getOverrideHiveTableLocation(String table) {
- return getOptional(HIVE_TABLE_LOCATION_PREFIX + table.toUpperCase());
- }
-
- public String getTempHDFSDir() {
- return getOptional(KYLIN_TMP_HDFS_DIR, "/tmp/kylin");
- }
-
- public String getYarnStatusServiceUrl() {
- return getOptional(KYLIN_JOB_YARN_APP_REST_CHECK_STATUS_URL, null);
- }
-
- public int getYarnStatusCheckIntervalSeconds() {
- return Integer.parseInt(getOptional(KYLIN_JOB_YARN_APP_REST_CHECK_INTERVAL_SECONDS, "60"));
- }
-
- /**
- * @return
- */
- public int getMaxConcurrentJobLimit() {
- return Integer.parseInt(getOptional(KYLIN_JOB_CONCURRENT_MAX_LIMIT, "10"));
- }
-
- /**
- * @return
- */
- public String getTimeZone() {
- return getOptional(KYLIN_REST_TIMEZONE, "PST");
- }
-
- public String[] getRestServers() {
- String nodes = getOptional(KYLIN_REST_SERVERS);
- if (nodes == null)
- return null;
- return nodes.split("\\s*,\\s*");
- }
-
- /**
- * @return
- */
- public String getAdminDls() {
- return getOptional("kylin.job.admin.dls", null);
- }
-
- public int getScanThreshold() {
- return Integer.parseInt(getOptional("kylin.query.scan.threshold", "10000000"));
- }
-
- /**
- * @return
- */
- public long getJobStepTimeout() {
- return Long.parseLong(getOptional("kylin.job.step.timeout", String.valueOf(2 * 60 * 60)));
- }
-
- public String getServerMode() {
- return this.getOptional("kylin.server.mode", "all");
- }
-
- public Long getQueryDurationCacheThreshold() {
- return Long.parseLong(this.getOptional("kylin.query.cache.threshold.duration", String.valueOf(2000)));
- }
-
- public Long getQueryScanCountCacheThreshold() {
- return Long.parseLong(this.getOptional("kylin.query.cache.threshold.scancount", String.valueOf(10 * 1024)));
- }
-
- public boolean isQuerySecureEnabled() {
- return Boolean.parseBoolean(this.getOptional("kylin.query.security.enabled", "false"));
- }
-
- public int getConcurrentScanThreadCount() {
- return Integer.parseInt(this.getOptional("kylin.query.scan.thread.count", "40"));
- }
-
- public boolean isQueryCacheEnabled() {
- return Boolean.parseBoolean(this.getOptional("kylin.query.cache.enabled", "true"));
- }
-
- public int getHBaseKeyValueSize() {
- return Integer.parseInt(this.getOptional("kylin.hbase.client.keyvalue.maxsize", "10485760"));
- }
-
- private String getOptional(String prop) {
- return kylinConfig.getString(prop);
- }
-
- private String getOptional(String prop, String dft) {
- return kylinConfig.getString(prop, dft);
- }
-
- private String getRequired(String prop) {
- String r = kylinConfig.getString(prop);
- if (StringUtils.isEmpty(r))
- throw new IllegalArgumentException("missing '" + prop + "' in conf/kylin_instance.properties");
- return r;
- }
-
- void reloadKylinConfig(InputStream is) {
- PropertiesConfiguration config = new PropertiesConfiguration();
- try {
- config.load(is);
- } catch (ConfigurationException e) {
- throw new RuntimeException("Cannot load kylin config.", e);
- } finally {
- try {
- is.close();
- } catch (IOException e) {
- logger.error("Failed to close inputstream.", e);
- }
- }
- this.kylinConfig = config;
- }
-
- public void writeProperties(File file) throws IOException {
- try {
- kylinConfig.save(file);
- } catch (ConfigurationException ex) {
- throw new IOException("Error writing KylinConfig to " + file, ex);
- }
- }
-
- public static InputStream getKylinPropertiesAsInputSteam() {
- File propFile = null;
-
- // 1st, find conf path from env
- String path = System.getProperty(KYLIN_CONF);
- if (path == null) {
- path = System.getenv(KYLIN_CONF);
- }
- propFile = getKylinPropertiesFile(path);
-
- // 2nd, find /etc/kylin
- if (propFile == null) {
- propFile = getKylinPropertiesFile(KYLIN_CONF_DEFAULT);
- }
- if (propFile != null) {
- logger.debug("Loading property file " + propFile.getAbsolutePath());
- try {
- return new FileInputStream(propFile);
- } catch (FileNotFoundException e) {
- logger.warn("Failed to read properties " + propFile.getAbsolutePath() + " and skip");
- }
- }
-
- // 3rd, find classpath
- logger.info("Search " + KYLIN_CONF_PROPERTIES_FILE + " from classpath ...");
- InputStream is = KylinConfig.class.getClassLoader().getResourceAsStream("kylin.properties");
- if (is == null) {
- logger.info("Did not find properties file " + KYLIN_CONF_PROPERTIES_FILE + " from classpath");
- }
- return is;
- }
-
- /**
- * Check if there is kylin.properties exist
- *
- *
- * @param path
- * @param env
- * @return the properties file
- */
- private static File getKylinPropertiesFile(String path) {
- if (path == null)
- return null;
-
- File propFile = new File(path, KYLIN_CONF_PROPERTIES_FILE);
- if (propFile.exists()) {
- logger.info(KYLIN_CONF_PROPERTIES_FILE + " was found at " + propFile.getAbsolutePath());
- return propFile;
- }
-
- logger.info(KYLIN_CONF_PROPERTIES_FILE + " was NOT found at " + propFile.getAbsolutePath());
- return null;
- }
-
- public String getMetadataUrl() {
- return getOptional(KYLIN_METADATA_URL);
- }
-
- public String getMetadataUrlPrefix() {
- String hbaseMetadataUrl = getMetadataUrl();
- String defaultPrefix = "kylin_metadata";
-
- if (org.apache.commons.lang3.StringUtils.containsIgnoreCase(hbaseMetadataUrl, "hbase:")) {
- int cut = hbaseMetadataUrl.indexOf('@');
- String tmp = cut < 0 ? defaultPrefix : hbaseMetadataUrl.substring(0, cut);
- return tmp;
- } else {
- return defaultPrefix;
- }
- }
-
- public void setMetadataUrl(String metadataUrl) {
- kylinConfig.setProperty(KYLIN_METADATA_URL, metadataUrl);
- }
-
- /**
- * return -1 if there is no setting
- *
- * @return
- */
- public int getPropScanThreshold() {
- return kylinConfig.getInt(PROP_SCAN_THRESHOLD, -1);
- }
-
- public String getProperty(String key, String defaultValue) {
- return kylinConfig.getString(key, defaultValue);
- }
-
- /**
- * Set a new key:value into the kylin config.
- *
- * @param key
- * @param value
- */
- public void setProperty(String key, String value) {
- logger.info("Kylin Config was updated with " + key + " : " + value);
- kylinConfig.setProperty(key, value);
- }
-
- public String getConfigAsString() throws IOException {
- try {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- kylinConfig.save(baos);
- String content = baos.toString();
- return content;
- } catch (ConfigurationException ex) {
- throw new IOException("Error writing KylinConfig to String", ex);
- }
- }
-
- public String toString() {
- return getMetadataUrl();
- }
-
-}
diff --git a/common/src/main/java/com/kylinolap/common/hll/HyperLogLogPlusCounter.java b/common/src/main/java/com/kylinolap/common/hll/HyperLogLogPlusCounter.java
deleted file mode 100644
index 06d2f5f034..0000000000
--- a/common/src/main/java/com/kylinolap/common/hll/HyperLogLogPlusCounter.java
+++ /dev/null
@@ -1,357 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.common.hll;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.zip.GZIPInputStream;
-import java.util.zip.GZIPOutputStream;
-
-import org.apache.commons.compress.utils.IOUtils;
-
-import com.google.common.hash.HashFunction;
-import com.google.common.hash.Hashing;
-import com.kylinolap.common.util.BytesUtil;
-import com.ning.compress.lzf.LZFDecoder;
-import com.ning.compress.lzf.LZFEncoder;
-
-/**
- * About compression, test on HLLC data shows
- *
- * - LZF compression ratio is around 65%-80%, fast - GZIP compression ratio is
- * around 41%-46%, very slow
- *
- * @author yangli9
- */
-public class HyperLogLogPlusCounter implements Comparable<HyperLogLogPlusCounter> {
-
- private final int p;
- private final int m;
- private final HashFunction hashFunc;
- byte[] registers;
-
- public HyperLogLogPlusCounter() {
- this(10);
- }
-
- public HyperLogLogPlusCounter(int p) {
- this(p, Hashing.murmur3_128());
- }
-
- public HyperLogLogPlusCounter(HyperLogLogPlusCounter another) {
- this(another.p, another.hashFunc);
- merge(another);
- }
-
- /** The larger p is, the more storage (2^p bytes), the better accuracy */
- private HyperLogLogPlusCounter(int p, HashFunction hashFunc) {
- this.p = p;
- this.m = (int) Math.pow(2, p);
- this.hashFunc = hashFunc;
- this.registers = new byte[m];
- }
-
- public void clear() {
- for (int i = 0; i < m; i++)
- registers[i] = 0;
- }
-
- public void add(String value) {
- add(hashFunc.hashString(value).asLong());
- }
-
- public void add(byte[] value) {
- add(hashFunc.hashBytes(value).asLong());
- }
-
- protected void add(long hash) {
- int bucketMask = m - 1;
- int bucket = (int) (hash & bucketMask);
- int firstOnePos = Long.numberOfLeadingZeros(hash | bucketMask) + 1;
-
- if (firstOnePos > registers[bucket])
- registers[bucket] = (byte) firstOnePos;
- }
-
- public void merge(HyperLogLogPlusCounter another) {
- assert this.p == another.p;
- assert this.hashFunc == another.hashFunc;
-
- for (int i = 0; i < m; i++) {
- if (registers[i] < another.registers[i])
- registers[i] = another.registers[i];
- }
- }
-
- public long getCountEstimate() {
- return new HLLCSnapshot(this).getCountEstimate();
- }
-
- public int getMemBytes() {
- return 12 + m;
- }
-
- public double getErrorRate() {
- return 1.04 / Math.sqrt(m);
- }
-
- private int size() {
- int size = 0;
- for (int i = 0; i < m; i++) {
- if (registers[i] > 0)
- size++;
- }
- return size;
- }
-
- // ============================================================================
-
- // a memory efficient snapshot of HLL registers which can yield count
- // estimate later
- public static class HLLCSnapshot {
- byte p;
- double registerSum;
- int zeroBuckets;
-
- public HLLCSnapshot(HyperLogLogPlusCounter hllc) {
- p = (byte) hllc.p;
- registerSum = 0;
- zeroBuckets = 0;
-
- byte[] registers = hllc.registers;
- for (int i = 0; i < hllc.m; i++) {
- if (registers[i] == 0) {
- registerSum++;
- zeroBuckets++;
- } else {
- registerSum += 1.0 / (1 << registers[i]);
- }
- }
- }
-
- public long getCountEstimate() {
- int m = (int) Math.pow(2, p);
- double alpha = 1 / (2 * Math.log(2) * (1 + (3 * Math.log(2) - 1) / m));
- double alphaMM = alpha * m * m;
- double estimate = alphaMM / registerSum;
-
- // small cardinality adjustment
- if (zeroBuckets >= m * 0.07) { // (reference presto's HLL impl)
- estimate = m * Math.log(m * 1.0 / zeroBuckets);
- } else if (HyperLogLogPlusTable.isBiasCorrection(m, estimate)) {
- estimate = HyperLogLogPlusTable.biasCorrection(p, estimate);
- }
-
- return Math.round(estimate);
- }
- }
-
- // ============================================================================
-
- public static interface Compressor {
-
- byte[] compress(ByteBuffer buf, int offset, int length) throws IOException;
-
- byte[] decompress(ByteBuffer buf, int offset, int length) throws IOException;
- }
-
- static final Compressor GZIP_COMPRESSOR = new Compressor() {
- @Override
- public byte[] compress(ByteBuffer buf, int offset, int length) throws IOException {
- ByteArrayOutputStream bout = new ByteArrayOutputStream();
- GZIPOutputStream gzout = new GZIPOutputStream(bout);
- gzout.write(buf.array(), offset, length);
- gzout.close();
- return bout.toByteArray();
- }
-
- @Override
- public byte[] decompress(ByteBuffer buf, int offset, int length) throws IOException {
- ByteArrayInputStream bin = new ByteArrayInputStream(buf.array(), offset, length);
- GZIPInputStream gzin = new GZIPInputStream(bin);
- ByteArrayOutputStream bout = new ByteArrayOutputStream();
- IOUtils.copy(gzin, bout);
- gzin.close();
- bout.close();
- return bout.toByteArray();
- }
- };
-
- static final Compressor LZF_COMPRESSOR = new Compressor() {
- @Override
- public byte[] compress(ByteBuffer buf, int offset, int length) throws IOException {
- return LZFEncoder.encode(buf.array(), offset, length);
- }
-
- @Override
- public byte[] decompress(ByteBuffer buf, int offset, int length) throws IOException {
- return LZFDecoder.decode(buf.array(), offset, length);
- }
- };
-
- public static final int COMPRESSION_THRESHOLD = Integer.MAX_VALUE; // bytes,
- // disable
- // due to
- // slowness
- public static final byte COMPRESSION_FLAG = (byte) 0x02;
- public static final Compressor DEFAULT_COMPRESSOR = GZIP_COMPRESSOR; // LZF
- // lib
- // has
- // a
- // bug
- // at
- // the
- // moment
-
- public void writeRegisters(final ByteBuffer out) throws IOException {
- int startPos = out.position();
-
- final int indexLen = getRegisterIndexSize();
- int size = size();
-
- // decide output scheme -- map (3*size bytes) or array (2^p bytes)
- byte scheme;
- if ((indexLen + 1) * size < m)
- scheme = 0; // map
- else
- scheme = 1; // array
- out.put(scheme);
-
- if (scheme == 0) { // map scheme
- BytesUtil.writeVInt(size, out);
- for (int i = 0; i < m; i++) {
- if (registers[i] > 0) {
- BytesUtil.writeUnsigned(i, indexLen, out);
- out.put(registers[i]);
- }
- }
- } else { // array scheme
- for (int i = 0; i < m; i++) {
- out.put(registers[i]);
- }
- }
-
- // do compression if needed
- int len = out.position() - startPos;
- if (len < COMPRESSION_THRESHOLD)
- return;
-
- scheme |= COMPRESSION_FLAG;
- byte[] compressed = DEFAULT_COMPRESSOR.compress(out, startPos + 1, len - 1);
- out.position(startPos);
- out.put(scheme);
- BytesUtil.writeVInt(compressed.length, out);
- out.put(compressed);
- }
-
- public void readRegisters(ByteBuffer in) throws IOException {
- byte scheme = in.get();
- if ((scheme & COMPRESSION_FLAG) > 0) {
- scheme ^= COMPRESSION_FLAG;
- int compressedLen = BytesUtil.readVInt(in);
- int end = in.position() + compressedLen;
- byte[] decompressed = DEFAULT_COMPRESSOR.decompress(in, in.position(), compressedLen);
- in.position(end);
- in = ByteBuffer.wrap(decompressed);
- }
-
- if (scheme == 0) { // map scheme
- clear();
- int size = BytesUtil.readVInt(in);
- if (size > m)
- throw new IllegalArgumentException("register size (" + size + ") cannot be larger than m (" + m + ")");
- int indexLen = getRegisterIndexSize();
- for (int i = 0; i < size; i++) {
- int key = BytesUtil.readUnsigned(in, indexLen);
- registers[key] = in.get();
- }
- } else { // array scheme
- for (int i = 0; i < m; i++) {
- registers[i] = in.get();
- }
- }
- }
-
- private int getRegisterIndexSize() {
- return (p - 1) / 8 + 1; // 2 when p=16, 3 when p=17
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((hashFunc == null) ? 0 : hashFunc.hashCode());
- result = prime * result + p;
- result = prime * result + Arrays.hashCode(registers);
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- HyperLogLogPlusCounter other = (HyperLogLogPlusCounter) obj;
- if (hashFunc == null) {
- if (other.hashFunc != null)
- return false;
- } else if (!hashFunc.equals(other.hashFunc))
- return false;
- if (p != other.p)
- return false;
- if (!Arrays.equals(registers, other.registers))
- return false;
- return true;
- }
-
- @Override
- public int compareTo(HyperLogLogPlusCounter o) {
- if (o == null)
- return 1;
-
- long e1 = this.getCountEstimate();
- long e2 = o.getCountEstimate();
-
- if (e1 == e2)
- return 0;
- else if (e1 > e2)
- return 1;
- else
- return -1;
- }
-
- public static void main(String[] args) throws IOException {
- dumpErrorRates();
- }
-
- static void dumpErrorRates() {
- for (int p = 10; p <= 18; p++) {
- double rate = new HyperLogLogPlusCounter(p).getErrorRate();
- double er = Math.round(rate * 10000) / 100D;
- double er2 = Math.round(rate * 2 * 10000) / 100D;
- double er3 = Math.round(rate * 3 * 10000) / 100D;
- long size = Math.round(Math.pow(2, p));
- System.out.println("HLLC" + p + ",\t" + size + " bytes,\t68% err<" + er + "%" + ",\t95% err<" + er2 + "%" + ",\t99.7% err<" + er3 + "%");
- }
- }
-}
diff --git a/common/src/main/java/com/kylinolap/common/hll/HyperLogLogPlusTable.java b/common/src/main/java/com/kylinolap/common/hll/HyperLogLogPlusTable.java
deleted file mode 100644
index 82811c3a2d..0000000000
--- a/common/src/main/java/com/kylinolap/common/hll/HyperLogLogPlusTable.java
+++ /dev/null
@@ -1,197 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.common.hll;
-
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-/**
- * @author xjiang
- *
- */
-public class HyperLogLogPlusTable {
-
- // threshold and bias data taken from google's bias correction data set:
- // https://docs.google.com/document/d/1gyjfMHy43U9OWBXxfaeG-3MjGzejW1dlpyMwEYAAWEI/view?fullscreen#
- public static final double[] thresholdData = { 10, 20, 40, 80, 220, 400, 900, 1800, 3100, 6500, 15500, 20000, 50000, 120000, 350000 };
-
- public static final double[][] rawEstimateData = {
- // precision 4
- { 11, 11.717, 12.207, 12.7896, 13.2882, 13.8204, 14.3772, 14.9342, 15.5202, 16.161, 16.7722, 17.4636, 18.0396, 18.6766, 19.3566, 20.0454, 20.7936, 21.4856, 22.2666, 22.9946, 23.766, 24.4692, 25.3638, 26.0764, 26.7864, 27.7602, 28.4814, 29.433, 30.2926, 31.0664, 31.9996, 32.7956, 33.5366, 34.5894, 35.5738, 36.2698, 37.3682, 38.0544, 39.2342, 40.0108, 40.7966, 41.9298, 42.8704, 43.6358, 44.5194, 45.773, 46.6772, 47.6174, 48.4888, 49.3304, 50.2506, 51.4996, 52.3824, 53.3078, 54.3984, 55.5838, 56.6618, 57.2174, 58.3514, 59.0802, 60.1482, 61.0376, 62.3598, 62.8078, 63.9744, 64.914, 65.781, 67.1806, 68.0594, 68.8446, 69.7928, 70.8248, 71.8324, 72.8598, 73.6246, 74.7014, 75.393, 76.6708, 77.2394, },
- // precision 5
- { 23, 23.1194, 23.8208, 24.2318, 24.77, 25.2436, 25.7774, 26.2848, 26.8224, 27.3742, 27.9336, 28.503, 29.0494, 29.6292, 30.2124, 30.798, 31.367, 31.9728, 32.5944, 33.217, 33.8438, 34.3696, 35.0956, 35.7044, 36.324, 37.0668, 37.6698, 38.3644, 39.049, 39.6918, 40.4146, 41.082, 41.687, 42.5398, 43.2462, 43.857, 44.6606, 45.4168, 46.1248, 46.9222, 47.6804, 48.447, 49.3454, 49.9594, 50.7636, 51.5776, 52.331, 53.19, 53.9676, 54.7564, 55.5314, 56.4442, 57.3708, 57.9774, 58.9624, 59.8796, 60.755, 61.472, 62.2076, 63.1024, 63.8908, 64.7338, 65.7728, 66.629, 67.413, 68.3266, 69.1524, 70.2642, 71.1806, 72.0566, 72.9192, 73.7598, 74.3516, 75.5802, 76.4386, 77.4916, 78.1524, 79.1892, 79.8414, 80.8798, 81.8376, 82.4698, 83.7656, 84.331, 85.5914, 86.6012, 87.7016, 88.5582, 89.3394, 90.3544, 91.4912, 92.308, 93.3552, 93.9746, 95.2052, 95.727, 97.1322, 98.3944, 98.7588, 100.242, 101.1914, 102.2538, 102.8776, 103.6292, 105.1932, 105.9152, 107.0868, 107.6728, 108.7144, 110.3114, 110.8716,
- 111.245, 112.7908, 113.7064, 114.636, 115.7464, 116.1788, 117.7464, 118.4896, 119.6166, 120.5082, 121.7798, 122.9028, 123.4426, 124.8854, 125.705, 126.4652, 128.3464, 128.3462, 130.0398, 131.0342, 131.0042, 132.4766, 133.511, 134.7252, 135.425, 136.5172, 138.0572, 138.6694, 139.3712, 140.8598, 141.4594, 142.554, 143.4006, 144.7374, 146.1634, 146.8994, 147.605, 147.9304, 149.1636, 150.2468, 151.5876, 152.2096, 153.7032, 154.7146, 155.807, 156.9228, 157.0372, 158.5852, },
- // precision 6
- { 46, 46.1902, 47.271, 47.8358, 48.8142, 49.2854, 50.317, 51.354, 51.8924, 52.9436, 53.4596, 54.5262, 55.6248, 56.1574, 57.2822, 57.837, 58.9636, 60.074, 60.7042, 61.7976, 62.4772, 63.6564, 64.7942, 65.5004, 66.686, 67.291, 68.5672, 69.8556, 70.4982, 71.8204, 72.4252, 73.7744, 75.0786, 75.8344, 77.0294, 77.8098, 79.0794, 80.5732, 81.1878, 82.5648, 83.2902, 84.6784, 85.3352, 86.8946, 88.3712, 89.0852, 90.499, 91.2686, 92.6844, 94.2234, 94.9732, 96.3356, 97.2286, 98.7262, 100.3284, 101.1048, 102.5962, 103.3562, 105.1272, 106.4184, 107.4974, 109.0822, 109.856, 111.48, 113.2834, 114.0208, 115.637, 116.5174, 118.0576, 119.7476, 120.427, 122.1326, 123.2372, 125.2788, 126.6776, 127.7926, 129.1952, 129.9564, 131.6454, 133.87, 134.5428, 136.2, 137.0294, 138.6278, 139.6782, 141.792, 143.3516, 144.2832, 146.0394, 147.0748, 148.4912, 150.849, 151.696, 153.5404, 154.073, 156.3714, 157.7216, 158.7328, 160.4208, 161.4184, 163.9424, 165.2772, 166.411, 168.1308, 168.769, 170.9258,
- 172.6828, 173.7502, 175.706, 176.3886, 179.0186, 180.4518, 181.927, 183.4172, 184.4114, 186.033, 188.5124, 189.5564, 191.6008, 192.4172, 193.8044, 194.997, 197.4548, 198.8948, 200.2346, 202.3086, 203.1548, 204.8842, 206.6508, 206.6772, 209.7254, 210.4752, 212.7228, 214.6614, 215.1676, 217.793, 218.0006, 219.9052, 221.66, 223.5588, 225.1636, 225.6882, 227.7126, 229.4502, 231.1978, 232.9756, 233.1654, 236.727, 238.1974, 237.7474, 241.1346, 242.3048, 244.1948, 245.3134, 246.879, 249.1204, 249.853, 252.6792, 253.857, 254.4486, 257.2362, 257.9534, 260.0286, 260.5632, 262.663, 264.723, 265.7566, 267.2566, 267.1624, 270.62, 272.8216, 273.2166, 275.2056, 276.2202, 278.3726, 280.3344, 281.9284, 283.9728, 284.1924, 286.4872, 287.587, 289.807, 291.1206, 292.769, 294.8708, 296.665, 297.1182, 299.4012, 300.6352, 302.1354, 304.1756, 306.1606, 307.3462, 308.5214, 309.4134, 310.8352, 313.9684, 315.837, 316.7796, 318.9858, },
- // precision 7
- { 92, 93.4934, 94.9758, 96.4574, 97.9718, 99.4954, 101.5302, 103.0756, 104.6374, 106.1782, 107.7888, 109.9522, 111.592, 113.2532, 114.9086, 116.5938, 118.9474, 120.6796, 122.4394, 124.2176, 125.9768, 128.4214, 130.2528, 132.0102, 133.8658, 135.7278, 138.3044, 140.1316, 142.093, 144.0032, 145.9092, 148.6306, 150.5294, 152.5756, 154.6508, 156.662, 159.552, 161.3724, 163.617, 165.5754, 167.7872, 169.8444, 172.7988, 174.8606, 177.2118, 179.3566, 181.4476, 184.5882, 186.6816, 189.0824, 191.0258, 193.6048, 196.4436, 198.7274, 200.957, 203.147, 205.4364, 208.7592, 211.3386, 213.781, 215.8028, 218.656, 221.6544, 223.996, 226.4718, 229.1544, 231.6098, 234.5956, 237.0616, 239.5758, 242.4878, 244.5244, 248.2146, 250.724, 252.8722, 255.5198, 258.0414, 261.941, 264.9048, 266.87, 269.4304, 272.028, 274.4708, 278.37, 281.0624, 283.4668, 286.5532, 289.4352, 293.2564, 295.2744, 298.2118, 300.7472, 304.1456, 307.2928, 309.7504, 312.5528, 315.979, 318.2102, 322.1834, 324.3494, 327.325,
- 330.6614, 332.903, 337.2544, 339.9042, 343.215, 345.2864, 348.0814, 352.6764, 355.301, 357.139, 360.658, 363.1732, 366.5902, 369.9538, 373.0828, 375.922, 378.9902, 382.7328, 386.4538, 388.1136, 391.2234, 394.0878, 396.708, 401.1556, 404.1852, 406.6372, 409.6822, 412.7796, 416.6078, 418.4916, 422.131, 424.5376, 428.1988, 432.211, 434.4502, 438.5282, 440.912, 444.0448, 447.7432, 450.8524, 453.7988, 456.7858, 458.8868, 463.9886, 466.5064, 468.9124, 472.6616, 475.4682, 478.582, 481.304, 485.2738, 488.6894, 490.329, 496.106, 497.6908, 501.1374, 504.5322, 506.8848, 510.3324, 513.4512, 516.179, 520.4412, 522.6066, 526.167, 528.7794, 533.379, 536.067, 538.46, 542.9116, 545.692, 547.9546, 552.493, 555.2722, 557.335, 562.449, 564.2014, 569.0738, 571.0974, 574.8564, 578.2996, 581.409, 583.9704, 585.8098, 589.6528, 594.5998, 595.958, 600.068, 603.3278, 608.2016, 609.9632, 612.864, 615.43, 620.7794, 621.272, 625.8644, 629.206, 633.219, 634.5154, 638.6102, },
- // precision 8
- { 184.2152, 187.2454, 190.2096, 193.6652, 196.6312, 199.6822, 203.249, 206.3296, 210.0038, 213.2074, 216.4612, 220.27, 223.5178, 227.4412, 230.8032, 234.1634, 238.1688, 241.6074, 245.6946, 249.2664, 252.8228, 257.0432, 260.6824, 264.9464, 268.6268, 272.2626, 276.8376, 280.4034, 284.8956, 288.8522, 292.7638, 297.3552, 301.3556, 305.7526, 309.9292, 313.8954, 318.8198, 322.7668, 327.298, 331.6688, 335.9466, 340.9746, 345.1672, 349.3474, 354.3028, 358.8912, 364.114, 368.4646, 372.9744, 378.4092, 382.6022, 387.843, 392.5684, 397.1652, 402.5426, 407.4152, 412.5388, 417.3592, 422.1366, 427.486, 432.3918, 437.5076, 442.509, 447.3834, 453.3498, 458.0668, 463.7346, 469.1228, 473.4528, 479.7, 484.644, 491.0518, 495.5774, 500.9068, 506.432, 512.1666, 517.434, 522.6644, 527.4894, 533.6312, 538.3804, 544.292, 550.5496, 556.0234, 562.8206, 566.6146, 572.4188, 579.117, 583.6762, 590.6576, 595.7864, 601.509, 607.5334, 612.9204, 619.772, 624.2924, 630.8654, 636.1836, 642.745, 649.1316,
- 655.0386, 660.0136, 666.6342, 671.6196, 678.1866, 684.4282, 689.3324, 695.4794, 702.5038, 708.129, 713.528, 720.3204, 726.463, 732.7928, 739.123, 744.7418, 751.2192, 756.5102, 762.6066, 769.0184, 775.2224, 781.4014, 787.7618, 794.1436, 798.6506, 805.6378, 811.766, 819.7514, 824.5776, 828.7322, 837.8048, 843.6302, 849.9336, 854.4798, 861.3388, 867.9894, 873.8196, 880.3136, 886.2308, 892.4588, 899.0816, 905.4076, 912.0064, 917.3878, 923.619, 929.998, 937.3482, 943.9506, 947.991, 955.1144, 962.203, 968.8222, 975.7324, 981.7826, 988.7666, 994.2648, 1000.3128, 1007.4082, 1013.7536, 1020.3376, 1026.7156, 1031.7478, 1037.4292, 1045.393, 1051.2278, 1058.3434, 1062.8726, 1071.884, 1076.806, 1082.9176, 1089.1678, 1095.5032, 1102.525, 1107.2264, 1115.315, 1120.93, 1127.252, 1134.1496, 1139.0408, 1147.5448, 1153.3296, 1158.1974, 1166.5262, 1174.3328, 1175.657, 1184.4222, 1190.9172, 1197.1292, 1204.4606, 1210.4578, 1218.8728, 1225.3336, 1226.6592, 1236.5768, 1241.363,
- 1249.4074, 1254.6566, 1260.8014, 1266.5454, 1274.5192, },
- // precision 9
- { 369, 374.8294, 381.2452, 387.6698, 394.1464, 400.2024, 406.8782, 413.6598, 420.462, 427.2826, 433.7102, 440.7416, 447.9366, 455.1046, 462.285, 469.0668, 476.306, 483.8448, 491.301, 498.9886, 506.2422, 513.8138, 521.7074, 529.7428, 537.8402, 545.1664, 553.3534, 561.594, 569.6886, 577.7876, 585.65, 594.228, 602.8036, 611.1666, 620.0818, 628.0824, 637.2574, 646.302, 655.1644, 664.0056, 672.3802, 681.7192, 690.5234, 700.2084, 708.831, 718.485, 728.1112, 737.4764, 746.76, 756.3368, 766.5538, 775.5058, 785.2646, 795.5902, 804.3818, 814.8998, 824.9532, 835.2062, 845.2798, 854.4728, 864.9582, 875.3292, 886.171, 896.781, 906.5716, 916.7048, 927.5322, 937.875, 949.3972, 958.3464, 969.7274, 980.2834, 992.1444, 1003.4264, 1013.0166, 1024.018, 1035.0438, 1046.34, 1057.6856, 1068.9836, 1079.0312, 1091.677, 1102.3188, 1113.4846, 1124.4424, 1135.739, 1147.1488, 1158.9202, 1169.406, 1181.5342, 1193.2834, 1203.8954, 1216.3286, 1226.2146, 1239.6684, 1251.9946, 1262.123, 1275.4338,
- 1285.7378, 1296.076, 1308.9692, 1320.4964, 1333.0998, 1343.9864, 1357.7754, 1368.3208, 1380.4838, 1392.7388, 1406.0758, 1416.9098, 1428.9728, 1440.9228, 1453.9292, 1462.617, 1476.05, 1490.2996, 1500.6128, 1513.7392, 1524.5174, 1536.6322, 1548.2584, 1562.3766, 1572.423, 1587.1232, 1596.5164, 1610.5938, 1622.5972, 1633.1222, 1647.7674, 1658.5044, 1671.57, 1683.7044, 1695.4142, 1708.7102, 1720.6094, 1732.6522, 1747.841, 1756.4072, 1769.9786, 1782.3276, 1797.5216, 1808.3186, 1819.0694, 1834.354, 1844.575, 1856.2808, 1871.1288, 1880.7852, 1893.9622, 1906.3418, 1920.6548, 1932.9302, 1945.8584, 1955.473, 1968.8248, 1980.6446, 1995.9598, 2008.349, 2019.8556, 2033.0334, 2044.0206, 2059.3956, 2069.9174, 2082.6084, 2093.7036, 2106.6108, 2118.9124, 2132.301, 2144.7628, 2159.8422, 2171.0212, 2183.101, 2193.5112, 2208.052, 2221.3194, 2233.3282, 2247.295, 2257.7222, 2273.342, 2286.5638, 2299.6786, 2310.8114, 2322.3312, 2335.516, 2349.874, 2363.5968, 2373.865, 2387.1918,
- 2401.8328, 2414.8496, 2424.544, 2436.7592, 2447.1682, 2464.1958, 2474.3438, 2489.0006, 2497.4526, 2513.6586, 2527.19, 2540.7028, 2553.768, },
- // precision 10
- { 738.1256, 750.4234, 763.1064, 775.4732, 788.4636, 801.0644, 814.488, 827.9654, 841.0832, 854.7864, 868.1992, 882.2176, 896.5228, 910.1716, 924.7752, 938.899, 953.6126, 968.6492, 982.9474, 998.5214, 1013.1064, 1028.6364, 1044.2468, 1059.4588, 1075.3832, 1091.0584, 1106.8606, 1123.3868, 1139.5062, 1156.1862, 1172.463, 1189.339, 1206.1936, 1223.1292, 1240.1854, 1257.2908, 1275.3324, 1292.8518, 1310.5204, 1328.4854, 1345.9318, 1364.552, 1381.4658, 1400.4256, 1419.849, 1438.152, 1456.8956, 1474.8792, 1494.118, 1513.62, 1532.5132, 1551.9322, 1570.7726, 1590.6086, 1610.5332, 1630.5918, 1650.4294, 1669.7662, 1690.4106, 1710.7338, 1730.9012, 1750.4486, 1770.1556, 1791.6338, 1812.7312, 1833.6264, 1853.9526, 1874.8742, 1896.8326, 1918.1966, 1939.5594, 1961.07, 1983.037, 2003.1804, 2026.071, 2047.4884, 2070.0848, 2091.2944, 2114.333, 2135.9626, 2158.2902, 2181.0814, 2202.0334, 2224.4832, 2246.39, 2269.7202, 2292.1714, 2314.2358, 2338.9346, 2360.891, 2384.0264, 2408.3834, 2430.1544,
- 2454.8684, 2476.9896, 2501.4368, 2522.8702, 2548.0408, 2570.6738, 2593.5208, 2617.0158, 2640.2302, 2664.0962, 2687.4986, 2714.2588, 2735.3914, 2759.6244, 2781.8378, 2808.0072, 2830.6516, 2856.2454, 2877.2136, 2903.4546, 2926.785, 2951.2294, 2976.468, 3000.867, 3023.6508, 3049.91, 3073.5984, 3098.162, 3121.5564, 3146.2328, 3170.9484, 3195.5902, 3221.3346, 3242.7032, 3271.6112, 3296.5546, 3317.7376, 3345.072, 3369.9518, 3394.326, 3418.1818, 3444.6926, 3469.086, 3494.2754, 3517.8698, 3544.248, 3565.3768, 3588.7234, 3616.979, 3643.7504, 3668.6812, 3695.72, 3719.7392, 3742.6224, 3770.4456, 3795.6602, 3819.9058, 3844.002, 3869.517, 3895.6824, 3920.8622, 3947.1364, 3973.985, 3995.4772, 4021.62, 4046.628, 4074.65, 4096.2256, 4121.831, 4146.6406, 4173.276, 4195.0744, 4223.9696, 4251.3708, 4272.9966, 4300.8046, 4326.302, 4353.1248, 4374.312, 4403.0322, 4426.819, 4450.0598, 4478.5206, 4504.8116, 4528.8928, 4553.9584, 4578.8712, 4603.8384, 4632.3872, 4655.5128, 4675.821,
- 4704.6222, 4731.9862, 4755.4174, 4781.2628, 4804.332, 4832.3048, 4862.8752, 4883.4148, 4906.9544, 4935.3516, 4954.3532, 4984.0248, 5011.217, 5035.3258, 5057.3672, 5084.1828, },
- // precision 11
- { 1477, 1501.6014, 1526.5802, 1551.7942, 1577.3042, 1603.2062, 1629.8402, 1656.2292, 1682.9462, 1709.9926, 1737.3026, 1765.4252, 1793.0578, 1821.6092, 1849.626, 1878.5568, 1908.527, 1937.5154, 1967.1874, 1997.3878, 2027.37, 2058.1972, 2089.5728, 2120.1012, 2151.9668, 2183.292, 2216.0772, 2247.8578, 2280.6562, 2313.041, 2345.714, 2380.3112, 2414.1806, 2447.9854, 2481.656, 2516.346, 2551.5154, 2586.8378, 2621.7448, 2656.6722, 2693.5722, 2729.1462, 2765.4124, 2802.8728, 2838.898, 2876.408, 2913.4926, 2951.4938, 2989.6776, 3026.282, 3065.7704, 3104.1012, 3143.7388, 3181.6876, 3221.1872, 3261.5048, 3300.0214, 3339.806, 3381.409, 3421.4144, 3461.4294, 3502.2286, 3544.651, 3586.6156, 3627.337, 3670.083, 3711.1538, 3753.5094, 3797.01, 3838.6686, 3882.1678, 3922.8116, 3967.9978, 4009.9204, 4054.3286, 4097.5706, 4140.6014, 4185.544, 4229.5976, 4274.583, 4316.9438, 4361.672, 4406.2786, 4451.8628, 4496.1834, 4543.505, 4589.1816, 4632.5188, 4678.2294, 4724.8908, 4769.0194, 4817.052,
- 4861.4588, 4910.1596, 4956.4344, 5002.5238, 5048.13, 5093.6374, 5142.8162, 5187.7894, 5237.3984, 5285.6078, 5331.0858, 5379.1036, 5428.6258, 5474.6018, 5522.7618, 5571.5822, 5618.59, 5667.9992, 5714.88, 5763.454, 5808.6982, 5860.3644, 5910.2914, 5953.571, 6005.9232, 6055.1914, 6104.5882, 6154.5702, 6199.7036, 6251.1764, 6298.7596, 6350.0302, 6398.061, 6448.4694, 6495.933, 6548.0474, 6597.7166, 6646.9416, 6695.9208, 6742.6328, 6793.5276, 6842.1934, 6894.2372, 6945.3864, 6996.9228, 7044.2372, 7094.1374, 7142.2272, 7192.2942, 7238.8338, 7288.9006, 7344.0908, 7394.8544, 7443.5176, 7490.4148, 7542.9314, 7595.6738, 7641.9878, 7694.3688, 7743.0448, 7797.522, 7845.53, 7899.594, 7950.3132, 7996.455, 8050.9442, 8092.9114, 8153.1374, 8197.4472, 8252.8278, 8301.8728, 8348.6776, 8401.4698, 8453.551, 8504.6598, 8553.8944, 8604.1276, 8657.6514, 8710.3062, 8758.908, 8807.8706, 8862.1702, 8910.4668, 8960.77, 9007.2766, 9063.164, 9121.0534, 9164.1354, 9218.1594, 9267.767,
- 9319.0594, 9372.155, 9419.7126, 9474.3722, 9520.1338, 9572.368, 9622.7702, 9675.8448, 9726.5396, 9778.7378, 9827.6554, 9878.1922, 9928.7782, 9978.3984, 10026.578, 10076.5626, 10137.1618, 10177.5244, 10229.9176, },
- // precision 12
- { 2954, 3003.4782, 3053.3568, 3104.3666, 3155.324, 3206.9598, 3259.648, 3312.539, 3366.1474, 3420.2576, 3474.8376, 3530.6076, 3586.451, 3643.38, 3700.4104, 3757.5638, 3815.9676, 3875.193, 3934.838, 3994.8548, 4055.018, 4117.1742, 4178.4482, 4241.1294, 4304.4776, 4367.4044, 4431.8724, 4496.3732, 4561.4304, 4627.5326, 4693.949, 4761.5532, 4828.7256, 4897.6182, 4965.5186, 5034.4528, 5104.865, 5174.7164, 5244.6828, 5316.6708, 5387.8312, 5459.9036, 5532.476, 5604.8652, 5679.6718, 5753.757, 5830.2072, 5905.2828, 5980.0434, 6056.6264, 6134.3192, 6211.5746, 6290.0816, 6367.1176, 6447.9796, 6526.5576, 6606.1858, 6686.9144, 6766.1142, 6847.0818, 6927.9664, 7010.9096, 7091.0816, 7175.3962, 7260.3454, 7344.018, 7426.4214, 7511.3106, 7596.0686, 7679.8094, 7765.818, 7852.4248, 7936.834, 8022.363, 8109.5066, 8200.4554, 8288.5832, 8373.366, 8463.4808, 8549.7682, 8642.0522, 8728.3288, 8820.9528, 8907.727, 9001.0794, 9091.2522, 9179.988, 9269.852, 9362.6394, 9453.642, 9546.9024, 9640.6616,
- 9732.6622, 9824.3254, 9917.7484, 10007.9392, 10106.7508, 10196.2152, 10289.8114, 10383.5494, 10482.3064, 10576.8734, 10668.7872, 10764.7156, 10862.0196, 10952.793, 11049.9748, 11146.0702, 11241.4492, 11339.2772, 11434.2336, 11530.741, 11627.6136, 11726.311, 11821.5964, 11918.837, 12015.3724, 12113.0162, 12213.0424, 12306.9804, 12408.4518, 12504.8968, 12604.586, 12700.9332, 12798.705, 12898.5142, 12997.0488, 13094.788, 13198.475, 13292.7764, 13392.9698, 13486.8574, 13590.1616, 13686.5838, 13783.6264, 13887.2638, 13992.0978, 14081.0844, 14189.9956, 14280.0912, 14382.4956, 14486.4384, 14588.1082, 14686.2392, 14782.276, 14888.0284, 14985.1864, 15088.8596, 15187.0998, 15285.027, 15383.6694, 15495.8266, 15591.3736, 15694.2008, 15790.3246, 15898.4116, 15997.4522, 16095.5014, 16198.8514, 16291.7492, 16402.6424, 16499.1266, 16606.2436, 16697.7186, 16796.3946, 16902.3376, 17005.7672, 17100.814, 17206.8282, 17305.8262, 17416.0744, 17508.4092, 17617.0178, 17715.4554,
- 17816.758, 17920.1748, 18012.9236, 18119.7984, 18223.2248, 18324.2482, 18426.6276, 18525.0932, 18629.8976, 18733.2588, 18831.0466, 18940.1366, 19032.2696, 19131.729, 19243.4864, 19349.6932, 19442.866, 19547.9448, 19653.2798, 19754.4034, 19854.0692, 19965.1224, 20065.1774, 20158.2212, 20253.353, 20366.3264, 20463.22, },
- // precision 13
- { 5908.5052, 6007.2672, 6107.347, 6208.5794, 6311.2622, 6414.5514, 6519.3376, 6625.6952, 6732.5988, 6841.3552, 6950.5972, 7061.3082, 7173.5646, 7287.109, 7401.8216, 7516.4344, 7633.3802, 7751.2962, 7870.3784, 7990.292, 8110.79, 8233.4574, 8356.6036, 8482.2712, 8607.7708, 8735.099, 8863.1858, 8993.4746, 9123.8496, 9255.6794, 9388.5448, 9522.7516, 9657.3106, 9792.6094, 9930.5642, 10068.794, 10206.7256, 10347.81, 10490.3196, 10632.0778, 10775.9916, 10920.4662, 11066.124, 11213.073, 11358.0362, 11508.1006, 11659.1716, 11808.7514, 11959.4884, 12112.1314, 12265.037, 12420.3756, 12578.933, 12734.311, 12890.0006, 13047.2144, 13207.3096, 13368.5144, 13528.024, 13689.847, 13852.7528, 14018.3168, 14180.5372, 14346.9668, 14513.5074, 14677.867, 14846.2186, 15017.4186, 15184.9716, 15356.339, 15529.2972, 15697.3578, 15871.8686, 16042.187, 16216.4094, 16389.4188, 16565.9126, 16742.3272, 16919.0042, 17094.7592, 17273.965, 17451.8342, 17634.4254, 17810.5984, 17988.9242, 18171.051,
- 18354.7938, 18539.466, 18721.0408, 18904.9972, 19081.867, 19271.9118, 19451.8694, 19637.9816, 19821.2922, 20013.1292, 20199.3858, 20387.8726, 20572.9514, 20770.7764, 20955.1714, 21144.751, 21329.9952, 21520.709, 21712.7016, 21906.3868, 22096.2626, 22286.0524, 22475.051, 22665.5098, 22862.8492, 23055.5294, 23249.6138, 23437.848, 23636.273, 23826.093, 24020.3296, 24213.3896, 24411.7392, 24602.9614, 24805.7952, 24998.1552, 25193.9588, 25389.0166, 25585.8392, 25780.6976, 25981.2728, 26175.977, 26376.5252, 26570.1964, 26773.387, 26962.9812, 27163.0586, 27368.164, 27565.0534, 27758.7428, 27961.1276, 28163.2324, 28362.3816, 28565.7668, 28758.644, 28956.9768, 29163.4722, 29354.7026, 29561.1186, 29767.9948, 29959.9986, 30164.0492, 30366.9818, 30562.5338, 30762.9928, 30976.1592, 31166.274, 31376.722, 31570.3734, 31770.809, 31974.8934, 32179.5286, 32387.5442, 32582.3504, 32794.076, 32989.9528, 33191.842, 33392.4684, 33595.659, 33801.8672, 34000.3414, 34200.0922, 34402.6792,
- 34610.0638, 34804.0084, 35011.13, 35218.669, 35418.6634, 35619.0792, 35830.6534, 36028.4966, 36229.7902, 36438.6422, 36630.7764, 36833.3102, 37048.6728, 37247.3916, 37453.5904, 37669.3614, 37854.5526, 38059.305, 38268.0936, 38470.2516, 38674.7064, 38876.167, 39068.3794, 39281.9144, 39492.8566, 39684.8628, 39898.4108, 40093.1836, 40297.6858, 40489.7086, 40717.2424, },
- // precision 14
- { 11817.475, 12015.0046, 12215.3792, 12417.7504, 12623.1814, 12830.0086, 13040.0072, 13252.503, 13466.178, 13683.2738, 13902.0344, 14123.9798, 14347.394, 14573.7784, 14802.6894, 15033.6824, 15266.9134, 15502.8624, 15741.4944, 15980.7956, 16223.8916, 16468.6316, 16715.733, 16965.5726, 17217.204, 17470.666, 17727.8516, 17986.7886, 18247.6902, 18510.9632, 18775.304, 19044.7486, 19314.4408, 19587.202, 19862.2576, 20135.924, 20417.0324, 20697.9788, 20979.6112, 21265.0274, 21550.723, 21841.6906, 22132.162, 22428.1406, 22722.127, 23020.5606, 23319.7394, 23620.4014, 23925.2728, 24226.9224, 24535.581, 24845.505, 25155.9618, 25470.3828, 25785.9702, 26103.7764, 26420.4132, 26742.0186, 27062.8852, 27388.415, 27714.6024, 28042.296, 28365.4494, 28701.1526, 29031.8008, 29364.2156, 29704.497, 30037.1458, 30380.111, 30723.8168, 31059.5114, 31404.9498, 31751.6752, 32095.2686, 32444.7792, 32794.767, 33145.204, 33498.4226, 33847.6502, 34209.006, 34560.849, 34919.4838, 35274.9778, 35635.1322,
- 35996.3266, 36359.1394, 36722.8266, 37082.8516, 37447.7354, 37815.9606, 38191.0692, 38559.4106, 38924.8112, 39294.6726, 39663.973, 40042.261, 40416.2036, 40779.2036, 41161.6436, 41540.9014, 41921.1998, 42294.7698, 42678.5264, 43061.3464, 43432.375, 43818.432, 44198.6598, 44583.0138, 44970.4794, 45353.924, 45729.858, 46118.2224, 46511.5724, 46900.7386, 47280.6964, 47668.1472, 48055.6796, 48446.9436, 48838.7146, 49217.7296, 49613.7796, 50010.7508, 50410.0208, 50793.7886, 51190.2456, 51583.1882, 51971.0796, 52376.5338, 52763.319, 53165.5534, 53556.5594, 53948.2702, 54346.352, 54748.7914, 55138.577, 55543.4824, 55941.1748, 56333.7746, 56745.1552, 57142.7944, 57545.2236, 57935.9956, 58348.5268, 58737.5474, 59158.5962, 59542.6896, 59958.8004, 60349.3788, 60755.0212, 61147.6144, 61548.194, 61946.0696, 62348.6042, 62763.603, 63162.781, 63560.635, 63974.3482, 64366.4908, 64771.5876, 65176.7346, 65597.3916, 65995.915, 66394.0384, 66822.9396, 67203.6336, 67612.2032,
- 68019.0078, 68420.0388, 68821.22, 69235.8388, 69640.0724, 70055.155, 70466.357, 70863.4266, 71276.2482, 71677.0306, 72080.2006, 72493.0214, 72893.5952, 73314.5856, 73714.9852, 74125.3022, 74521.2122, 74933.6814, 75341.5904, 75743.0244, 76166.0278, 76572.1322, 76973.1028, 77381.6284, 77800.6092, 78189.328, 78607.0962, 79012.2508, 79407.8358, 79825.725, 80238.701, 80646.891, 81035.6436, 81460.0448, 81876.3884, },
- // precision 15
- { 23635.0036, 24030.8034, 24431.4744, 24837.1524, 25246.7928, 25661.326, 26081.3532, 26505.2806, 26933.9892, 27367.7098, 27805.318, 28248.799, 28696.4382, 29148.8244, 29605.5138, 30066.8668, 30534.2344, 31006.32, 31480.778, 31962.2418, 32447.3324, 32938.0232, 33432.731, 33930.728, 34433.9896, 34944.1402, 35457.5588, 35974.5958, 36497.3296, 37021.9096, 37554.326, 38088.0826, 38628.8816, 39171.3192, 39723.2326, 40274.5554, 40832.3142, 41390.613, 41959.5908, 42532.5466, 43102.0344, 43683.5072, 44266.694, 44851.2822, 45440.7862, 46038.0586, 46640.3164, 47241.064, 47846.155, 48454.7396, 49076.9168, 49692.542, 50317.4778, 50939.65, 51572.5596, 52210.2906, 52843.7396, 53481.3996, 54127.236, 54770.406, 55422.6598, 56078.7958, 56736.7174, 57397.6784, 58064.5784, 58730.308, 59404.9784, 60077.0864, 60751.9158, 61444.1386, 62115.817, 62808.7742, 63501.4774, 64187.5454, 64883.6622, 65582.7468, 66274.5318, 66976.9276, 67688.7764, 68402.138, 69109.6274, 69822.9706, 70543.6108,
- 71265.5202, 71983.3848, 72708.4656, 73433.384, 74158.4664, 74896.4868, 75620.9564, 76362.1434, 77098.3204, 77835.7662, 78582.6114, 79323.9902, 80067.8658, 80814.9246, 81567.0136, 82310.8536, 83061.9952, 83821.4096, 84580.8608, 85335.547, 86092.5802, 86851.6506, 87612.311, 88381.2016, 89146.3296, 89907.8974, 90676.846, 91451.4152, 92224.5518, 92995.8686, 93763.5066, 94551.2796, 95315.1944, 96096.1806, 96881.0918, 97665.679, 98442.68, 99229.3002, 100011.0994, 100790.6386, 101580.1564, 102377.7484, 103152.1392, 103944.2712, 104730.216, 105528.6336, 106324.9398, 107117.6706, 107890.3988, 108695.2266, 109485.238, 110294.7876, 111075.0958, 111878.0496, 112695.2864, 113464.5486, 114270.0474, 115068.608, 115884.3626, 116673.2588, 117483.3716, 118275.097, 119085.4092, 119879.2808, 120687.5868, 121499.9944, 122284.916, 123095.9254, 123912.5038, 124709.0454, 125503.7182, 126323.259, 127138.9412, 127943.8294, 128755.646, 129556.5354, 130375.3298, 131161.4734, 131971.1962,
- 132787.5458, 133588.1056, 134431.351, 135220.2906, 136023.398, 136846.6558, 137667.0004, 138463.663, 139283.7154, 140074.6146, 140901.3072, 141721.8548, 142543.2322, 143356.1096, 144173.7412, 144973.0948, 145794.3162, 146609.5714, 147420.003, 148237.9784, 149050.5696, 149854.761, 150663.1966, 151494.0754, 152313.1416, 153112.6902, 153935.7206, 154746.9262, 155559.547, 156401.9746, 157228.7036, 158008.7254, 158820.75, 159646.9184, 160470.4458, 161279.5348, 162093.3114, 162918.542, 163729.2842, },
- // precision 16
- { 47271, 48062.3584, 48862.7074, 49673.152, 50492.8416, 51322.9514, 52161.03, 53009.407, 53867.6348, 54734.206, 55610.5144, 56496.2096, 57390.795, 58297.268, 59210.6448, 60134.665, 61068.0248, 62010.4472, 62962.5204, 63923.5742, 64895.0194, 65876.4182, 66862.6136, 67862.6968, 68868.8908, 69882.8544, 70911.271, 71944.0924, 72990.0326, 74040.692, 75100.6336, 76174.7826, 77252.5998, 78340.2974, 79438.2572, 80545.4976, 81657.2796, 82784.6336, 83915.515, 85059.7362, 86205.9368, 87364.4424, 88530.3358, 89707.3744, 90885.9638, 92080.197, 93275.5738, 94479.391, 95695.918, 96919.2236, 98148.4602, 99382.3474, 100625.6974, 101878.0284, 103141.6278, 104409.4588, 105686.2882, 106967.5402, 108261.6032, 109548.1578, 110852.0728, 112162.231, 113479.0072, 114806.2626, 116137.9072, 117469.5048, 118813.5186, 120165.4876, 121516.2556, 122875.766, 124250.5444, 125621.2222, 127003.2352, 128387.848, 129775.2644, 131181.7776, 132577.3086, 133979.9458, 135394.1132, 136800.9078, 138233.217,
- 139668.5308, 141085.212, 142535.2122, 143969.0684, 145420.2872, 146878.1542, 148332.7572, 149800.3202, 151269.66, 152743.6104, 154213.0948, 155690.288, 157169.4246, 158672.1756, 160160.059, 161650.6854, 163145.7772, 164645.6726, 166159.1952, 167682.1578, 169177.3328, 170700.0118, 172228.8964, 173732.6664, 175265.5556, 176787.799, 178317.111, 179856.6914, 181400.865, 182943.4612, 184486.742, 186033.4698, 187583.7886, 189148.1868, 190688.4526, 192250.1926, 193810.9042, 195354.2972, 196938.7682, 198493.5898, 200079.2824, 201618.912, 203205.5492, 204765.5798, 206356.1124, 207929.3064, 209498.7196, 211086.229, 212675.1324, 214256.7892, 215826.2392, 217412.8474, 218995.6724, 220618.6038, 222207.1166, 223781.0364, 225387.4332, 227005.7928, 228590.4336, 230217.8738, 231805.1054, 233408.9, 234995.3432, 236601.4956, 238190.7904, 239817.2548, 241411.2832, 243002.4066, 244640.1884, 246255.3128, 247849.3508, 249479.9734, 251106.8822, 252705.027, 254332.9242, 255935.129,
- 257526.9014, 259154.772, 260777.625, 262390.253, 264004.4906, 265643.59, 267255.4076, 268873.426, 270470.7252, 272106.4804, 273722.4456, 275337.794, 276945.7038, 278592.9154, 280204.3726, 281841.1606, 283489.171, 285130.1716, 286735.3362, 288364.7164, 289961.1814, 291595.5524, 293285.683, 294899.6668, 296499.3434, 298128.0462, 299761.8946, 301394.2424, 302997.6748, 304615.1478, 306269.7724, 307886.114, 309543.1028, 311153.2862, 312782.8546, 314421.2008, 316033.2438, 317692.9636, 319305.2648, 320948.7406, 322566.3364, 324228.4224, 325847.1542, },
- // precision 17
- { 94542, 96125.811, 97728.019, 99348.558, 100987.9705, 102646.7565, 104324.5125, 106021.7435, 107736.7865, 109469.272, 111223.9465, 112995.219, 114787.432, 116593.152, 118422.71, 120267.2345, 122134.6765, 124020.937, 125927.2705, 127851.255, 129788.9485, 131751.016, 133726.8225, 135722.592, 137736.789, 139770.568, 141821.518, 143891.343, 145982.1415, 148095.387, 150207.526, 152355.649, 154515.6415, 156696.05, 158887.7575, 161098.159, 163329.852, 165569.053, 167837.4005, 170121.6165, 172420.4595, 174732.6265, 177062.77, 179412.502, 181774.035, 184151.939, 186551.6895, 188965.691, 191402.8095, 193857.949, 196305.0775, 198774.6715, 201271.2585, 203764.78, 206299.3695, 208818.1365, 211373.115, 213946.7465, 216532.076, 219105.541, 221714.5375, 224337.5135, 226977.5125, 229613.0655, 232270.2685, 234952.2065, 237645.3555, 240331.1925, 243034.517, 245756.0725, 248517.6865, 251232.737, 254011.3955, 256785.995, 259556.44, 262368.335, 265156.911, 267965.266, 270785.583, 273616.0495,
- 276487.4835, 279346.639, 282202.509, 285074.3885, 287942.2855, 290856.018, 293774.0345, 296678.5145, 299603.6355, 302552.6575, 305492.9785, 308466.8605, 311392.581, 314347.538, 317319.4295, 320285.9785, 323301.7325, 326298.3235, 329301.3105, 332301.987, 335309.791, 338370.762, 341382.923, 344431.1265, 347464.1545, 350507.28, 353619.2345, 356631.2005, 359685.203, 362776.7845, 365886.488, 368958.2255, 372060.6825, 375165.4335, 378237.935, 381328.311, 384430.5225, 387576.425, 390683.242, 393839.648, 396977.8425, 400101.9805, 403271.296, 406409.8425, 409529.5485, 412678.7, 415847.423, 419020.8035, 422157.081, 425337.749, 428479.6165, 431700.902, 434893.1915, 438049.582, 441210.5415, 444379.2545, 447577.356, 450741.931, 453959.548, 457137.0935, 460329.846, 463537.4815, 466732.3345, 469960.5615, 473164.681, 476347.6345, 479496.173, 482813.1645, 486025.6995, 489249.4885, 492460.1945, 495675.8805, 498908.0075, 502131.802, 505374.3855, 508550.9915, 511806.7305, 515026.776,
- 518217.0005, 521523.9855, 524705.9855, 527950.997, 531210.0265, 534472.497, 537750.7315, 540926.922, 544207.094, 547429.4345, 550666.3745, 553975.3475, 557150.7185, 560399.6165, 563662.697, 566916.7395, 570146.1215, 573447.425, 576689.6245, 579874.5745, 583202.337, 586503.0255, 589715.635, 592910.161, 596214.3885, 599488.035, 602740.92, 605983.0685, 609248.67, 612491.3605, 615787.912, 619107.5245, 622307.9555, 625577.333, 628840.4385, 632085.2155, 635317.6135, 638691.7195, 641887.467, 645139.9405, 648441.546, 651666.252, 654941.845, },
- // precision 18
- { 189084, 192250.913, 195456.774, 198696.946, 201977.762, 205294.444, 208651.754, 212042.099, 215472.269, 218941.91, 222443.912, 225996.845, 229568.199, 233193.568, 236844.457, 240543.233, 244279.475, 248044.27, 251854.588, 255693.2, 259583.619, 263494.621, 267445.385, 271454.061, 275468.769, 279549.456, 283646.446, 287788.198, 291966.099, 296181.164, 300431.469, 304718.618, 309024.004, 313393.508, 317760.803, 322209.731, 326675.061, 331160.627, 335654.47, 340241.442, 344841.833, 349467.132, 354130.629, 358819.432, 363574.626, 368296.587, 373118.482, 377914.93, 382782.301, 387680.669, 392601.981, 397544.323, 402529.115, 407546.018, 412593.658, 417638.657, 422762.865, 427886.169, 433017.167, 438213.273, 443441.254, 448692.421, 453937.533, 459239.049, 464529.569, 469910.083, 475274.03, 480684.473, 486070.26, 491515.237, 496995.651, 502476.617, 507973.609, 513497.19, 519083.233, 524726.509, 530305.505, 535945.728, 541584.404, 547274.055, 552967.236, 558667.862, 564360.216,
- 570128.148, 575965.08, 581701.952, 587532.523, 593361.144, 599246.128, 605033.418, 610958.779, 616837.117, 622772.818, 628672.04, 634675.369, 640574.831, 646585.739, 652574.547, 658611.217, 664642.684, 670713.914, 676737.681, 682797.313, 688837.897, 694917.874, 701009.882, 707173.648, 713257.254, 719415.392, 725636.761, 731710.697, 737906.209, 744103.074, 750313.39, 756504.185, 762712.579, 768876.985, 775167.859, 781359, 787615.959, 793863.597, 800245.477, 806464.582, 812785.294, 819005.925, 825403.057, 831676.197, 837936.284, 844266.968, 850642.711, 856959.756, 863322.774, 869699.931, 876102.478, 882355.787, 888694.463, 895159.952, 901536.143, 907872.631, 914293.672, 920615.14, 927130.974, 933409.404, 939922.178, 946331.47, 952745.93, 959209.264, 965590.224, 972077.284, 978501.961, 984953.19, 991413.271, 997817.479, 1004222.658, 1010725.676, 1017177.138, 1023612.529, 1030098.236, 1036493.719, 1043112.207, 1049537.036, 1056008.096, 1062476.184, 1068942.337,
- 1075524.95, 1081932.864, 1088426.025, 1094776.005, 1101327.448, 1107901.673, 1114423.639, 1120884.602, 1127324.923, 1133794.24, 1140328.886, 1146849.376, 1153346.682, 1159836.502, 1166478.703, 1172953.304, 1179391.502, 1185950.982, 1192544.052, 1198913.41, 1205430.994, 1212015.525, 1218674.042, 1225121.683, 1231551.101, 1238126.379, 1244673.795, 1251260.649, 1257697.86, 1264320.983, 1270736.319, 1277274.694, 1283804.95, 1290211.514, 1296858.568, 1303455.691, } };
-
- public static final double[][] biasData = {
- // precision 4
- { 10, 9.717, 9.207, 8.7896, 8.2882, 7.8204, 7.3772, 6.9342, 6.5202, 6.161, 5.7722, 5.4636, 5.0396, 4.6766, 4.3566, 4.0454, 3.7936, 3.4856, 3.2666, 2.9946, 2.766, 2.4692, 2.3638, 2.0764, 1.7864, 1.7602, 1.4814, 1.433, 1.2926, 1.0664, 0.999600000000001, 0.7956, 0.5366, 0.589399999999998, 0.573799999999999, 0.269799999999996, 0.368200000000002, 0.0544000000000011, 0.234200000000001, 0.0108000000000033, -0.203400000000002, -0.0701999999999998, -0.129600000000003, -0.364199999999997, -0.480600000000003, -0.226999999999997, -0.322800000000001, -0.382599999999996, -0.511200000000002, -0.669600000000003, -0.749400000000001, -0.500399999999999, -0.617600000000003, -0.6922, -0.601599999999998, -0.416200000000003, -0.338200000000001, -0.782600000000002, -0.648600000000002, -0.919800000000002, -0.851799999999997, -0.962400000000002, -0.6402, -1.1922, -1.0256, -1.086, -1.21899999999999, -0.819400000000002, -0.940600000000003, -1.1554, -1.2072, -1.1752, -1.16759999999999,
- -1.14019999999999, -1.3754, -1.29859999999999, -1.607, -1.3292, -1.7606, },
- // precision 5
- { 22, 21.1194, 20.8208, 20.2318, 19.77, 19.2436, 18.7774, 18.2848, 17.8224, 17.3742, 16.9336, 16.503, 16.0494, 15.6292, 15.2124, 14.798, 14.367, 13.9728, 13.5944, 13.217, 12.8438, 12.3696, 12.0956, 11.7044, 11.324, 11.0668, 10.6698, 10.3644, 10.049, 9.6918, 9.4146, 9.082, 8.687, 8.5398, 8.2462, 7.857, 7.6606, 7.4168, 7.1248, 6.9222, 6.6804, 6.447, 6.3454, 5.9594, 5.7636, 5.5776, 5.331, 5.19, 4.9676, 4.7564, 4.5314, 4.4442, 4.3708, 3.9774, 3.9624, 3.8796, 3.755, 3.472, 3.2076, 3.1024, 2.8908, 2.7338, 2.7728, 2.629, 2.413, 2.3266, 2.1524, 2.2642, 2.1806, 2.0566, 1.9192, 1.7598, 1.3516, 1.5802, 1.43859999999999, 1.49160000000001, 1.1524, 1.1892, 0.841399999999993, 0.879800000000003, 0.837599999999995, 0.469800000000006, 0.765600000000006, 0.331000000000003, 0.591399999999993, 0.601200000000006, 0.701599999999999, 0.558199999999999, 0.339399999999998, 0.354399999999998, 0.491200000000006, 0.308000000000007, 0.355199999999996, -0.0254000000000048, 0.205200000000005,
- -0.272999999999996, 0.132199999999997, 0.394400000000005, -0.241200000000006, 0.242000000000004, 0.191400000000002, 0.253799999999998, -0.122399999999999, -0.370800000000003, 0.193200000000004, -0.0848000000000013, 0.0867999999999967, -0.327200000000005, -0.285600000000002, 0.311400000000006, -0.128399999999999, -0.754999999999995, -0.209199999999996, -0.293599999999998, -0.364000000000004, -0.253600000000006, -0.821200000000005, -0.253600000000006, -0.510400000000004, -0.383399999999995, -0.491799999999998, -0.220200000000006, -0.0972000000000008, -0.557400000000001, -0.114599999999996, -0.295000000000002, -0.534800000000004, 0.346399999999988, -0.65379999999999, 0.0398000000000138, 0.0341999999999985, -0.995800000000003, -0.523400000000009, -0.489000000000004, -0.274799999999999, -0.574999999999989, -0.482799999999997, 0.0571999999999946, -0.330600000000004, -0.628800000000012, -0.140199999999993, -0.540600000000012, -0.445999999999998, -0.599400000000003,
- -0.262599999999992, 0.163399999999996, -0.100599999999986, -0.39500000000001, -1.06960000000001, -0.836399999999998, -0.753199999999993, -0.412399999999991, -0.790400000000005, -0.29679999999999, -0.28540000000001, -0.193000000000012, -0.0772000000000048, -0.962799999999987, -0.414800000000014, },
- // precision 6
- { 45, 44.1902, 43.271, 42.8358, 41.8142, 41.2854, 40.317, 39.354, 38.8924, 37.9436, 37.4596, 36.5262, 35.6248, 35.1574, 34.2822, 33.837, 32.9636, 32.074, 31.7042, 30.7976, 30.4772, 29.6564, 28.7942, 28.5004, 27.686, 27.291, 26.5672, 25.8556, 25.4982, 24.8204, 24.4252, 23.7744, 23.0786, 22.8344, 22.0294, 21.8098, 21.0794, 20.5732, 20.1878, 19.5648, 19.2902, 18.6784, 18.3352, 17.8946, 17.3712, 17.0852, 16.499, 16.2686, 15.6844, 15.2234, 14.9732, 14.3356, 14.2286, 13.7262, 13.3284, 13.1048, 12.5962, 12.3562, 12.1272, 11.4184, 11.4974, 11.0822, 10.856, 10.48, 10.2834, 10.0208, 9.637, 9.51739999999999, 9.05759999999999, 8.74760000000001, 8.42700000000001, 8.1326, 8.2372, 8.2788, 7.6776, 7.79259999999999, 7.1952, 6.9564, 6.6454, 6.87, 6.5428, 6.19999999999999, 6.02940000000001, 5.62780000000001, 5.6782, 5.792, 5.35159999999999, 5.28319999999999, 5.0394, 5.07480000000001, 4.49119999999999, 4.84899999999999, 4.696, 4.54040000000001, 4.07300000000001, 4.37139999999999, 3.7216,
- 3.7328, 3.42080000000001, 3.41839999999999, 3.94239999999999, 3.27719999999999, 3.411, 3.13079999999999, 2.76900000000001, 2.92580000000001, 2.68279999999999, 2.75020000000001, 2.70599999999999, 2.3886, 3.01859999999999, 2.45179999999999, 2.92699999999999, 2.41720000000001, 2.41139999999999, 2.03299999999999, 2.51240000000001, 2.5564, 2.60079999999999, 2.41720000000001, 1.80439999999999, 1.99700000000001, 2.45480000000001, 1.8948, 2.2346, 2.30860000000001, 2.15479999999999, 1.88419999999999, 1.6508, 0.677199999999999, 1.72540000000001, 1.4752, 1.72280000000001, 1.66139999999999, 1.16759999999999, 1.79300000000001, 1.00059999999999, 0.905200000000008, 0.659999999999997, 1.55879999999999, 1.1636, 0.688199999999995, 0.712600000000009, 0.450199999999995, 1.1978, 0.975599999999986, 0.165400000000005, 1.727, 1.19739999999999, -0.252600000000001, 1.13460000000001, 1.3048, 1.19479999999999, 0.313400000000001, 0.878999999999991, 1.12039999999999, 0.853000000000009,
- 1.67920000000001, 0.856999999999999, 0.448599999999999, 1.2362, 0.953399999999988, 1.02859999999998, 0.563199999999995, 0.663000000000011, 0.723000000000013, 0.756599999999992, 0.256599999999992, -0.837600000000009, 0.620000000000005, 0.821599999999989, 0.216600000000028, 0.205600000000004, 0.220199999999977, 0.372599999999977, 0.334400000000016, 0.928400000000011, 0.972800000000007, 0.192400000000021, 0.487199999999973, -0.413000000000011, 0.807000000000016, 0.120600000000024, 0.769000000000005, 0.870799999999974, 0.66500000000002, 0.118200000000002, 0.401200000000017, 0.635199999999998, 0.135400000000004, 0.175599999999974, 1.16059999999999, 0.34620000000001, 0.521400000000028, -0.586599999999976, -1.16480000000001, 0.968399999999974, 0.836999999999989, 0.779600000000016, 0.985799999999983, },
- // precision 7
- { 91, 89.4934, 87.9758, 86.4574, 84.9718, 83.4954, 81.5302, 80.0756, 78.6374, 77.1782, 75.7888, 73.9522, 72.592, 71.2532, 69.9086, 68.5938, 66.9474, 65.6796, 64.4394, 63.2176, 61.9768, 60.4214, 59.2528, 58.0102, 56.8658, 55.7278, 54.3044, 53.1316, 52.093, 51.0032, 49.9092, 48.6306, 47.5294, 46.5756, 45.6508, 44.662, 43.552, 42.3724, 41.617, 40.5754, 39.7872, 38.8444, 37.7988, 36.8606, 36.2118, 35.3566, 34.4476, 33.5882, 32.6816, 32.0824, 31.0258, 30.6048, 29.4436, 28.7274, 27.957, 27.147, 26.4364, 25.7592, 25.3386, 24.781, 23.8028, 23.656, 22.6544, 21.996, 21.4718, 21.1544, 20.6098, 19.5956, 19.0616, 18.5758, 18.4878, 17.5244, 17.2146, 16.724, 15.8722, 15.5198, 15.0414, 14.941, 14.9048, 13.87, 13.4304, 13.028, 12.4708, 12.37, 12.0624, 11.4668, 11.5532, 11.4352, 11.2564, 10.2744, 10.2118, 9.74720000000002, 10.1456, 9.2928, 8.75040000000001, 8.55279999999999, 8.97899999999998, 8.21019999999999, 8.18340000000001, 7.3494, 7.32499999999999, 7.66140000000001, 6.90300000000002,
- 7.25439999999998, 6.9042, 7.21499999999997, 6.28640000000001, 6.08139999999997, 6.6764, 6.30099999999999, 5.13900000000001, 5.65800000000002, 5.17320000000001, 4.59019999999998, 4.9538, 5.08280000000002, 4.92200000000003, 4.99020000000002, 4.7328, 5.4538, 4.11360000000002, 4.22340000000003, 4.08780000000002, 3.70800000000003, 4.15559999999999, 4.18520000000001, 3.63720000000001, 3.68220000000002, 3.77960000000002, 3.6078, 2.49160000000001, 3.13099999999997, 2.5376, 3.19880000000001, 3.21100000000001, 2.4502, 3.52820000000003, 2.91199999999998, 3.04480000000001, 2.7432, 2.85239999999999, 2.79880000000003, 2.78579999999999, 1.88679999999999, 2.98860000000002, 2.50639999999999, 1.91239999999999, 2.66160000000002, 2.46820000000002, 1.58199999999999, 1.30399999999997, 2.27379999999999, 2.68939999999998, 1.32900000000001, 3.10599999999999, 1.69080000000002, 2.13740000000001, 2.53219999999999, 1.88479999999998, 1.33240000000001, 1.45119999999997, 1.17899999999997,
- 2.44119999999998, 1.60659999999996, 2.16700000000003, 0.77940000000001, 2.37900000000002, 2.06700000000001, 1.46000000000004, 2.91160000000002, 1.69200000000001, 0.954600000000028, 2.49300000000005, 2.2722, 1.33500000000004, 2.44899999999996, 1.20140000000004, 3.07380000000001, 2.09739999999999, 2.85640000000001, 2.29960000000005, 2.40899999999999, 1.97040000000004, 0.809799999999996, 1.65279999999996, 2.59979999999996, 0.95799999999997, 2.06799999999998, 2.32780000000002, 4.20159999999998, 1.96320000000003, 1.86400000000003, 1.42999999999995, 3.77940000000001, 1.27200000000005, 1.86440000000005, 2.20600000000002, 3.21900000000005, 1.5154, 2.61019999999996, },
- // precision 8
- { 183.2152, 180.2454, 177.2096, 173.6652, 170.6312, 167.6822, 164.249, 161.3296, 158.0038, 155.2074, 152.4612, 149.27, 146.5178, 143.4412, 140.8032, 138.1634, 135.1688, 132.6074, 129.6946, 127.2664, 124.8228, 122.0432, 119.6824, 116.9464, 114.6268, 112.2626, 109.8376, 107.4034, 104.8956, 102.8522, 100.7638, 98.3552, 96.3556, 93.7526, 91.9292, 89.8954, 87.8198, 85.7668, 83.298, 81.6688, 79.9466, 77.9746, 76.1672, 74.3474, 72.3028, 70.8912, 69.114, 67.4646, 65.9744, 64.4092, 62.6022, 60.843, 59.5684, 58.1652, 56.5426, 55.4152, 53.5388, 52.3592, 51.1366, 49.486, 48.3918, 46.5076, 45.509, 44.3834, 43.3498, 42.0668, 40.7346, 40.1228, 38.4528, 37.7, 36.644, 36.0518, 34.5774, 33.9068, 32.432, 32.1666, 30.434, 29.6644, 28.4894, 27.6312, 26.3804, 26.292, 25.5496000000001, 25.0234, 24.8206, 22.6146, 22.4188, 22.117, 20.6762, 20.6576, 19.7864, 19.509, 18.5334, 17.9204, 17.772, 16.2924, 16.8654, 15.1836, 15.745, 15.1316, 15.0386, 14.0136, 13.6342, 12.6196, 12.1866, 12.4281999999999,
- 11.3324, 10.4794000000001, 11.5038, 10.129, 9.52800000000002, 10.3203999999999, 9.46299999999997, 9.79280000000006, 9.12300000000005, 8.74180000000001, 9.2192, 7.51020000000005, 7.60659999999996, 7.01840000000004, 7.22239999999999, 7.40139999999997, 6.76179999999999, 7.14359999999999, 5.65060000000005, 5.63779999999997, 5.76599999999996, 6.75139999999999, 5.57759999999996, 3.73220000000003, 5.8048, 5.63019999999995, 4.93359999999996, 3.47979999999995, 4.33879999999999, 3.98940000000005, 3.81960000000004, 3.31359999999995, 3.23080000000004, 3.4588, 3.08159999999998, 3.4076, 3.00639999999999, 2.38779999999997, 2.61900000000003, 1.99800000000005, 3.34820000000002, 2.95060000000001, 0.990999999999985, 2.11440000000005, 2.20299999999997, 2.82219999999995, 2.73239999999998, 2.7826, 3.76660000000004, 2.26480000000004, 2.31280000000004, 2.40819999999997, 2.75360000000001, 3.33759999999995, 2.71559999999999, 1.7478000000001, 1.42920000000004, 2.39300000000003,
- 2.22779999999989, 2.34339999999997, 0.87259999999992, 3.88400000000001, 1.80600000000004, 1.91759999999999, 1.16779999999994, 1.50320000000011, 2.52500000000009, 0.226400000000012, 2.31500000000005, 0.930000000000064, 1.25199999999995, 2.14959999999996, 0.0407999999999902, 2.5447999999999, 1.32960000000003, 0.197400000000016, 2.52620000000002, 3.33279999999991, -1.34300000000007, 0.422199999999975, 0.917200000000093, 1.12920000000008, 1.46060000000011, 1.45779999999991, 2.8728000000001, 3.33359999999993, -1.34079999999994, 1.57680000000005, 0.363000000000056, 1.40740000000005, 0.656600000000026, 0.801400000000058, -0.454600000000028, 1.51919999999996, },
- // precision 9
- { 368, 361.8294, 355.2452, 348.6698, 342.1464, 336.2024, 329.8782, 323.6598, 317.462, 311.2826, 305.7102, 299.7416, 293.9366, 288.1046, 282.285, 277.0668, 271.306, 265.8448, 260.301, 254.9886, 250.2422, 244.8138, 239.7074, 234.7428, 229.8402, 225.1664, 220.3534, 215.594, 210.6886, 205.7876, 201.65, 197.228, 192.8036, 188.1666, 184.0818, 180.0824, 176.2574, 172.302, 168.1644, 164.0056, 160.3802, 156.7192, 152.5234, 149.2084, 145.831, 142.485, 139.1112, 135.4764, 131.76, 129.3368, 126.5538, 122.5058, 119.2646, 116.5902, 113.3818, 110.8998, 107.9532, 105.2062, 102.2798, 99.4728, 96.9582, 94.3292, 92.171, 89.7809999999999, 87.5716, 84.7048, 82.5322, 79.875, 78.3972, 75.3464, 73.7274, 71.2834, 70.1444, 68.4263999999999, 66.0166, 64.018, 62.0437999999999, 60.3399999999999, 58.6856, 57.9836, 55.0311999999999, 54.6769999999999, 52.3188, 51.4846, 49.4423999999999, 47.739, 46.1487999999999, 44.9202, 43.4059999999999, 42.5342000000001, 41.2834, 38.8954000000001, 38.3286000000001,
- 36.2146, 36.6684, 35.9946, 33.123, 33.4338, 31.7378000000001, 29.076, 28.9692, 27.4964, 27.0998, 25.9864, 26.7754, 24.3208, 23.4838, 22.7388000000001, 24.0758000000001, 21.9097999999999, 20.9728, 19.9228000000001, 19.9292, 16.617, 17.05, 18.2996000000001, 15.6128000000001, 15.7392, 14.5174, 13.6322, 12.2583999999999, 13.3766000000001, 11.423, 13.1232, 9.51639999999998, 10.5938000000001, 9.59719999999993, 8.12220000000002, 9.76739999999995, 7.50440000000003, 7.56999999999994, 6.70440000000008, 6.41419999999994, 6.71019999999999, 5.60940000000005, 4.65219999999999, 6.84099999999989, 3.4072000000001, 3.97859999999991, 3.32760000000007, 5.52160000000003, 3.31860000000006, 2.06940000000009, 4.35400000000004, 1.57500000000005, 0.280799999999999, 2.12879999999996, -0.214799999999968, -0.0378000000000611, -0.658200000000079, 0.654800000000023, -0.0697999999999865, 0.858400000000074, -2.52700000000004, -2.1751999999999, -3.35539999999992, -1.04019999999991,
- -0.651000000000067, -2.14439999999991, -1.96659999999997, -3.97939999999994, -0.604400000000169, -3.08260000000018, -3.39159999999993, -5.29640000000018, -5.38920000000007, -5.08759999999984, -4.69900000000007, -5.23720000000003, -3.15779999999995, -4.97879999999986, -4.89899999999989, -7.48880000000008, -5.94799999999987, -5.68060000000014, -6.67180000000008, -4.70499999999993, -7.27779999999984, -4.6579999999999, -4.4362000000001, -4.32139999999981, -5.18859999999995, -6.66879999999992, -6.48399999999992, -5.1260000000002, -4.4032000000002, -6.13500000000022, -5.80819999999994, -4.16719999999987, -4.15039999999999, -7.45600000000013, -7.24080000000004, -9.83179999999993, -5.80420000000004, -8.6561999999999, -6.99940000000015, -10.5473999999999, -7.34139999999979, -6.80999999999995, -6.29719999999998, -6.23199999999997, },
- // precision 10
- { 737.1256, 724.4234, 711.1064, 698.4732, 685.4636, 673.0644, 660.488, 647.9654, 636.0832, 623.7864, 612.1992, 600.2176, 588.5228, 577.1716, 565.7752, 554.899, 543.6126, 532.6492, 521.9474, 511.5214, 501.1064, 490.6364, 480.2468, 470.4588, 460.3832, 451.0584, 440.8606, 431.3868, 422.5062, 413.1862, 404.463, 395.339, 386.1936, 378.1292, 369.1854, 361.2908, 353.3324, 344.8518, 337.5204, 329.4854, 321.9318, 314.552, 306.4658, 299.4256, 292.849, 286.152, 278.8956, 271.8792, 265.118, 258.62, 252.5132, 245.9322, 239.7726, 233.6086, 227.5332, 222.5918, 216.4294, 210.7662, 205.4106, 199.7338, 194.9012, 188.4486, 183.1556, 178.6338, 173.7312, 169.6264, 163.9526, 159.8742, 155.8326, 151.1966, 147.5594, 143.07, 140.037, 134.1804, 131.071, 127.4884, 124.0848, 120.2944, 117.333, 112.9626, 110.2902, 107.0814, 103.0334, 99.4832000000001, 96.3899999999999, 93.7202000000002, 90.1714000000002, 87.2357999999999, 85.9346, 82.8910000000001, 80.0264000000002, 78.3834000000002,
- 75.1543999999999, 73.8683999999998, 70.9895999999999, 69.4367999999999, 64.8701999999998, 65.0408000000002, 61.6738, 59.5207999999998, 57.0158000000001, 54.2302, 53.0962, 50.4985999999999, 52.2588000000001, 47.3914, 45.6244000000002, 42.8377999999998, 43.0072, 40.6516000000001, 40.2453999999998, 35.2136, 36.4546, 33.7849999999999, 33.2294000000002, 32.4679999999998, 30.8670000000002, 28.6507999999999, 28.9099999999999, 27.5983999999999, 26.1619999999998, 24.5563999999999, 23.2328000000002, 21.9484000000002, 21.5902000000001, 21.3346000000001, 17.7031999999999, 20.6111999999998, 19.5545999999999, 15.7375999999999, 17.0720000000001, 16.9517999999998, 15.326, 13.1817999999998, 14.6925999999999, 13.0859999999998, 13.2754, 10.8697999999999, 11.248, 7.3768, 4.72339999999986, 7.97899999999981, 8.7503999999999, 7.68119999999999, 9.7199999999998, 7.73919999999998, 5.6224000000002, 7.44560000000001, 6.6601999999998, 5.9058, 4.00199999999995, 4.51699999999983,
- 4.68240000000014, 3.86220000000003, 5.13639999999987, 5.98500000000013, 2.47719999999981, 2.61999999999989, 1.62800000000016, 4.65000000000009, 0.225599999999758, 0.831000000000131, -0.359400000000278, 1.27599999999984, -2.92559999999958, -0.0303999999996449, 2.37079999999969, -2.0033999999996, 0.804600000000391, 0.30199999999968, 1.1247999999996, -2.6880000000001, 0.0321999999996478, -1.18099999999959, -3.9402, -1.47940000000017, -0.188400000000001, -2.10720000000038, -2.04159999999956, -3.12880000000041, -4.16160000000036, -0.612799999999879, -3.48719999999958, -8.17900000000009, -5.37780000000021, -4.01379999999972, -5.58259999999973, -5.73719999999958, -7.66799999999967, -5.69520000000011, -1.1247999999996, -5.58520000000044, -8.04560000000038, -4.64840000000004, -11.6468000000004, -7.97519999999986, -5.78300000000036, -7.67420000000038, -10.6328000000003, -9.81720000000041, },
- // precision 11
- { 1476, 1449.6014, 1423.5802, 1397.7942, 1372.3042, 1347.2062, 1321.8402, 1297.2292, 1272.9462, 1248.9926, 1225.3026, 1201.4252, 1178.0578, 1155.6092, 1132.626, 1110.5568, 1088.527, 1066.5154, 1045.1874, 1024.3878, 1003.37, 982.1972, 962.5728, 942.1012, 922.9668, 903.292, 884.0772, 864.8578, 846.6562, 828.041, 809.714, 792.3112, 775.1806, 757.9854, 740.656, 724.346, 707.5154, 691.8378, 675.7448, 659.6722, 645.5722, 630.1462, 614.4124, 600.8728, 585.898, 572.408, 558.4926, 544.4938, 531.6776, 517.282, 505.7704, 493.1012, 480.7388, 467.6876, 456.1872, 445.5048, 433.0214, 420.806, 411.409, 400.4144, 389.4294, 379.2286, 369.651, 360.6156, 350.337, 342.083, 332.1538, 322.5094, 315.01, 305.6686, 298.1678, 287.8116, 280.9978, 271.9204, 265.3286, 257.5706, 249.6014, 242.544, 235.5976, 229.583, 220.9438, 214.672, 208.2786, 201.8628, 195.1834, 191.505, 186.1816, 178.5188, 172.2294, 167.8908, 161.0194, 158.052, 151.4588, 148.1596, 143.4344, 138.5238, 133.13, 127.6374, 124.8162,
- 118.7894, 117.3984, 114.6078, 109.0858, 105.1036, 103.6258, 98.6018000000004, 95.7618000000002, 93.5821999999998, 88.5900000000001, 86.9992000000002, 82.8800000000001, 80.4539999999997, 74.6981999999998, 74.3644000000004, 73.2914000000001, 65.5709999999999, 66.9232000000002, 65.1913999999997, 62.5882000000001, 61.5702000000001, 55.7035999999998, 56.1764000000003, 52.7596000000003, 53.0302000000001, 49.0609999999997, 48.4694, 44.933, 46.0474000000004, 44.7165999999997, 41.9416000000001, 39.9207999999999, 35.6328000000003, 35.5276000000003, 33.1934000000001, 33.2371999999996, 33.3864000000003, 33.9228000000003, 30.2371999999996, 29.1373999999996, 25.2272000000003, 24.2942000000003, 19.8338000000003, 18.9005999999999, 23.0907999999999, 21.8544000000002, 19.5176000000001, 15.4147999999996, 16.9314000000004, 18.6737999999996, 12.9877999999999, 14.3688000000002, 12.0447999999997, 15.5219999999999, 12.5299999999997, 14.5940000000001, 14.3131999999996, 9.45499999999993,
- 12.9441999999999, 3.91139999999996, 13.1373999999996, 5.44720000000052, 9.82779999999912, 7.87279999999919, 3.67760000000089, 5.46980000000076, 5.55099999999948, 5.65979999999945, 3.89439999999922, 3.1275999999998, 5.65140000000065, 6.3062000000009, 3.90799999999945, 1.87060000000019, 5.17020000000048, 2.46680000000015, 0.770000000000437, -3.72340000000077, 1.16400000000067, 8.05340000000069, 0.135399999999208, 2.15940000000046, 0.766999999999825, 1.0594000000001, 3.15500000000065, -0.287399999999252, 2.37219999999979, -2.86620000000039, -1.63199999999961, -2.22979999999916, -0.15519999999924, -1.46039999999994, -0.262199999999211, -2.34460000000036, -2.8078000000005, -3.22179999999935, -5.60159999999996, -8.42200000000048, -9.43740000000071, 0.161799999999857, -10.4755999999998, -10.0823999999993, },
- // precision 12
- { 2953, 2900.4782, 2848.3568, 2796.3666, 2745.324, 2694.9598, 2644.648, 2595.539, 2546.1474, 2498.2576, 2450.8376, 2403.6076, 2357.451, 2311.38, 2266.4104, 2221.5638, 2176.9676, 2134.193, 2090.838, 2048.8548, 2007.018, 1966.1742, 1925.4482, 1885.1294, 1846.4776, 1807.4044, 1768.8724, 1731.3732, 1693.4304, 1657.5326, 1621.949, 1586.5532, 1551.7256, 1517.6182, 1483.5186, 1450.4528, 1417.865, 1385.7164, 1352.6828, 1322.6708, 1291.8312, 1260.9036, 1231.476, 1201.8652, 1173.6718, 1145.757, 1119.2072, 1092.2828, 1065.0434, 1038.6264, 1014.3192, 988.5746, 965.0816, 940.1176, 917.9796, 894.5576, 871.1858, 849.9144, 827.1142, 805.0818, 783.9664, 763.9096, 742.0816, 724.3962, 706.3454, 688.018, 667.4214, 650.3106, 633.0686, 613.8094, 597.818, 581.4248, 563.834, 547.363, 531.5066, 520.455400000001, 505.583199999999, 488.366, 476.480799999999, 459.7682, 450.0522, 434.328799999999, 423.952799999999, 408.727000000001, 399.079400000001, 387.252200000001, 373.987999999999,
- 360.852000000001, 351.6394, 339.642, 330.902400000001, 322.661599999999, 311.662200000001, 301.3254, 291.7484, 279.939200000001, 276.7508, 263.215200000001, 254.811400000001, 245.5494, 242.306399999999, 234.8734, 223.787200000001, 217.7156, 212.0196, 200.793, 195.9748, 189.0702, 182.449199999999, 177.2772, 170.2336, 164.741, 158.613600000001, 155.311, 147.5964, 142.837, 137.3724, 132.0162, 130.0424, 121.9804, 120.451800000001, 114.8968, 111.585999999999, 105.933199999999, 101.705, 98.5141999999996, 95.0488000000005, 89.7880000000005, 91.4750000000004, 83.7764000000006, 80.9698000000008, 72.8574000000008, 73.1615999999995, 67.5838000000003, 62.6263999999992, 63.2638000000006, 66.0977999999996, 52.0843999999997, 58.9956000000002, 47.0912000000008, 46.4956000000002, 48.4383999999991, 47.1082000000006, 43.2392, 37.2759999999998, 40.0283999999992, 35.1864000000005, 35.8595999999998, 32.0998, 28.027, 23.6694000000007, 33.8266000000003, 26.3736000000008,
- 27.2008000000005, 21.3245999999999, 26.4115999999995, 23.4521999999997, 19.5013999999992, 19.8513999999996, 10.7492000000002, 18.6424000000006, 13.1265999999996, 18.2436000000016, 6.71860000000015, 3.39459999999963, 6.33759999999893, 7.76719999999841, 0.813999999998487, 3.82819999999992, 0.826199999999517, 8.07440000000133, -1.59080000000176, 5.01780000000144, 0.455399999998917, -0.24199999999837, 0.174800000000687, -9.07640000000174, -4.20160000000033, -3.77520000000004, -4.75179999999818, -5.3724000000002, -8.90680000000066, -6.10239999999976, -5.74120000000039, -9.95339999999851, -3.86339999999836, -13.7304000000004, -16.2710000000006, -7.51359999999841, -3.30679999999847, -13.1339999999982, -10.0551999999989, -6.72019999999975, -8.59660000000076, -10.9307999999983, -1.8775999999998, -4.82259999999951, -13.7788, -21.6470000000008, -10.6735999999983, -15.7799999999988, },
- // precision 13
- { 5907.5052, 5802.2672, 5697.347, 5593.5794, 5491.2622, 5390.5514, 5290.3376, 5191.6952, 5093.5988, 4997.3552, 4902.5972, 4808.3082, 4715.5646, 4624.109, 4533.8216, 4444.4344, 4356.3802, 4269.2962, 4183.3784, 4098.292, 4014.79, 3932.4574, 3850.6036, 3771.2712, 3691.7708, 3615.099, 3538.1858, 3463.4746, 3388.8496, 3315.6794, 3244.5448, 3173.7516, 3103.3106, 3033.6094, 2966.5642, 2900.794, 2833.7256, 2769.81, 2707.3196, 2644.0778, 2583.9916, 2523.4662, 2464.124, 2406.073, 2347.0362, 2292.1006, 2238.1716, 2182.7514, 2128.4884, 2077.1314, 2025.037, 1975.3756, 1928.933, 1879.311, 1831.0006, 1783.2144, 1738.3096, 1694.5144, 1649.024, 1606.847, 1564.7528, 1525.3168, 1482.5372, 1443.9668, 1406.5074, 1365.867, 1329.2186, 1295.4186, 1257.9716, 1225.339, 1193.2972, 1156.3578, 1125.8686, 1091.187, 1061.4094, 1029.4188, 1000.9126, 972.3272, 944.004199999999, 915.7592, 889.965, 862.834200000001, 840.4254, 812.598399999999, 785.924200000001, 763.050999999999, 741.793799999999, 721.466,
- 699.040799999999, 677.997200000002, 649.866999999998, 634.911800000002, 609.8694, 591.981599999999, 570.2922, 557.129199999999, 538.3858, 521.872599999999, 502.951400000002, 495.776399999999, 475.171399999999, 459.751, 439.995200000001, 426.708999999999, 413.7016, 402.3868, 387.262599999998, 372.0524, 357.050999999999, 342.5098, 334.849200000001, 322.529399999999, 311.613799999999, 295.848000000002, 289.273000000001, 274.093000000001, 263.329600000001, 251.389599999999, 245.7392, 231.9614, 229.7952, 217.155200000001, 208.9588, 199.016599999999, 190.839199999999, 180.6976, 176.272799999999, 166.976999999999, 162.5252, 151.196400000001, 149.386999999999, 133.981199999998, 130.0586, 130.164000000001, 122.053400000001, 110.7428, 108.1276, 106.232400000001, 100.381600000001, 98.7668000000012, 86.6440000000002, 79.9768000000004, 82.4722000000002, 68.7026000000005, 70.1186000000016, 71.9948000000004, 58.998599999999, 59.0492000000013, 56.9818000000014, 47.5338000000011,
- 42.9928, 51.1591999999982, 37.2740000000013, 42.7220000000016, 31.3734000000004, 26.8090000000011, 25.8934000000008, 26.5286000000015, 29.5442000000003, 19.3503999999994, 26.0760000000009, 17.9527999999991, 14.8419999999969, 10.4683999999979, 8.65899999999965, 9.86720000000059, 4.34139999999752, -0.907800000000861, -3.32080000000133, -0.936199999996461, -11.9916000000012, -8.87000000000262, -6.33099999999831, -11.3366000000024, -15.9207999999999, -9.34659999999712, -15.5034000000014, -19.2097999999969, -15.357799999998, -28.2235999999975, -30.6898000000001, -19.3271999999997, -25.6083999999973, -24.409599999999, -13.6385999999984, -33.4473999999973, -32.6949999999997, -28.9063999999998, -31.7483999999968, -32.2935999999972, -35.8329999999987, -47.620600000002, -39.0855999999985, -33.1434000000008, -46.1371999999974, -37.5892000000022, -46.8164000000033, -47.3142000000007, -60.2914000000019, -37.7575999999972, },
- // precision 14
- { 11816.475, 11605.0046, 11395.3792, 11188.7504, 10984.1814, 10782.0086, 10582.0072, 10384.503, 10189.178, 9996.2738, 9806.0344, 9617.9798, 9431.394, 9248.7784, 9067.6894, 8889.6824, 8712.9134, 8538.8624, 8368.4944, 8197.7956, 8031.8916, 7866.6316, 7703.733, 7544.5726, 7386.204, 7230.666, 7077.8516, 6926.7886, 6778.6902, 6631.9632, 6487.304, 6346.7486, 6206.4408, 6070.202, 5935.2576, 5799.924, 5671.0324, 5541.9788, 5414.6112, 5290.0274, 5166.723, 5047.6906, 4929.162, 4815.1406, 4699.127, 4588.5606, 4477.7394, 4369.4014, 4264.2728, 4155.9224, 4055.581, 3955.505, 3856.9618, 3761.3828, 3666.9702, 3575.7764, 3482.4132, 3395.0186, 3305.8852, 3221.415, 3138.6024, 3056.296, 2970.4494, 2896.1526, 2816.8008, 2740.2156, 2670.497, 2594.1458, 2527.111, 2460.8168, 2387.5114, 2322.9498, 2260.6752, 2194.2686, 2133.7792, 2074.767, 2015.204, 1959.4226, 1898.6502, 1850.006, 1792.849, 1741.4838, 1687.9778, 1638.1322, 1589.3266, 1543.1394, 1496.8266, 1447.8516, 1402.7354, 1361.9606,
- 1327.0692, 1285.4106, 1241.8112, 1201.6726, 1161.973, 1130.261, 1094.2036, 1048.2036, 1020.6436, 990.901400000002, 961.199800000002, 924.769800000002, 899.526400000002, 872.346400000002, 834.375, 810.432000000001, 780.659800000001, 756.013800000001, 733.479399999997, 707.923999999999, 673.858, 652.222399999999, 636.572399999997, 615.738599999997, 586.696400000001, 564.147199999999, 541.679600000003, 523.943599999999, 505.714599999999, 475.729599999999, 461.779600000002, 449.750800000002, 439.020799999998, 412.7886, 400.245600000002, 383.188199999997, 362.079599999997, 357.533799999997, 334.319000000003, 327.553399999997, 308.559399999998, 291.270199999999, 279.351999999999, 271.791400000002, 252.576999999997, 247.482400000001, 236.174800000001, 218.774599999997, 220.155200000001, 208.794399999999, 201.223599999998, 182.995600000002, 185.5268, 164.547400000003, 176.5962, 150.689599999998, 157.8004, 138.378799999999, 134.021200000003, 117.614399999999,
- 108.194000000003, 97.0696000000025, 89.6042000000016, 95.6030000000028, 84.7810000000027, 72.635000000002, 77.3482000000004, 59.4907999999996, 55.5875999999989, 50.7346000000034, 61.3916000000027, 50.9149999999936, 39.0384000000049, 58.9395999999979, 29.633600000001, 28.2032000000036, 26.0078000000067, 17.0387999999948, 9.22000000000116, 13.8387999999977, 8.07240000000456, 14.1549999999988, 15.3570000000036, 3.42660000000615, 6.24820000000182, -2.96940000000177, -8.79940000000352, -5.97860000000219, -14.4048000000039, -3.4143999999942, -13.0148000000045, -11.6977999999945, -25.7878000000055, -22.3185999999987, -24.409599999999, -31.9756000000052, -18.9722000000038, -22.8678000000073, -30.8972000000067, -32.3715999999986, -22.3907999999938, -43.6720000000059, -35.9038, -39.7492000000057, -54.1641999999993, -45.2749999999942, -42.2989999999991, -44.1089999999967, -64.3564000000042, -49.9551999999967, -42.6116000000038, },
- // precision 15
- { 23634.0036, 23210.8034, 22792.4744, 22379.1524, 21969.7928, 21565.326, 21165.3532, 20770.2806, 20379.9892, 19994.7098, 19613.318, 19236.799, 18865.4382, 18498.8244, 18136.5138, 17778.8668, 17426.2344, 17079.32, 16734.778, 16397.2418, 16063.3324, 15734.0232, 15409.731, 15088.728, 14772.9896, 14464.1402, 14157.5588, 13855.5958, 13559.3296, 13264.9096, 12978.326, 12692.0826, 12413.8816, 12137.3192, 11870.2326, 11602.5554, 11340.3142, 11079.613, 10829.5908, 10583.5466, 10334.0344, 10095.5072, 9859.694, 9625.2822, 9395.7862, 9174.0586, 8957.3164, 8738.064, 8524.155, 8313.7396, 8116.9168, 7913.542, 7718.4778, 7521.65, 7335.5596, 7154.2906, 6968.7396, 6786.3996, 6613.236, 6437.406, 6270.6598, 6107.7958, 5945.7174, 5787.6784, 5635.5784, 5482.308, 5337.9784, 5190.0864, 5045.9158, 4919.1386, 4771.817, 4645.7742, 4518.4774, 4385.5454, 4262.6622, 4142.74679999999, 4015.5318, 3897.9276, 3790.7764, 3685.13800000001, 3573.6274, 3467.9706, 3368.61079999999, 3271.5202, 3170.3848,
- 3076.4656, 2982.38400000001, 2888.4664, 2806.4868, 2711.9564, 2634.1434, 2551.3204, 2469.7662, 2396.61139999999, 2318.9902, 2243.8658, 2171.9246, 2105.01360000001, 2028.8536, 1960.9952, 1901.4096, 1841.86079999999, 1777.54700000001, 1714.5802, 1654.65059999999, 1596.311, 1546.2016, 1492.3296, 1433.8974, 1383.84600000001, 1339.4152, 1293.5518, 1245.8686, 1193.50659999999, 1162.27959999999, 1107.19439999999, 1069.18060000001, 1035.09179999999, 999.679000000004, 957.679999999993, 925.300199999998, 888.099400000006, 848.638600000006, 818.156400000007, 796.748399999997, 752.139200000005, 725.271200000003, 692.216, 671.633600000001, 647.939799999993, 621.670599999998, 575.398799999995, 561.226599999995, 532.237999999998, 521.787599999996, 483.095799999996, 467.049599999998, 465.286399999997, 415.548599999995, 401.047399999996, 380.607999999993, 377.362599999993, 347.258799999996, 338.371599999999, 310.096999999994, 301.409199999995, 276.280799999993, 265.586800000005,
- 258.994399999996, 223.915999999997, 215.925399999993, 213.503800000006, 191.045400000003, 166.718200000003, 166.259000000005, 162.941200000001, 148.829400000002, 141.645999999993, 123.535399999993, 122.329800000007, 89.473399999988, 80.1962000000058, 77.5457999999926, 59.1056000000099, 83.3509999999951, 52.2906000000075, 36.3979999999865, 40.6558000000077, 42.0003999999899, 19.6630000000005, 19.7153999999864, -8.38539999999921, -0.692799999989802, 0.854800000000978, 3.23219999999856, -3.89040000000386, -5.25880000001052, -24.9052000000083, -22.6837999999989, -26.4286000000138, -34.997000000003, -37.0216000000073, -43.430400000012, -58.2390000000014, -68.8034000000043, -56.9245999999985, -57.8583999999973, -77.3097999999882, -73.2793999999994, -81.0738000000129, -87.4530000000086, -65.0254000000132, -57.296399999992, -96.2746000000043, -103.25, -96.081600000005, -91.5542000000132, -102.465200000006, -107.688599999994, -101.458000000013, -109.715800000005, },
- // precision 16
- { 47270, 46423.3584, 45585.7074, 44757.152, 43938.8416, 43130.9514, 42330.03, 41540.407, 40759.6348, 39988.206, 39226.5144, 38473.2096, 37729.795, 36997.268, 36272.6448, 35558.665, 34853.0248, 34157.4472, 33470.5204, 32793.5742, 32127.0194, 31469.4182, 30817.6136, 30178.6968, 29546.8908, 28922.8544, 28312.271, 27707.0924, 27114.0326, 26526.692, 25948.6336, 25383.7826, 24823.5998, 24272.2974, 23732.2572, 23201.4976, 22674.2796, 22163.6336, 21656.515, 21161.7362, 20669.9368, 20189.4424, 19717.3358, 19256.3744, 18795.9638, 18352.197, 17908.5738, 17474.391, 17052.918, 16637.2236, 16228.4602, 15823.3474, 15428.6974, 15043.0284, 14667.6278, 14297.4588, 13935.2882, 13578.5402, 13234.6032, 12882.1578, 12548.0728, 12219.231, 11898.0072, 11587.2626, 11279.9072, 10973.5048, 10678.5186, 10392.4876, 10105.2556, 9825.766, 9562.5444, 9294.2222, 9038.2352, 8784.848, 8533.2644, 8301.7776, 8058.30859999999, 7822.94579999999, 7599.11319999999, 7366.90779999999, 7161.217, 6957.53080000001,
- 6736.212, 6548.21220000001, 6343.06839999999, 6156.28719999999, 5975.15419999999, 5791.75719999999, 5621.32019999999, 5451.66, 5287.61040000001, 5118.09479999999, 4957.288, 4798.4246, 4662.17559999999, 4512.05900000001, 4364.68539999999, 4220.77720000001, 4082.67259999999, 3957.19519999999, 3842.15779999999, 3699.3328, 3583.01180000001, 3473.8964, 3338.66639999999, 3233.55559999999, 3117.799, 3008.111, 2909.69140000001, 2814.86499999999, 2719.46119999999, 2624.742, 2532.46979999999, 2444.7886, 2370.1868, 2272.45259999999, 2196.19260000001, 2117.90419999999, 2023.2972, 1969.76819999999, 1885.58979999999, 1833.2824, 1733.91200000001, 1682.54920000001, 1604.57980000001, 1556.11240000001, 1491.3064, 1421.71960000001, 1371.22899999999, 1322.1324, 1264.7892, 1196.23920000001, 1143.8474, 1088.67240000001, 1073.60380000001, 1023.11660000001, 959.036400000012, 927.433199999999, 906.792799999996, 853.433599999989, 841.873800000001, 791.1054, 756.899999999994,
- 704.343200000003, 672.495599999995, 622.790399999998, 611.254799999995, 567.283200000005, 519.406599999988, 519.188400000014, 495.312800000014, 451.350799999986, 443.973399999988, 431.882199999993, 392.027000000002, 380.924200000009, 345.128999999986, 298.901400000002, 287.771999999997, 272.625, 247.253000000026, 222.490600000019, 223.590000000026, 196.407599999977, 176.425999999978, 134.725199999986, 132.4804, 110.445599999977, 86.7939999999944, 56.7038000000175, 64.915399999998, 38.3726000000024, 37.1606000000029, 46.170999999973, 49.1716000000015, 15.3362000000197, 6.71639999997569, -34.8185999999987, -39.4476000000141, 12.6830000000191, -12.3331999999937, -50.6565999999875, -59.9538000000175, -65.1054000000004, -70.7576000000117, -106.325200000021, -126.852200000023, -110.227599999984, -132.885999999999, -113.897200000007, -142.713800000027, -151.145399999979, -150.799200000009, -177.756200000003, -156.036399999983, -182.735199999996, -177.259399999981,
- -198.663600000029, -174.577600000019, -193.84580000001, },
- // precision 17
- { 94541, 92848.811, 91174.019, 89517.558, 87879.9705, 86262.7565, 84663.5125, 83083.7435, 81521.7865, 79977.272, 78455.9465, 76950.219, 75465.432, 73994.152, 72546.71, 71115.2345, 69705.6765, 68314.937, 66944.2705, 65591.255, 64252.9485, 62938.016, 61636.8225, 60355.592, 59092.789, 57850.568, 56624.518, 55417.343, 54231.1415, 53067.387, 51903.526, 50774.649, 49657.6415, 48561.05, 47475.7575, 46410.159, 45364.852, 44327.053, 43318.4005, 42325.6165, 41348.4595, 40383.6265, 39436.77, 38509.502, 37594.035, 36695.939, 35818.6895, 34955.691, 34115.8095, 33293.949, 32465.0775, 31657.6715, 30877.2585, 30093.78, 29351.3695, 28594.1365, 27872.115, 27168.7465, 26477.076, 25774.541, 25106.5375, 24452.5135, 23815.5125, 23174.0655, 22555.2685, 21960.2065, 21376.3555, 20785.1925, 20211.517, 19657.0725, 19141.6865, 18579.737, 18081.3955, 17578.995, 17073.44, 16608.335, 16119.911, 15651.266, 15194.583, 14749.0495, 14343.4835, 13925.639, 13504.509, 13099.3885, 12691.2855, 12328.018,
- 11969.0345, 11596.5145, 11245.6355, 10917.6575, 10580.9785, 10277.8605, 9926.58100000001, 9605.538, 9300.42950000003, 8989.97850000003, 8728.73249999998, 8448.3235, 8175.31050000002, 7898.98700000002, 7629.79100000003, 7413.76199999999, 7149.92300000001, 6921.12650000001, 6677.1545, 6443.28000000003, 6278.23450000002, 6014.20049999998, 5791.20299999998, 5605.78450000001, 5438.48800000001, 5234.2255, 5059.6825, 4887.43349999998, 4682.935, 4496.31099999999, 4322.52250000002, 4191.42499999999, 4021.24200000003, 3900.64799999999, 3762.84250000003, 3609.98050000001, 3502.29599999997, 3363.84250000003, 3206.54849999998, 3079.70000000001, 2971.42300000001, 2867.80349999998, 2727.08100000001, 2630.74900000001, 2496.6165, 2440.902, 2356.19150000002, 2235.58199999999, 2120.54149999999, 2012.25449999998, 1933.35600000003, 1820.93099999998, 1761.54800000001, 1663.09350000002, 1578.84600000002, 1509.48149999999, 1427.3345, 1379.56150000001, 1306.68099999998, 1212.63449999999,
- 1084.17300000001, 1124.16450000001, 1060.69949999999, 1007.48849999998, 941.194499999983, 879.880500000028, 836.007500000007, 782.802000000025, 748.385499999975, 647.991500000004, 626.730500000005, 570.776000000013, 484.000500000024, 513.98550000001, 418.985499999952, 386.996999999974, 370.026500000036, 355.496999999974, 356.731499999994, 255.92200000002, 259.094000000041, 205.434499999974, 165.374500000034, 197.347500000033, 95.718499999959, 67.6165000000037, 54.6970000000438, 31.7395000000251, -15.8784999999916, 8.42500000004657, -26.3754999999655, -118.425500000012, -66.6629999999423, -42.9745000000112, -107.364999999991, -189.839000000036, -162.611499999999, -164.964999999967, -189.079999999958, -223.931499999948, -235.329999999958, -269.639500000048, -249.087999999989, -206.475499999942, -283.04449999996, -290.667000000016, -304.561499999953, -336.784499999951, -380.386500000022, -283.280499999993, -364.533000000054, -389.059499999974, -364.454000000027,
- -415.748000000021, -417.155000000028, },
- // precision 18
- { 189083, 185696.913, 182348.774, 179035.946, 175762.762, 172526.444, 169329.754, 166166.099, 163043.269, 159958.91, 156907.912, 153906.845, 150924.199, 147996.568, 145093.457, 142239.233, 139421.475, 136632.27, 133889.588, 131174.2, 128511.619, 125868.621, 123265.385, 120721.061, 118181.769, 115709.456, 113252.446, 110840.198, 108465.099, 106126.164, 103823.469, 101556.618, 99308.004, 97124.508, 94937.803, 92833.731, 90745.061, 88677.627, 86617.47, 84650.442, 82697.833, 80769.132, 78879.629, 77014.432, 75215.626, 73384.587, 71652.482, 69895.93, 68209.301, 66553.669, 64921.981, 63310.323, 61742.115, 60205.018, 58698.658, 57190.657, 55760.865, 54331.169, 52908.167, 51550.273, 50225.254, 48922.421, 47614.533, 46362.049, 45098.569, 43926.083, 42736.03, 41593.473, 40425.26, 39316.237, 38243.651, 37170.617, 36114.609, 35084.19, 34117.233, 33206.509, 32231.505, 31318.728, 30403.404, 29540.0550000001, 28679.236, 27825.862, 26965.216, 26179.148, 25462.08, 24645.952, 23922.523,
- 23198.144, 22529.128, 21762.4179999999, 21134.779, 20459.117, 19840.818, 19187.04, 18636.3689999999, 17982.831, 17439.7389999999, 16874.547, 16358.2169999999, 15835.684, 15352.914, 14823.681, 14329.313, 13816.897, 13342.874, 12880.882, 12491.648, 12021.254, 11625.392, 11293.7610000001, 10813.697, 10456.209, 10099.074, 9755.39000000001, 9393.18500000006, 9047.57900000003, 8657.98499999999, 8395.85900000005, 8033, 7736.95900000003, 7430.59699999995, 7258.47699999996, 6924.58200000005, 6691.29399999999, 6357.92500000005, 6202.05700000003, 5921.19700000004, 5628.28399999999, 5404.96799999999, 5226.71100000001, 4990.75600000005, 4799.77399999998, 4622.93099999998, 4472.478, 4171.78700000001, 3957.46299999999, 3868.95200000005, 3691.14300000004, 3474.63100000005, 3341.67200000002, 3109.14000000001, 3071.97400000005, 2796.40399999998, 2756.17799999996, 2611.46999999997, 2471.93000000005, 2382.26399999997, 2209.22400000005, 2142.28399999999, 2013.96100000001,
- 1911.18999999994, 1818.27099999995, 1668.47900000005, 1519.65800000005, 1469.67599999998, 1367.13800000004, 1248.52899999998, 1181.23600000003, 1022.71900000004, 1088.20700000005, 959.03600000008, 876.095999999903, 791.183999999892, 703.337000000058, 731.949999999953, 586.86400000006, 526.024999999907, 323.004999999888, 320.448000000091, 340.672999999952, 309.638999999966, 216.601999999955, 102.922999999952, 19.2399999999907, -0.114000000059605, -32.6240000000689, -89.3179999999702, -153.497999999905, -64.2970000000205, -143.695999999996, -259.497999999905, -253.017999999924, -213.948000000091, -397.590000000084, -434.006000000052, -403.475000000093, -297.958000000101, -404.317000000039, -528.898999999976, -506.621000000043, -513.205000000075, -479.351000000024, -596.139999999898, -527.016999999993, -664.681000000099, -680.306000000099, -704.050000000047, -850.486000000034, -757.43200000003, -713.308999999892, } };
-
- public static boolean isLinearCounting(int precision, double estimate) {
- return estimate <= thresholdData[precision - 4];
- }
-
- public static boolean isBiasCorrection(int numberOfBuckets, double estimate) {
- return estimate <= 5 * numberOfBuckets;
- }
-
- public static double biasCorrection(int precision, double estimate) {
- double[] estimateVector = rawEstimateData[precision - 4];
- SortedMap<Double, Integer> estimateDistances = calcDistances(estimate, estimateVector);
- int[] nearestNeighbors = getNearestNeighbors(estimateDistances);
- double bias = getBias(precision, nearestNeighbors);
- return estimate - bias;
- }
-
- public static SortedMap<Double, Integer> calcDistances(double estimate, double[] estimateVector) {
- SortedMap<Double, Integer> distances = new TreeMap<Double, Integer>();
- int index = 0;
- for (double anEstimateVector : estimateVector) {
- distances.put(Math.pow(estimate - anEstimateVector, 2), index++);
- }
- return distances;
- }
-
- public static double getBias(int precision, int[] nearestNeighbors) {
- double[] biasVector = biasData[precision - 4];
- double biasTotal = 0.0d;
- for (int nearestNeighbor : nearestNeighbors) {
- biasTotal += biasVector[nearestNeighbor];
- }
- return biasTotal / (nearestNeighbors.length);
- }
-
- public static int[] getNearestNeighbors(SortedMap<Double, Integer> distanceMap) {
- int[] nearest = new int[6];
- int i = 0;
- for (Integer index : distanceMap.values()) {
- nearest[i++] = index;
- if (i >= 6) {
- break;
- }
- }
- return nearest;
- }
-}
diff --git a/common/src/main/java/com/kylinolap/common/persistence/AclEntity.java b/common/src/main/java/com/kylinolap/common/persistence/AclEntity.java
deleted file mode 100644
index 519f701f6a..0000000000
--- a/common/src/main/java/com/kylinolap/common/persistence/AclEntity.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.common.persistence;
-
-/**
- * @author xduo
- *
- */
-public interface AclEntity {
-
- public String getId();
-
-}
diff --git a/common/src/main/java/com/kylinolap/common/persistence/FileResourceStore.java b/common/src/main/java/com/kylinolap/common/persistence/FileResourceStore.java
deleted file mode 100644
index fcc031271c..0000000000
--- a/common/src/main/java/com/kylinolap/common/persistence/FileResourceStore.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.common.persistence;
-
-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-
-import org.apache.commons.io.IOUtils;
-
-import com.kylinolap.common.KylinConfig;
-
-public class FileResourceStore extends ResourceStore {
-
- File root;
-
- public FileResourceStore(KylinConfig kylinConfig) {
- super(kylinConfig);
- root = new File(kylinConfig.getMetadataUrl()).getAbsoluteFile();
- if (root.exists() == false)
- throw new IllegalArgumentException("File not exist by '" + kylinConfig.getMetadataUrl() + "': " + root.getAbsolutePath());
- }
-
- @Override
- protected ArrayList<String> listResourcesImpl(String resPath) throws IOException {
- String[] names = file(resPath).list();
- if (names == null) // not a directory
- return null;
-
- ArrayList<String> r = new ArrayList<String>(names.length);
- String prefix = resPath.endsWith("/") ? resPath : resPath + "/";
- for (String n : names) {
- r.add(prefix + n);
- }
- return r;
- }
-
- @Override
- protected boolean existsImpl(String resPath) throws IOException {
- File f = file(resPath);
- return f.exists() && f.isFile(); // directory is not considered a
- // resource
- }
-
- @Override
- protected InputStream getResourceImpl(String resPath) throws IOException {
- File f = file(resPath);
- if (f.exists() && f.isFile())
- return new FileInputStream(file(resPath));
- else
- return null;
- }
-
- @Override
- protected long getResourceTimestampImpl(String resPath) throws IOException {
- File f = file(resPath);
- return f.lastModified();
- }
-
- @Override
- protected void putResourceImpl(String resPath, InputStream content, long ts) throws IOException {
- File f = file(resPath);
- f.getParentFile().mkdirs();
- FileOutputStream out = new FileOutputStream(f);
- try {
- IOUtils.copy(content, out);
- } finally {
- IOUtils.closeQuietly(out);
- }
-
- f.setLastModified(ts);
- }
-
- @Override
- protected long checkAndPutResourceImpl(String resPath, byte[] content, long oldTS, long newTS) throws IOException, IllegalStateException {
- File f = file(resPath);
- if ((f.exists() && f.lastModified() != oldTS) || (f.exists() == false && oldTS != 0))
- throw new IllegalStateException("Overwriting conflict " + resPath + ", expect old TS " + oldTS + ", but found " + f.lastModified());
-
- putResourceImpl(resPath, new ByteArrayInputStream(content), newTS);
-
- // some FS lose precision on given time stamp
- return getResourceTimestamp(resPath);
- }
-
- @Override
- protected void deleteResourceImpl(String resPath) throws IOException {
- File f = file(resPath);
- f.delete();
- }
-
- @Override
- protected String getReadableResourcePathImpl(String resPath) {
- return file(resPath).toString();
- }
-
- private File file(String resPath) {
- if (resPath.equals("/"))
- return root;
- else
- return new File(root, resPath);
- }
-
-}
diff --git a/common/src/main/java/com/kylinolap/common/persistence/HBaseConnection.java b/common/src/main/java/com/kylinolap/common/persistence/HBaseConnection.java
deleted file mode 100644
index fe90cb53a0..0000000000
--- a/common/src/main/java/com/kylinolap/common/persistence/HBaseConnection.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.common.persistence;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.kylinolap.common.util.HadoopUtil;
-
-/**
- * @author yangli9
- *
- */
-public class HBaseConnection {
-
- private static final Logger logger = LoggerFactory.getLogger(HBaseConnection.class);
-
- private static final Map<String, Configuration> ConfigCache = new ConcurrentHashMap<String, Configuration>();
- private static final Map<String, HConnection> ConnPool = new ConcurrentHashMap<String, HConnection>();
-
- static {
- Runtime.getRuntime().addShutdownHook(new Thread() {
- @Override
- public void run() {
- for (HConnection conn : ConnPool.values()) {
- try {
- conn.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- }
- });
- }
-
- public static HConnection get(String url) {
- // find configuration
- Configuration conf = ConfigCache.get(url);
- if (conf == null) {
- conf = HadoopUtil.newHBaseConfiguration(url);
- ConfigCache.put(url, conf);
- }
-
- HConnection connection = ConnPool.get(url);
- try {
- // I don't use DCL since recreate a connection is not a big issue.
- if (connection == null) {
- connection = HConnectionManager.createConnection(conf);
- ConnPool.put(url, connection);
- }
- } catch (Throwable t) {
- throw new StorageException("Error when open connection " + url, t);
- }
-
- return connection;
- }
-
- public static void createHTableIfNeeded(String hbaseUrl, String tableName, String... families) throws IOException {
- createHTableIfNeeded(HBaseConnection.get(hbaseUrl), tableName, families);
- }
-
- public static void createHTableIfNeeded(HConnection conn, String tableName, String... families) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
-
- try {
- boolean tableExist = false;
- try {
- hbase.getTableDescriptor(TableName.valueOf(tableName));
- tableExist = true;
- } catch (TableNotFoundException e) {
- }
-
- if (tableExist) {
- logger.debug("HTable '" + tableName + "' already exists");
- return;
- }
-
- logger.debug("Creating HTable '" + tableName + "'");
-
- HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
-
- if (null != families && families.length > 0) {
- for (String family : families) {
- HColumnDescriptor fd = new HColumnDescriptor(family);
- fd.setInMemory(true); // metadata tables are best in memory
- desc.addFamily(fd);
- }
- }
- hbase.createTable(desc);
-
- logger.debug("HTable '" + tableName + "' created");
- } finally {
- hbase.close();
- }
- }
-}
diff --git a/common/src/main/java/com/kylinolap/common/persistence/HBaseResourceStore.java b/common/src/main/java/com/kylinolap/common/persistence/HBaseResourceStore.java
deleted file mode 100644
index 3e101afc9b..0000000000
--- a/common/src/main/java/com/kylinolap/common/persistence/HBaseResourceStore.java
+++ /dev/null
@@ -1,307 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.common.persistence;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
-import org.apache.hadoop.hbase.util.Bytes;
-
-import com.kylinolap.common.KylinConfig;
-import com.kylinolap.common.util.BytesUtil;
-import com.kylinolap.common.util.HadoopUtil;
-
-public class HBaseResourceStore extends ResourceStore {
-
- private static final String DEFAULT_TABLE_NAME = "kylin_metadata";
- private static final String FAMILY = "f";
- private static final byte[] B_FAMILY = Bytes.toBytes(FAMILY);
- private static final String COLUMN = "c";
- private static final byte[] B_COLUMN = Bytes.toBytes(COLUMN);
- private static final String COLUMN_TS = "t";
- private static final byte[] B_COLUMN_TS = Bytes.toBytes(COLUMN_TS);
-
- private static final Map<String, String> TABLE_SUFFIX_MAP = new LinkedHashMap<String, String>();
-
- static {
- TABLE_SUFFIX_MAP.put(CUBE_RESOURCE_ROOT + "/", "_cube");
- TABLE_SUFFIX_MAP.put(DICT_RESOURCE_ROOT + "/", "_dict");
- TABLE_SUFFIX_MAP.put("/invertedindex/", "_invertedindex");
- TABLE_SUFFIX_MAP.put(JOB_PATH_ROOT + "/", "_job");
- TABLE_SUFFIX_MAP.put(JOB_OUTPUT_PATH_ROOT + "/", "_job_output");
- TABLE_SUFFIX_MAP.put(PROJECT_RESOURCE_ROOT + "/", "_proj");
- TABLE_SUFFIX_MAP.put(SNAPSHOT_RESOURCE_ROOT + "/", "_table_snapshot");
- TABLE_SUFFIX_MAP.put("", ""); // DEFAULT CASE
- }
-
- final String tableNameBase;
- final String hbaseUrl;
-
- final Map<String, String> tableNameMap; // path prefix ==> HBase table name
-
- private HConnection getConnection() throws IOException {
- return HBaseConnection.get(hbaseUrl);
- }
-
- public HBaseResourceStore(KylinConfig kylinConfig) throws IOException {
- super(kylinConfig);
-
- String metadataUrl = kylinConfig.getMetadataUrl();
- // split TABLE@HBASE_URL
- int cut = metadataUrl.indexOf('@');
- tableNameBase = cut < 0 ? DEFAULT_TABLE_NAME : metadataUrl.substring(0, cut);
- hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1);
-
- tableNameMap = new LinkedHashMap<String, String>();
- for (Entry<String, String> entry : TABLE_SUFFIX_MAP.entrySet()) {
- String pathPrefix = entry.getKey();
- String tableName = tableNameBase + entry.getValue();
- tableNameMap.put(pathPrefix, tableName);
- createHTableIfNeeded(tableName);
- }
-
- }
-
- private void createHTableIfNeeded(String tableName) throws IOException {
- HBaseConnection.createHTableIfNeeded(getConnection(), tableName, FAMILY);
- }
-
- private String getTableName(String path) {
- for (Entry<String, String> entry : tableNameMap.entrySet()) {
- String pathPrefix = entry.getKey();
- if (path.startsWith(pathPrefix))
- return entry.getValue();
- }
- throw new IllegalStateException("failed to find HBase table for path -- " + path);
- }
-
- @Override
- protected ArrayList<String> listResourcesImpl(String resPath) throws IOException {
- assert resPath.startsWith("/");
- String lookForPrefix = resPath.endsWith("/") ? resPath : resPath + "/";
- byte[] startRow = Bytes.toBytes(lookForPrefix);
- byte[] endRow = Bytes.toBytes(lookForPrefix);
- endRow[endRow.length - 1]++;
-
- ArrayList<String> result = new ArrayList<String>();
-
- for (Entry<String, String> entry : tableNameMap.entrySet()) {
- String pathPrefix = entry.getKey();
- String tableName = entry.getValue();
-
- if ((pathPrefix.startsWith(lookForPrefix) || lookForPrefix.startsWith(pathPrefix)) == false)
- continue;
-
- HTableInterface table = getConnection().getTable(tableName);
-
- Scan scan = new Scan(startRow, endRow);
- scan.setFilter(new KeyOnlyFilter());
- try {
- ResultScanner scanner = table.getScanner(scan);
- for (Result r : scanner) {
- String path = Bytes.toString(r.getRow());
- assert path.startsWith(lookForPrefix);
- int cut = path.indexOf('/', lookForPrefix.length());
- String child = cut < 0 ? path : path.substring(0, cut);
- if (result.contains(child) == false)
- result.add(child);
- }
- } finally {
- IOUtils.closeQuietly(table);
- }
- }
- // return null to indicate not a folder
- return result.isEmpty() ? null : result;
- }
-
- @Override
- protected boolean existsImpl(String resPath) throws IOException {
- Result r = getByScan(resPath, null, null);
- return r != null;
- }
-
- @Override
- protected InputStream getResourceImpl(String resPath) throws IOException {
- Result r = getByScan(resPath, B_FAMILY, B_COLUMN);
- if (r == null)
- return null;
-
- byte[] value = r.getValue(B_FAMILY, B_COLUMN);
- if (value.length == 0) {
- Path redirectPath = bigCellHDFSPath(resPath);
- Configuration hconf = HadoopUtil.getDefaultConfiguration();
- FileSystem fileSystem = FileSystem.get(hconf);
-
- return fileSystem.open(redirectPath);
- } else {
- return new ByteArrayInputStream(value);
- }
- }
-
- @Override
- protected long getResourceTimestampImpl(String resPath) throws IOException {
- Result r = getByScan(resPath, B_FAMILY, B_COLUMN_TS);
- if (r == null)
- return 0;
- else
- return Bytes.toLong(r.getValue(B_FAMILY, B_COLUMN_TS));
- }
-
- @Override
- protected void putResourceImpl(String resPath, InputStream content, long ts) throws IOException {
- ByteArrayOutputStream bout = new ByteArrayOutputStream();
- IOUtils.copy(content, bout);
- bout.close();
-
- HTableInterface table = getConnection().getTable(getTableName(resPath));
- try {
- byte[] row = Bytes.toBytes(resPath);
- Put put = buildPut(resPath, ts, row, bout.toByteArray(), table);
-
- table.put(put);
- table.flushCommits();
- } finally {
- IOUtils.closeQuietly(table);
- }
- }
-
- @Override
- protected long checkAndPutResourceImpl(String resPath, byte[] content, long oldTS, long newTS) throws IOException, IllegalStateException {
- HTableInterface table = getConnection().getTable(getTableName(resPath));
- try {
- byte[] row = Bytes.toBytes(resPath);
- byte[] bOldTS = oldTS == 0 ? null : Bytes.toBytes(oldTS);
- Put put = buildPut(resPath, newTS, row, content, table);
-
- boolean ok = table.checkAndPut(row, B_FAMILY, B_COLUMN_TS, bOldTS, put);
- if (!ok)
- throw new IllegalStateException("Overwriting conflict " + resPath + ", expect old TS " + oldTS + ", but it is " + getResourceTimestamp(resPath));
-
- table.flushCommits();
-
- return newTS;
- } finally {
- IOUtils.closeQuietly(table);
- }
- }
-
- @Override
- protected void deleteResourceImpl(String resPath) throws IOException {
- HTableInterface table = getConnection().getTable(getTableName(resPath));
- try {
- Delete del = new Delete(Bytes.toBytes(resPath));
- table.delete(del);
- table.flushCommits();
- } finally {
- IOUtils.closeQuietly(table);
- }
- }
-
- @Override
- protected String getReadableResourcePathImpl(String resPath) {
- return tableNameBase + "(key='" + resPath + "')@" + kylinConfig.getMetadataUrl();
- }
-
- private Result getByScan(String path, byte[] family, byte[] column) throws IOException {
- byte[] startRow = Bytes.toBytes(path);
- byte[] endRow = plusZero(startRow);
-
- Scan scan = new Scan(startRow, endRow);
- if (family == null || column == null) {
- scan.setFilter(new KeyOnlyFilter());
- } else {
- scan.addColumn(family, column);
- }
-
- HTableInterface table = getConnection().getTable(getTableName(path));
- try {
- ResultScanner scanner = table.getScanner(scan);
- Result result = null;
- for (Result r : scanner) {
- result = r;
- }
- return result == null || result.isEmpty() ? null : result;
- } finally {
- IOUtils.closeQuietly(table);
- }
- }
-
- private byte[] plusZero(byte[] startRow) {
- byte[] endRow = Arrays.copyOf(startRow, startRow.length + 1);
- endRow[endRow.length - 1] = 0;
- return endRow;
- }
-
- private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, HTableInterface table) throws IOException {
- Path redirectPath = bigCellHDFSPath(resPath);
- Configuration hconf = HadoopUtil.getDefaultConfiguration();
- FileSystem fileSystem = FileSystem.get(hconf);
-
- if (fileSystem.exists(redirectPath)) {
- fileSystem.delete(redirectPath, true);
- }
-
- FSDataOutputStream out = fileSystem.create(redirectPath);
-
- try {
- out.write(largeColumn);
- } finally {
- IOUtils.closeQuietly(out);
- }
-
- return redirectPath;
- }
-
- public Path bigCellHDFSPath(String resPath) {
- String hdfsWorkingDirectory = this.kylinConfig.getHdfsWorkingDirectory();
- Path redirectPath = new Path(hdfsWorkingDirectory, "resources" + resPath);
- return redirectPath;
- }
-
- private Put buildPut(String resPath, long ts, byte[] row, byte[] content, HTableInterface table) throws IOException {
- int kvSizeLimit = this.kylinConfig.getHBaseKeyValueSize();
- if (content.length > kvSizeLimit) {
- writeLargeCellToHdfs(resPath, content, table);
- content = BytesUtil.EMPTY_BYTE_ARRAY;
- }
-
- Put put = new Put(row);
- put.add(B_FAMILY, B_COLUMN, content);
- put.add(B_FAMILY, B_COLUMN_TS, Bytes.toBytes(ts));
-
- return put;
- }
-}
diff --git a/common/src/main/java/com/kylinolap/common/persistence/JsonSerializer.java b/common/src/main/java/com/kylinolap/common/persistence/JsonSerializer.java
deleted file mode 100644
index 1471d09b7c..0000000000
--- a/common/src/main/java/com/kylinolap/common/persistence/JsonSerializer.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.common.persistence;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-import com.kylinolap.common.util.JsonUtil;
-
-/**
- * @author yangli9
- */
-public class JsonSerializer<T extends RootPersistentEntity> implements Serializer<T> {
-
- Class<T> clz;
-
- public JsonSerializer(Class<T> clz) {
- this.clz = clz;
- }
-
- @Override
- public T deserialize(DataInputStream in) throws IOException {
- return JsonUtil.readValue(in, clz);
- }
-
- @Override
- public void serialize(T obj, DataOutputStream out) throws IOException {
- JsonUtil.writeValueIndent(out, obj);
- }
-}
\ No newline at end of file
diff --git a/common/src/main/java/com/kylinolap/common/persistence/ResourceStore.java b/common/src/main/java/com/kylinolap/common/persistence/ResourceStore.java
deleted file mode 100644
index 36abfddca5..0000000000
--- a/common/src/main/java/com/kylinolap/common/persistence/ResourceStore.java
+++ /dev/null
@@ -1,256 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.common.persistence;
-
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.commons.io.IOUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Lists;
-import com.kylinolap.common.KylinConfig;
-
-abstract public class ResourceStore {
-
- private static final Logger logger = LoggerFactory.getLogger(ResourceStore.class);
-
- public static final String CUBE_RESOURCE_ROOT = "/cube";
- public static final String CUBE_DESC_RESOURCE_ROOT = "/cube_desc";
- public static final String DICT_RESOURCE_ROOT = "/dict";
- public static final String IIDESC_RESOURCE_ROOT = "/invertedindex_desc";
- public static final String JOB_PATH_ROOT = "/job";
- public static final String JOB_OUTPUT_PATH_ROOT = "/job_output";
- public static final String PROJECT_RESOURCE_ROOT = "/project";
- public static final String SNAPSHOT_RESOURCE_ROOT = "/table_snapshot";
- public static final String TABLE_EXD_RESOURCE_ROOT = "/table_exd";
- public static final String TABLE_RESOURCE_ROOT = "/table";
-
- private static ConcurrentHashMap<KylinConfig, ResourceStore> CACHE = new ConcurrentHashMap<KylinConfig, ResourceStore>();
-
- public static final ArrayList<Class<? extends ResourceStore>> knownImpl = new ArrayList<Class<? extends ResourceStore>>();
-
- static {
- knownImpl.add(HBaseResourceStore.class);
- knownImpl.add(FileResourceStore.class);
- }
-
- public static ResourceStore getStore(KylinConfig kylinConfig) {
- ResourceStore r = CACHE.get(kylinConfig);
- List<Throwable> es = new ArrayList<Throwable>();
- if (r == null) {
- logger.info("Using metadata url " + kylinConfig.getMetadataUrl() + " for resource store");
- for (Class<? extends ResourceStore> cls : knownImpl) {
-
- try {
- r = cls.getConstructor(KylinConfig.class).newInstance(kylinConfig);
- } catch (Exception e) {
- es.add(e);
- } catch (NoClassDefFoundError er) {
- // may throw NoClassDefFoundError
- es.add(er);
- }
- if (r != null) {
- break;
- }
- }
- if (r == null) {
- for (Throwable exceptionOrError : es) {
- logger.error("Create new store instance failed ", exceptionOrError);
- }
- throw new IllegalArgumentException("Failed to find metadata store by url: " + kylinConfig.getMetadataUrl());
- }
-
- CACHE.put(kylinConfig, r);
- }
- return r;
- }
-
- // ============================================================================
-
- KylinConfig kylinConfig;
-
- ResourceStore(KylinConfig kylinConfig) {
- this.kylinConfig = kylinConfig;
- }
-
- /**
- * return a list of child resources & folders under given path, return null
- * if given path is not a folder
- */
- final public ArrayList<String> listResources(String resPath) throws IOException {
- resPath = norm(resPath);
- return listResourcesImpl(resPath);
- }
-
- abstract protected ArrayList<String> listResourcesImpl(String resPath) throws IOException;
-
- /**
- * return true if a resource exists, return false in case of folder or
- * non-exist
- */
- final public boolean exists(String resPath) throws IOException {
- return existsImpl(norm(resPath));
- }
-
- abstract protected boolean existsImpl(String resPath) throws IOException;
-
- /**
- * read a resource, return null in case of not found
- */
- final public <T extends RootPersistentEntity> T getResource(String resPath, Class<T> clz, Serializer<T> serializer) throws IOException {
- resPath = norm(resPath);
- InputStream in = getResourceImpl(resPath);
- if (in == null)
- return null;
-
- DataInputStream din = new DataInputStream(in);
- try {
- T r = serializer.deserialize(din);
- r.setLastModified(getResourceTimestamp(resPath));
- return r;
- } finally {
- IOUtils.closeQuietly(din);
- IOUtils.closeQuietly(in);
- }
- }
-
- final public InputStream getResource(String resPath) throws IOException {
- return getResourceImpl(norm(resPath));
- }
-
- abstract protected InputStream getResourceImpl(String resPath) throws IOException;
-
- final public long getResourceTimestamp(String resPath) throws IOException {
- return getResourceTimestampImpl(norm(resPath));
- }
-
- abstract protected long getResourceTimestampImpl(String resPath) throws IOException;
-
- /**
- * overwrite a resource without write conflict check
- */
- final public void putResource(String resPath, InputStream content, long ts) throws IOException {
- resPath = norm(resPath);
- logger.debug("Saving resource " + resPath + " (Store " + kylinConfig.getMetadataUrl() + ")");
- putResourceImpl(resPath, content, ts);
- }
-
- abstract protected void putResourceImpl(String resPath, InputStream content, long ts) throws IOException;
-
- /**
- * check & set, overwrite a resource
- */
- final public <T extends RootPersistentEntity> void putResource(String resPath, T obj, Serializer<T> serializer) throws IOException {
- resPath = norm(resPath);
- logger.debug("Saving resource " + resPath + " (Store " + kylinConfig.getMetadataUrl() + ")");
-
- long oldTS = obj.getLastModified();
- long newTS = System.currentTimeMillis();
- obj.setLastModified(newTS);
-
- try {
- ByteArrayOutputStream buf = new ByteArrayOutputStream();
- DataOutputStream dout = new DataOutputStream(buf);
- serializer.serialize(obj, dout);
- dout.close();
- buf.close();
-
- newTS = checkAndPutResourceImpl(resPath, buf.toByteArray(), oldTS, newTS);
- obj.setLastModified(newTS); // update again the confirmed TS
-
- } catch (IOException e) {
- obj.setLastModified(oldTS); // roll back TS when write fail
- throw e;
- } catch (RuntimeException e) {
- obj.setLastModified(oldTS); // roll back TS when write fail
- throw e;
- }
- }
-
- /**
- * checks old timestamp when overwriting existing
- */
- abstract protected long checkAndPutResourceImpl(String resPath, byte[] content, long oldTS, long newTS) throws IOException, IllegalStateException;
-
- /**
- * delete a resource, does nothing on a folder
- */
- final public void deleteResource(String resPath) throws IOException {
- logger.debug("Deleting resource " + resPath + " (Store " + kylinConfig.getMetadataUrl() + ")");
- deleteResourceImpl(norm(resPath));
- }
-
- abstract protected void deleteResourceImpl(String resPath) throws IOException;
-
- /**
- * get a readable string of a resource path
- */
- final public String getReadableResourcePath(String resPath) {
- return getReadableResourcePathImpl(norm(resPath));
- }
-
- abstract protected String getReadableResourcePathImpl(String resPath);
-
- private String norm(String resPath) {
- resPath = resPath.trim();
- while (resPath.startsWith("//"))
- resPath = resPath.substring(1);
- while (resPath.endsWith("/"))
- resPath = resPath.substring(0, resPath.length() - 1);
- if (resPath.startsWith("/") == false)
- resPath = "/" + resPath;
- return resPath;
- }
-
- // ============================================================================
-
- public static interface Visitor {
- void visit(String path) throws IOException;
- }
-
- public void scanRecursively(String path, Visitor visitor) throws IOException {
- ArrayList<String> children = listResources(path);
- if (children != null) {
- for (String child : children)
- scanRecursively(child, visitor);
- return;
- }
-
- if (exists(path))
- visitor.visit(path);
- }
-
- public List<String> collectResourceRecursively(String root, final String suffix) throws IOException {
- final ArrayList<String> collector = Lists.newArrayList();
- scanRecursively(root, new Visitor() {
- @Override
- public void visit(String path) {
- if (path.endsWith(suffix))
- collector.add(path);
- }
- });
- return collector;
- }
-
-}
diff --git a/common/src/main/java/com/kylinolap/common/persistence/ResourceTool.java b/common/src/main/java/com/kylinolap/common/persistence/ResourceTool.java
deleted file mode 100644
index 66783ca230..0000000000
--- a/common/src/main/java/com/kylinolap/common/persistence/ResourceTool.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.common.persistence;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-
-import com.kylinolap.common.KylinConfig;
-import com.kylinolap.common.util.StringUtil;
-
-public class ResourceTool {
-
- private static String[] excludes = null;
-
- public static void main(String[] args) throws IOException {
- args = StringUtil.filterSystemArgs(args);
-
- if (args.length == 0) {
- System.out.println("Usage: MetadataTool reset METADATA_URI");
- System.out.println("Usage: MetadataTool copy METADATA_URI_SRC METADATA_URI_DST");
- System.out.println("Usage: MetadataTool list METADATA_URI PATH");
- return;
- }
-
- String exclude = System.getProperty("exclude");
- if (exclude != null) {
- excludes = exclude.split("\\s*,\\s*");
- }
-
- String cmd = args[0];
- if (cmd.equals("reset"))
- reset(args.length == 1 ? KylinConfig.getInstanceFromEnv() : KylinConfig.createInstanceFromUri(args[1]));
- else if (cmd.equals("copy"))
- copy(args[1], args[2]);
- else if (cmd.equals("list"))
- list(args[1], args[2]);
- else if (cmd.equals("download"))
- copy(KylinConfig.getInstanceFromEnv(), KylinConfig.createInstanceFromUri(args[1]));
- else if (cmd.equals("upload"))
- copy(KylinConfig.createInstanceFromUri(args[1]), KylinConfig.getInstanceFromEnv());
- else if (cmd.equals("remove"))
- remove(KylinConfig.getInstanceFromEnv(), args[1]);
- else
- System.out.println("Unknown cmd: " + cmd);
- }
-
- public static void list(KylinConfig config, String path) throws IOException {
- ResourceStore store = ResourceStore.getStore(config);
- ArrayList<String> result = store.listResources(path);
- System.out.println("" + result);
- }
-
- private static void list(String metadataUri, String path) throws IOException {
- KylinConfig config = KylinConfig.createInstanceFromUri(metadataUri);
- list(config, path);
- }
-
- public static void copy(KylinConfig srcConfig, KylinConfig dstConfig) throws IOException {
-
- ResourceStore src = ResourceStore.getStore(srcConfig);
- ResourceStore dst = ResourceStore.getStore(dstConfig);
- copyR(src, dst, "/");
- }
-
- private static void copy(String srcUri, String dstUri) throws IOException {
-
- System.out.println("Copy from " + srcUri + " to " + dstUri);
-
- KylinConfig srcConfig = KylinConfig.createInstanceFromUri(srcUri);
- KylinConfig dstConfig = KylinConfig.createInstanceFromUri(dstUri);
- copy(srcConfig, dstConfig);
-
- }
-
- private static void copyR(ResourceStore src, ResourceStore dst, String path) throws IOException {
- ArrayList<String> children = src.listResources(path);
-
- // case of resource (not a folder)
- if (children == null) {
- if (matchExclude(path) == false) {
- InputStream content = src.getResource(path);
- long ts = src.getResourceTimestamp(path);
- if (content != null)
- dst.putResource(path, content, ts);
- else
- System.out.println("Null inputstream for " + path);
- }
- }
- // case of folder
- else {
- for (String child : children)
- copyR(src, dst, child);
- }
- }
-
- private static boolean matchExclude(String path) {
- if (excludes == null)
- return false;
- for (String exclude : excludes) {
- if (path.startsWith(exclude))
- return true;
- }
- return false;
- }
-
- public static void reset(KylinConfig config) throws IOException {
- ResourceStore store = ResourceStore.getStore(config);
- resetR(store, "/");
- }
-
- private static void resetR(ResourceStore store, String path) throws IOException {
- ArrayList<String> children = store.listResources(path);
- if (children == null) { // path is a resource (not a folder)
- if (matchExclude(path) == false) {
- store.deleteResource(path);
- }
- } else {
- for (String child : children)
- resetR(store, child);
- }
- }
-
- private static void remove(KylinConfig config, String path) throws IOException {
- ResourceStore store = ResourceStore.getStore(config);
- resetR(store, path);
- }
-}
diff --git a/common/src/main/java/com/kylinolap/common/persistence/RootPersistentEntity.java b/common/src/main/java/com/kylinolap/common/persistence/RootPersistentEntity.java
deleted file mode 100644
index d19d58321d..0000000000
--- a/common/src/main/java/com/kylinolap/common/persistence/RootPersistentEntity.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.common.persistence;
-
-import java.util.UUID;
-
-import org.apache.commons.lang.time.FastDateFormat;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- * Marks the root entity of JSON persistence. Unit of read, write, cache, and
- * refresh.
- *
- * - CubeInstance - CubeDesc - SourceTable - JobMeta - Dictionary (not JSON but
- * also top level persistence)
- *
- * @author yangli9
- */
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-abstract public class RootPersistentEntity implements AclEntity {
-
- static FastDateFormat format = FastDateFormat.getInstance("yyyy-MM-dd HH:mm:ss z");
-
- public static String formatTime(long millis) {
- return format.format(millis);
- }
-
- // ============================================================================
-
- @JsonProperty("uuid")
- protected String uuid;
-
- @JsonProperty("last_modified")
- protected long lastModified;
-
- public String getUuid() {
- return uuid;
- }
-
- public void setUuid(String uuid) {
- this.uuid = uuid;
- }
-
- public String getId() {
- return uuid;
- }
-
- public long getLastModified() {
- return lastModified;
- }
-
- public void setLastModified(long lastModified) {
- this.lastModified = lastModified;
- }
-
- public void updateRandomUuid() {
- setUuid(UUID.randomUUID().toString());
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + (int) (lastModified ^ (lastModified >>> 32));
- result = prime * result + ((uuid == null) ? 0 : uuid.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- RootPersistentEntity other = (RootPersistentEntity) obj;
- if (lastModified != other.lastModified)
- return false;
- if (uuid == null) {
- if (other.uuid != null)
- return false;
- } else if (!uuid.equals(other.uuid))
- return false;
- return true;
- }
-
-}
diff --git a/common/src/main/java/com/kylinolap/common/persistence/Serializer.java b/common/src/main/java/com/kylinolap/common/persistence/Serializer.java
deleted file mode 100644
index d0637457e3..0000000000
--- a/common/src/main/java/com/kylinolap/common/persistence/Serializer.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.common.persistence;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-/**
- * @author yangli9
- *
- */
-public interface Serializer<T extends RootPersistentEntity> {
-
- public void serialize(T obj, DataOutputStream out) throws IOException;
-
- public T deserialize(DataInputStream in) throws IOException;
-
-}
diff --git a/common/src/main/java/com/kylinolap/common/persistence/StorageException.java b/common/src/main/java/com/kylinolap/common/persistence/StorageException.java
deleted file mode 100644
index 62f12cd9e1..0000000000
--- a/common/src/main/java/com/kylinolap/common/persistence/StorageException.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.common.persistence;
-
-/**
- *
- * @author xjiang
- *
- */
-public class StorageException extends RuntimeException {
-
- private static final long serialVersionUID = -3748712888242406257L;
-
- public StorageException(String msg, Throwable t) {
- super(msg, t);
- }
-
- public StorageException(String msg) {
- super(msg);
- }
-
-}
diff --git a/common/src/main/java/com/kylinolap/common/restclient/AbstractRestCache.java b/common/src/main/java/com/kylinolap/common/restclient/AbstractRestCache.java
deleted file mode 100644
index 8bc2468bff..0000000000
--- a/common/src/main/java/com/kylinolap/common/restclient/AbstractRestCache.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.common.restclient;
-
-/**
- * @author xjiang
- *
- */
-public abstract class AbstractRestCache<K, V> {
-
- protected final Broadcaster.TYPE syncType;
-
- protected AbstractRestCache(Broadcaster.TYPE syncType) {
- this.syncType = syncType;
- }
-
- protected final void syncRemote(Object key, Broadcaster.EVENT syncAction) {
- String syncKey = (syncType == Broadcaster.TYPE.METADATA) ? "metadata" : key.toString();
- Broadcaster.queue(syncType.getType(), syncAction.getType(), syncKey);
- }
-
- public abstract void put(K key, V value);
-
- public abstract void putLocal(K key, V value);
-
- public abstract void remove(K key);
-
- public abstract void removeLocal(K key);
-
- public abstract void clear();
-
- public abstract int size();
-}
diff --git a/common/src/main/java/com/kylinolap/common/restclient/Broadcaster.java b/common/src/main/java/com/kylinolap/common/restclient/Broadcaster.java
deleted file mode 100644
index 60f096389c..0000000000
--- a/common/src/main/java/com/kylinolap/common/restclient/Broadcaster.java
+++ /dev/null
@@ -1,261 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.common.restclient;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-import java.util.Timer;
-import java.util.TimerTask;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.kylinolap.common.KylinConfig;
-
-/**
- * Broadcast kylin event out
- *
- * @author jianliu
- *
- */
-public class Broadcaster {
-
- private static final Logger logger = LoggerFactory.getLogger(Broadcaster.class);
-
- private static List<BroadcastEvent> broadcaseEvents = new ArrayList<BroadcastEvent>();
-
- static class BroadcasterHolder {
- static final Broadcaster INSTANCE = new Broadcaster();
- }
-
- private Broadcaster() {
- Timer timer = new Timer();
- TimerTask task = new TimerTask() {
- public void run() {
- Broadcaster.flush();
- }
- };
-
- timer.schedule(task, new Date(), 10 * 1000);
- }
-
- public Broadcaster getInstance() {
- return BroadcasterHolder.INSTANCE;
- }
-
- public static void queueSyncMetadata() {
- queue(TYPE.METADATA.getType(), EVENT.CREATE.getType(), "metadata");
- }
-
- /**
- * Broadcast the cubedesc event out
- *
- * @param action
- * event action
- */
- public static synchronized void queue(String type, String action, String key) {
- BroadcastEvent event = BroadcasterHolder.INSTANCE.new BroadcastEvent(type, action, key);
-
- if (!broadcaseEvents.contains(event)) {
- broadcaseEvents.add(event);
- }
- }
-
- public static synchronized void flush() {
- String[] nodes = KylinConfig.getInstanceFromEnv().getRestServers();
- if (nodes == null)
- return;
-
- for (BroadcastEvent event : broadcaseEvents) {
- for (String nodeUri : nodes) {
- logger.debug("Broadcast nodeUri: " + nodeUri + ", type: " + event.getType() + ", action: " + event.getAction() + ", name: " + event.getName());
- WipeCacheThread thread = BroadcasterHolder.INSTANCE.new WipeCacheThread(nodeUri, event.getType(), event.getAction(), event.getName());
- thread.start();
- }
- }
-
- broadcaseEvents.clear();
- }
-
- public static String genEventkey(String type, String action, String name) {
- String time = String.valueOf(System.currentTimeMillis());
- return time + "_" + type + "_" + action + "_" + name;
- }
-
- protected class WipeCacheThread extends Thread {
- private String nodeUri;
- private String type;
- private String action;
- private String name;
-
- public WipeCacheThread(String nodeUri, String type, String action, String name) {
- this.nodeUri = nodeUri;
- this.type = type;
- this.action = action;
- this.name = name;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.lang.Thread#run()
- */
- @Override
- public void run() {
- RestClient restClient = new RestClient(nodeUri);
- try {
- restClient.wipeCache(this.type, this.action, this.name);
- } catch (IOException e) {
- logger.warn("Thread failed during wipe cache at " + type + "." + action + "." + name + ", " + e.toString());
- }
- }
- }
-
- public enum EVENT {
- CREATE("create"), UPDATE("update"), DROP("drop");
- private String text;
-
- private EVENT(String text) {
- this.text = text;
- }
-
- public String getType() {
- return text;
- }
-
- public static EVENT getEvent(String event) {
- for (EVENT one : values()) {
- if (one.getType().equalsIgnoreCase(event)) {
- return one;
- }
- }
-
- return null;
- }
- }
-
- public enum TYPE {
- CUBE("cube"), METADATA("metadata"), PROJECT("project");
- private String text;
-
- private TYPE(String text) {
- this.text = text;
- }
-
- public String getType() {
- return text;
- }
-
- /**
- * @param type
- * @return
- */
- public static TYPE getType(String type) {
- for (TYPE one : values()) {
- if (one.getType().equalsIgnoreCase(type)) {
- return one;
- }
- }
-
- return null;
- }
- }
-
- public class BroadcastEvent {
- private String type;
- private String action;
- private String name;
-
- public BroadcastEvent(String type, String action, String name) {
- super();
- this.type = type;
- this.action = action;
- this.name = name;
- }
-
- public String getType() {
- return type;
- }
-
- public void setType(String type) {
- this.type = type;
- }
-
- public String getAction() {
- return action;
- }
-
- public void setAction(String action) {
- this.action = action;
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + getOuterType().hashCode();
- result = prime * result + ((action == null) ? 0 : action.hashCode());
- result = prime * result + ((name == null) ? 0 : name.hashCode());
- result = prime * result + ((type == null) ? 0 : type.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- BroadcastEvent other = (BroadcastEvent) obj;
- if (!getOuterType().equals(other.getOuterType()))
- return false;
- if (action == null) {
- if (other.action != null)
- return false;
- } else if (!action.equals(other.action))
- return false;
- if (name == null) {
- if (other.name != null)
- return false;
- } else if (!name.equals(other.name))
- return false;
- if (type == null) {
- if (other.type != null)
- return false;
- } else if (!type.equals(other.type))
- return false;
- return true;
- }
-
- private Broadcaster getOuterType() {
- return Broadcaster.this;
- }
-
- }
-}
diff --git a/common/src/main/java/com/kylinolap/common/restclient/MultiValueCache.java b/common/src/main/java/com/kylinolap/common/restclient/MultiValueCache.java
deleted file mode 100644
index 3ecaac61e1..0000000000
--- a/common/src/main/java/com/kylinolap/common/restclient/MultiValueCache.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.common.restclient;
-
-import java.util.Set;
-
-import com.google.common.collect.HashMultimap;
-
-/**
- * @author xjiang
- *
- */
-public class MultiValueCache<K, V> extends AbstractRestCache<K, V> {
-
- private final HashMultimap<K, V> innerCache;
-
- public MultiValueCache(Broadcaster.TYPE syncType) {
- super(syncType);
- innerCache = HashMultimap.create();
- }
-
- public void put(K key, V value) {
- Broadcaster.EVENT eventType = innerCache.containsKey(key) ? Broadcaster.EVENT.UPDATE : Broadcaster.EVENT.CREATE;
- synchronized (this) {
- innerCache.put(key, value);
- }
- syncRemote(key, eventType);
- }
-
- public void putLocal(K key, V value) {
- synchronized (this) {
- innerCache.put(key, value);
- }
- }
-
- public void remove(K key) {
- if (innerCache.containsKey(key)) {
- innerCache.removeAll(key);
- syncRemote(key, Broadcaster.EVENT.DROP);
- }
- }
-
- public void removeLocal(K key) {
- if (innerCache.containsKey(key)) {
- innerCache.removeAll(key);
- }
- }
-
- public void clear() {
- innerCache.clear();
- }
-
- public int size() {
- return innerCache.size();
- }
-
- public Set<V> get(K key) {
- return innerCache.get(key);
- }
-
- public Set<K> keySet() {
- return innerCache.keySet();
- }
-
- public boolean containsKey(Object key) {
- return innerCache.containsKey(key);
- }
-
- public boolean containsEntry(Object key, Object value) {
- return innerCache.containsEntry(key, value);
- }
-}
diff --git a/common/src/main/java/com/kylinolap/common/restclient/RestClient.java b/common/src/main/java/com/kylinolap/common/restclient/RestClient.java
deleted file mode 100644
index 5f1367fd44..0000000000
--- a/common/src/main/java/com/kylinolap/common/restclient/RestClient.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.common.restclient;
-
-import java.io.IOException;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.commons.httpclient.Credentials;
-import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.HttpException;
-import org.apache.commons.httpclient.HttpMethod;
-import org.apache.commons.httpclient.UsernamePasswordCredentials;
-import org.apache.commons.httpclient.auth.AuthScope;
-import org.apache.commons.httpclient.methods.GetMethod;
-import org.apache.commons.httpclient.methods.PutMethod;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.codehaus.jettison.json.JSONException;
-import org.codehaus.jettison.json.JSONObject;
-
-/**
- * @author yangli9
- */
-public class RestClient {
-
- String host;
- int port;
- String baseUrl;
- String userName;
- String password;
- HttpClient client;
-
- private static Pattern fullRestPattern = Pattern.compile("(?:([^:]+)[:]([^@]+)[@])?([^:]+)(?:[:](\\d+))?");
-
- public static boolean matchFullRestPattern(String uri) {
- Matcher m = fullRestPattern.matcher(uri);
- return m.matches();
- }
-
- /**
- * @param uri
- * "user:pwd@host:port"
- */
- public RestClient(String uri) {
- Matcher m = fullRestPattern.matcher(uri);
- if (!m.matches())
- throw new IllegalArgumentException("URI: " + uri + " -- does not match pattern " + fullRestPattern);
-
- String user = m.group(1);
- String pwd = m.group(2);
- String host = m.group(3);
- String portStr = m.group(4);
- int port = Integer.parseInt(portStr == null ? "7070" : portStr);
-
- init(host, port, user, pwd);
- }
-
- private void init(String host, int port, String userName, String password) {
- this.host = host;
- this.port = port;
- this.userName = userName;
- this.password = password;
- this.baseUrl = "http://" + host + ":" + port + "/kylin/api";
-
- client = new HttpClient();
-
- if (userName != null && password != null) {
- client.getParams().setAuthenticationPreemptive(true);
- Credentials creds = new UsernamePasswordCredentials(userName, password);
- client.getState().setCredentials(new AuthScope(host, port, AuthScope.ANY_REALM), creds);
- }
- }
-
- public void wipeCache(String type, String action, String name) throws IOException {
- String url = baseUrl + "/cache/" + type + "/" + name + "/" + action;
- HttpMethod get = new PutMethod(url);
-
- try {
- int code = client.executeMethod(get);
- String msg = Bytes.toString(get.getResponseBody());
-
- if (code != 200)
- throw new IOException("Invalid response " + code + " with cache wipe url " + url + "\n" + msg);
-
- } catch (HttpException ex) {
- throw new IOException(ex);
- } finally {
- get.releaseConnection();
- }
- }
-
- public String getKylinProperties() throws IOException {
- String url = baseUrl + "/admin/config";
- HttpMethod get = new GetMethod(url);
- try {
- int code = client.executeMethod(get);
- String msg = Bytes.toString(get.getResponseBody());
- JSONObject obj = new JSONObject(msg);
- msg = obj.getString("config");
-
- if (code != 200)
- throw new IOException("Invalid response " + code + " with cache wipe url " + url + "\n" + msg);
-
- return msg;
-
- } catch (JSONException e) {
- throw new IOException("Error when parsing json response from REST");
- } finally {
- get.releaseConnection();
- }
- }
-
-}
diff --git a/common/src/main/java/com/kylinolap/common/restclient/SingleValueCache.java b/common/src/main/java/com/kylinolap/common/restclient/SingleValueCache.java
deleted file mode 100644
index 394697cdd0..0000000000
--- a/common/src/main/java/com/kylinolap/common/restclient/SingleValueCache.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.common.restclient;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-
-/**
- * @author xjiang
- *
- */
-public class SingleValueCache<K, V> extends AbstractRestCache<K, V> {
-
- private final Map<K, V> innerCache;
-
- public SingleValueCache(Broadcaster.TYPE syncType) {
- super(syncType);
- innerCache = new ConcurrentHashMap<K, V>();
- }
-
- public void put(K key, V value) {
- Broadcaster.EVENT eventType = innerCache.containsKey(key) ? Broadcaster.EVENT.UPDATE : Broadcaster.EVENT.CREATE;
- innerCache.put(key, value);
- syncRemote(key, eventType);
- }
-
- public void putLocal(K key, V value) {
- innerCache.put(key, value);
- }
-
- public void remove(K key) {
- if (innerCache.containsKey(key)) {
- innerCache.remove(key);
- syncRemote(key, Broadcaster.EVENT.DROP);
- }
- }
-
- public void removeLocal(K key) {
- innerCache.remove(key);
- }
-
- public void clear() {
- innerCache.clear();
- }
-
- public int size() {
- return innerCache.size();
- }
-
- public V get(K key) {
- return innerCache.get(key);
- }
-
- public Collection<V> values() {
- return innerCache.values();
- }
-
- public boolean containsKey(String key) {
- return innerCache.containsKey(key);
- }
-
- public Map<K, V> getMap() {
- return Collections.unmodifiableMap(innerCache);
- }
-
- public Set<K> keySet() {
- return innerCache.keySet();
- }
-}
diff --git a/common/src/main/java/com/kylinolap/common/util/AbstractKylinTestCase.java b/common/src/main/java/com/kylinolap/common/util/AbstractKylinTestCase.java
deleted file mode 100644
index 9b1005e8dc..0000000000
--- a/common/src/main/java/com/kylinolap/common/util/AbstractKylinTestCase.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.common.util;
-
-import com.kylinolap.common.KylinConfig;
-
-/**
- * @author ysong1
- *
- */
-public abstract class AbstractKylinTestCase {
-
- public static final String LOCALMETA_TEST_DATA = "../examples/test_case_data/localmeta";
-
- public static final String MINICLUSTER_TEST_DATA = "../examples/test_case_data/minicluster";
-
- public static final String SANDBOX_TEST_DATA = "../examples/test_case_data/sandbox";
-
- public abstract void createTestMetadata();
-
- public abstract void cleanupTestMetadata();
-
- public KylinConfig getTestConfig() {
- return KylinConfig.getInstanceFromEnv();
- }
-
-}
diff --git a/common/src/main/java/com/kylinolap/common/util/Array.java b/common/src/main/java/com/kylinolap/common/util/Array.java
deleted file mode 100644
index 0b4165f472..0000000000
--- a/common/src/main/java/com/kylinolap/common/util/Array.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.common.util;
-
-import java.util.Arrays;
-
-/*
- * An array with correct equals(), hashCode(), compareTo() and toString()
- */
-public class Array<T> implements Comparable<Array<T>> {
- public T[] data;
-
- public Array(T[] data) {
- this.data = data;
- }
-
- public String toString() {
- return Arrays.toString(data);
- }
-
- @Override
- public boolean equals(Object o) {
- if (o != null && o instanceof Array) {
- return Arrays.equals(this.data, ((Array<?>) o).data);
- }
- return false;
- }
-
- @Override
- public int hashCode() {
- return Arrays.hashCode(data);
- }
-
- @Override
- public int compareTo(Array<T> other) {
- return compare(this.data, other.data, null);
- }
-
- @SuppressWarnings("unchecked")
- public static <T> int compare(T[] a, T[] b, boolean[] ascending) {
- int r = 0;
- int n = Math.min(a.length, b.length);
- boolean asc = true;
-
- for (int i = 0; i < n; i++) {
- r = ((Comparable<T>) a[i]).compareTo(b[i]);
- if (r != 0) {
- asc = (ascending != null && ascending.length > i) ? ascending[i] : true;
- return asc ? r : -r;
- }
- }
- return a.length - b.length;
- }
-
-}
diff --git a/common/src/main/java/com/kylinolap/common/util/ByteArray.java b/common/src/main/java/com/kylinolap/common/util/ByteArray.java
deleted file mode 100644
index d6aee89aa1..0000000000
--- a/common/src/main/java/com/kylinolap/common/util/ByteArray.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.common.util;
-
-import java.util.Arrays;
-
-import org.apache.hadoop.hbase.util.Bytes;
-
-/**
- * @author yangli9
- */
-public class ByteArray implements Comparable<ByteArray> {
-
- public byte[] data;
-
- public ByteArray(byte[] data) {
- this.data = data;
- }
-
- @Override
- public int hashCode() {
- return Bytes.hashCode(data);
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- ByteArray other = (ByteArray) obj;
- if (!Arrays.equals(data, other.data))
- return false;
- return true;
- }
-
- @Override
- public int compareTo(ByteArray o) {
- return Bytes.compareTo(this.data, o.data);
- }
-}
diff --git a/common/src/main/java/com/kylinolap/common/util/BytesSerializer.java b/common/src/main/java/com/kylinolap/common/util/BytesSerializer.java
deleted file mode 100644
index 5d4603736d..0000000000
--- a/common/src/main/java/com/kylinolap/common/util/BytesSerializer.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.common.util;
-
-import java.nio.ByteBuffer;
-
-/**
- * @author yangli9
- *
- */
-public interface BytesSerializer<T> {
-
- public static final int SERIALIZE_BUFFER_SIZE = 65536;
-
- abstract public void serialize(T value, ByteBuffer out);
-
- abstract public T deserialize(ByteBuffer in);
-
-}
diff --git a/common/src/main/java/com/kylinolap/common/util/BytesUtil.java b/common/src/main/java/com/kylinolap/common/util/BytesUtil.java
deleted file mode 100644
index 93d72a47de..0000000000
--- a/common/src/main/java/com/kylinolap/common/util/BytesUtil.java
+++ /dev/null
@@ -1,348 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.common.util;
-
-import java.io.ByteArrayOutputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.nio.ByteBuffer;
-
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.io.Writable;
-
-public class BytesUtil {
-
- public static final byte[] EMPTY_BYTE_ARRAY = new byte[0];
-
- public static void writeLong(long num, byte[] bytes, int offset, int size) {
- for (int i = offset + size - 1; i >= offset; i--) {
- bytes[i] = (byte) num;
- num >>>= 8;
- }
- }
-
- public static void writeUnsigned(int num, byte[] bytes, int offset, int size) {
- for (int i = offset + size - 1; i >= offset; i--) {
- bytes[i] = (byte) num;
- num >>>= 8;
- }
- }
-
- public static long readLong(byte[] bytes, int offset, int size) {
- long integer = 0;
- for (int i = offset, n = offset + size; i < n; i++) {
- integer <<= 8;
- integer |= (long) bytes[i] & 0xFF;
- }
- return integer;
- }
-
- public static int readUnsigned(byte[] bytes, int offset, int size) {
- int integer = 0;
- for (int i = offset, n = offset + size; i < n; i++) {
- integer <<= 8;
- integer |= (int) bytes[i] & 0xFF;
- }
- return integer;
- }
-
- public static void writeSigned(int num, byte[] bytes, int offset, int size) {
- writeUnsigned(num, bytes, offset, size);
- }
-
- public static int readSigned(byte[] bytes, int offset, int size) {
- int integer = (bytes[offset] & 0x80) == 0 ? 0 : -1;
- for (int i = offset, n = offset + size; i < n; i++) {
- integer <<= 8;
- integer |= (int) bytes[i] & 0xFF;
- }
- return integer;
- }
-
- /**
- * No. bytes needed to store a value as big as the given
- */
- public static int sizeForValue(int maxValue) {
- int size = 0;
- while (maxValue > 0) {
- size++;
- maxValue >>>= 8;
- }
- return size;
- }
-
- public static int compareByteUnsigned(byte b1, byte b2) {
- int i1 = (int) b1 & 0xFF;
- int i2 = (int) b2 & 0xFF;
- return i1 - i2;
- }
-
- public static byte[] subarray(byte[] bytes, int start, int end) {
- byte[] r = new byte[end - start];
- System.arraycopy(bytes, start, r, 0, r.length);
- return r;
- }
-
- public static int compareBytes(byte[] src, int srcOffset, byte[] dst, int dstOffset, int length) {
- int r = 0;
- for (int i = 0; i < length; i++) {
- r = src[srcOffset + i] - dst[dstOffset + i];
- if (r != 0)
- break;
- }
- return r;
- }
-
- // from WritableUtils
- // ============================================================================
-
- public static void writeVInt(int i, ByteBuffer out) {
- writeVLong(i, out);
- }
-
- public static void writeVLong(long i, ByteBuffer out) {
- if (i >= -112 && i <= 127) {
- out.put((byte) i);
- return;
- }
-
- int len = -112;
- if (i < 0) {
- i ^= -1L; // take one's complement'
- len = -120;
- }
-
- long tmp = i;
- while (tmp != 0) {
- tmp = tmp >> 8;
- len--;
- }
-
- out.put((byte) len);
-
- len = (len < -120) ? -(len + 120) : -(len + 112);
-
- for (int idx = len; idx != 0; idx--) {
- int shiftbits = (idx - 1) * 8;
- long mask = 0xFFL << shiftbits;
- out.put((byte) ((i & mask) >> shiftbits));
- }
- }
-
- public static long readVLong(ByteBuffer in) {
- byte firstByte = in.get();
- int len = decodeVIntSize(firstByte);
- if (len == 1) {
- return firstByte;
- }
- long i = 0;
- for (int idx = 0; idx < len - 1; idx++) {
- byte b = in.get();
- i = i << 8;
- i = i | (b & 0xFF);
- }
- return (isNegativeVInt(firstByte) ? (i ^ -1L) : i);
- }
-
- public static int readVInt(ByteBuffer in) {
- long n = readVLong(in);
- if ((n > Integer.MAX_VALUE) || (n < Integer.MIN_VALUE)) {
- throw new IllegalArgumentException("value too long to fit in integer");
- }
- return (int) n;
- }
-
- private static boolean isNegativeVInt(byte value) {
- return value < -120 || (value >= -112 && value < 0);
- }
-
- private static int decodeVIntSize(byte value) {
- if (value >= -112) {
- return 1;
- } else if (value < -120) {
- return -119 - value;
- }
- return -111 - value;
- }
-
- public static void writeUnsigned(int num, int size, ByteBuffer out) {
- for (int i = 0; i < size; i++) {
- out.put((byte) num);
- num >>>= 8;
- }
- }
-
- public static int readUnsigned(ByteBuffer in, int size) {
- int integer = 0;
- int mask = 0xff;
- int shift = 0;
- for (int i = 0; i < size; i++) {
- integer |= (in.get() << shift) & mask;
- mask = mask << 8;
- shift += 8;
- }
- return integer;
- }
-
- public static void writeLong(long num, ByteBuffer out) {
- for (int i = 0; i < 8; i++) {
- out.put((byte) num);
- num >>>= 8;
- }
- }
-
- public static long readLong(ByteBuffer in) {
- long integer = 0;
- int mask = 0xff;
- int shift = 0;
- for (int i = 0; i < 8; i++) {
- integer |= (in.get() << shift) & mask;
- mask = mask << 8;
- shift += 8;
- }
- return integer;
- }
-
- public static void writeUTFString(String str, ByteBuffer out) {
- byte[] bytes = str == null ? null : Bytes.toBytes(str);
- writeByteArray(bytes, out);
- }
-
- public static String readUTFString(ByteBuffer in) {
- byte[] bytes = readByteArray(in);
- return bytes == null ? null : Bytes.toString(bytes);
- }
-
- public static void writeAsciiString(String str, ByteBuffer out) {
- if (str == null) {
- BytesUtil.writeVInt(-1, out);
- return;
- }
- int len = str.length();
- BytesUtil.writeVInt(len, out);
- for (int i = 0; i < len; i++) {
- out.put((byte) str.charAt(i));
- }
- }
-
- public static String readAsciiString(ByteBuffer in) {
- int len = BytesUtil.readVInt(in);
- if (len < 0) {
- return null;
- }
- String result;
- try {
- if (in.hasArray()) {
- int pos = in.position();
- result = new String(in.array(), pos, len, "ISO-8859-1");
- in.position(pos + len);
- } else {
- byte[] tmp = new byte[len];
- in.get(tmp);
- result = new String(tmp, "ISO-8859-1");
- }
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e); // never happen
- }
- return result;
- }
-
- public static void writeAsciiStringArray(String[] strs, ByteBuffer out) {
- writeVInt(strs.length, out);
- for (int i = 0; i < strs.length; i++)
- writeAsciiString(strs[i], out);
- }
-
- public static String[] readAsciiStringArray(ByteBuffer in) {
- int len = readVInt(in);
- String[] strs = new String[len];
- for (int i = 0; i < len; i++)
- strs[i] = readAsciiString(in);
- return strs;
- }
-
- public static void writeIntArray(int[] array, ByteBuffer out) {
- if (array == null) {
- writeVInt(-1, out);
- return;
- }
- writeVInt(array.length, out);
- out.asIntBuffer().put(array);
- }
-
- public static int[] readIntArray(ByteBuffer in) {
- int len = readVInt(in);
- if (len < 0)
- return null;
- int[] array = new int[len];
- in.asIntBuffer().get(array);
- return array;
- }
-
- public static void writeByteArray(byte[] array, ByteBuffer out) {
- if (array == null) {
- writeVInt(-1, out);
- return;
- }
- writeVInt(array.length, out);
- out.put(array);
- }
-
- public static byte[] readByteArray(ByteBuffer in) {
- int len = readVInt(in);
- if (len < 0)
- return null;
-
- byte[] array = new byte[len];
- in.get(array);
- return array;
- }
-
- public static byte[] toBytes(Writable writable) {
- try {
- ByteArrayOutputStream bout = new ByteArrayOutputStream();
- DataOutputStream out = new DataOutputStream(bout);
- writable.write(out);
- out.close();
- bout.close();
- return bout.toByteArray();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- public static String toHex(byte[] array) {
- return toHex(new ImmutableBytesWritable(array));
- }
-
- public static String toHex(ImmutableBytesWritable bytes) {
- byte[] array = bytes.get();
- int offset = bytes.getOffset();
- int length = bytes.getLength();
- StringBuilder sb = new StringBuilder(length * 4);
- for (int i = 0; i < length; i++) {
- int b = array[offset + i];
- sb.append(String.format("\\x%02X", b & 0xFF));
- }
- return sb.toString();
- }
-
- public static void main(String[] args) throws Exception {
- }
-
-}
diff --git a/common/src/main/java/com/kylinolap/common/util/ClasspathUtil.java b/common/src/main/java/com/kylinolap/common/util/ClasspathUtil.java
deleted file mode 100644
index 6be3708c75..0000000000
--- a/common/src/main/java/com/kylinolap/common/util/ClasspathUtil.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.common.util;
-
-import java.io.File;
-import java.lang.reflect.Method;
-import java.net.URL;
-import java.net.URLClassLoader;
-
-/**
- * @author xduo
- *
- */
-public class ClasspathUtil {
-
- public static void addClasspath(String path) throws Exception {
- System.out.println("Adding path " + path + " to class path");
- File file = new File(path);
-
- if (file.exists()) {
- URLClassLoader urlClassLoader = (URLClassLoader) ClassLoader.getSystemClassLoader();
- Class<URLClassLoader> urlClass = URLClassLoader.class;
- Method method = urlClass.getDeclaredMethod("addURL", new Class[] { URL.class });
- method.setAccessible(true);
- method.invoke(urlClassLoader, new Object[] { file.toURI().toURL() });
- }
- }
-
-}
diff --git a/common/src/main/java/com/kylinolap/common/util/CliCommandExecutor.java b/common/src/main/java/com/kylinolap/common/util/CliCommandExecutor.java
deleted file mode 100644
index 3f4ab7b721..0000000000
--- a/common/src/main/java/com/kylinolap/common/util/CliCommandExecutor.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.common.util;
-
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.hbase.util.Pair;
-
-/**
- * @author yangli9
- */
-public class CliCommandExecutor {
-
- private String remoteHost;
- private String remoteUser;
- private String remotePwd;
- private int remoteTimeoutSeconds = 3600;
-
- public CliCommandExecutor() {
- }
-
- public void setRunAtRemote(String host, String user, String pwd) {
- this.remoteHost = host;
- this.remoteUser = user;
- this.remotePwd = pwd;
- }
-
- public void setRunAtLocal() {
- this.remoteHost = null;
- this.remoteUser = null;
- this.remotePwd = null;
- }
-
- public void copyFile(String localFile, String destDir) throws IOException {
- if (remoteHost == null)
- copyNative(localFile, destDir);
- else
- copyRemote(localFile, destDir);
- }
-
- private void copyNative(String localFile, String destDir) throws IOException {
- File src = new File(localFile);
- File dest = new File(destDir, src.getName());
- FileUtils.copyFile(src, dest);
- }
-
- private void copyRemote(String localFile, String destDir) throws IOException {
- SSHClient ssh = new SSHClient(remoteHost, remoteUser, remotePwd, null);
- try {
- ssh.scpFileToRemote(localFile, destDir);
- } catch (IOException e) {
- throw e;
- } catch (Exception e) {
- throw new IOException(e.getMessage(), e);
- }
- }
-
- public String execute(String command) throws IOException {
- Pair<Integer, String> r;
- if (remoteHost == null)
- r = runNativeCommand(command);
- else
- r = runRemoteCommand(command);
-
- if (r.getFirst() != 0)
- throw new IOException("OS command error exit with " + r.getFirst() + " -- " + command + "\n" + r.getSecond());
-
- return r.getSecond();
- }
-
- private Pair<Integer, String> runRemoteCommand(String command) throws IOException {
- SSHClient ssh = new SSHClient(remoteHost, remoteUser, remotePwd, null);
-
- SSHClientOutput sshOutput;
- try {
- sshOutput = ssh.execCommand(command, remoteTimeoutSeconds);
- int exitCode = sshOutput.getExitCode();
- String output = sshOutput.getText();
- return new Pair<Integer, String>(exitCode, output);
- } catch (IOException e) {
- throw e;
- } catch (Exception e) {
- throw new IOException(e.getMessage(), e);
- }
- }
-
- private Pair<Integer, String> runNativeCommand(String command) throws IOException {
- String[] cmd = new String[3];
- String osName = System.getProperty("os.name");
- if (osName.startsWith("Windows")) {
- cmd[0] = "cmd.exe";
- cmd[1] = "/C";
- } else {
- cmd[0] = "/bin/bash";
- cmd[1] = "-c";
- }
- cmd[2] = command;
-
- ProcessBuilder builder = new ProcessBuilder(cmd);
- builder.redirectErrorStream(true);
- Process proc = builder.start();
-
- ByteArrayOutputStream buf = new ByteArrayOutputStream();
- IOUtils.copy(proc.getInputStream(), buf);
- String output = buf.toString("UTF-8");
-
- try {
- int exitCode = proc.waitFor();
- return new Pair<Integer, String>(exitCode, output);
- } catch (InterruptedException e) {
- throw new IOException(e);
- }
- }
-
-}
diff --git a/common/src/main/java/com/kylinolap/common/util/HBaseMetadataTestCase.java b/common/src/main/java/com/kylinolap/common/util/HBaseMetadataTestCase.java
deleted file mode 100644
index a1dda9aa9b..0000000000
--- a/common/src/main/java/com/kylinolap/common/util/HBaseMetadataTestCase.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.common.util;
-
-import com.kylinolap.common.KylinConfig;
-
-/**
- * @author ysong1
- */
-public class HBaseMetadataTestCase extends AbstractKylinTestCase {
-
- public static void staticCreateTestMetadata() {
-
- KylinConfig.destoryInstance();
-
- if (System.getProperty(KylinConfig.KYLIN_CONF) == null && System.getenv(KylinConfig.KYLIN_CONF) == null)
- System.setProperty(KylinConfig.KYLIN_CONF, SANDBOX_TEST_DATA);
-
- }
-
- public static void staticCleanupTestMetadata() {
- System.clearProperty(KylinConfig.KYLIN_CONF);
- KylinConfig.destoryInstance();
- }
-
- @Override
- public void createTestMetadata() {
- staticCreateTestMetadata();
- }
-
- @Override
- public void cleanupTestMetadata() {
- staticCleanupTestMetadata();
- }
-
-}
diff --git a/common/src/main/java/com/kylinolap/common/util/HBaseRegionSizeCalculator.java b/common/src/main/java/com/kylinolap/common/util/HBaseRegionSizeCalculator.java
deleted file mode 100644
index 36ffc29386..0000000000
--- a/common/src/main/java/com/kylinolap/common/util/HBaseRegionSizeCalculator.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/** This class will come with HBase 2.0 in package org.apache.hadoop.hbase.util **/
-package com.kylinolap.common.util;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.TreeSet;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.ClusterStatus;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.RegionLoad;
-import org.apache.hadoop.hbase.ServerLoad;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class HBaseRegionSizeCalculator {
-
- private static final Logger logger = LoggerFactory.getLogger(HBaseRegionSizeCalculator.class);
-
- /**
- * Maps each region to its size in bytes.
- **/
- private final Map<byte[], Long> sizeMap = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
-
- static final String ENABLE_REGIONSIZECALCULATOR = "hbase.regionsizecalculator.enable";
-
- /**
- * Computes size of each region for table and given column families.
- * */
- public HBaseRegionSizeCalculator(HTable table) throws IOException {
- this(table, new HBaseAdmin(table.getConfiguration()));
- }
-
- /** Constructor for unit testing */
- HBaseRegionSizeCalculator(HTable table, HBaseAdmin hBaseAdmin) throws IOException {
-
- try {
- if (!enabled(table.getConfiguration())) {
- logger.info("Region size calculation disabled.");
- return;
- }
-
- logger.info("Calculating region sizes for table \"" + new String(table.getTableName()) + "\".");
-
- // Get regions for table.
- Set<HRegionInfo> tableRegionInfos = table.getRegionLocations().keySet();
- Set<byte[]> tableRegions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
-
- for (HRegionInfo regionInfo : tableRegionInfos) {
- tableRegions.add(regionInfo.getRegionName());
- }
-
- ClusterStatus clusterStatus = hBaseAdmin.getClusterStatus();
- Collection<ServerName> servers = clusterStatus.getServers();
- final long megaByte = 1024L * 1024L;
-
- // Iterate all cluster regions, filter regions from our table and
- // compute their size.
- for (ServerName serverName : servers) {
- ServerLoad serverLoad = clusterStatus.getLoad(serverName);
-
- for (RegionLoad regionLoad : serverLoad.getRegionsLoad().values()) {
- byte[] regionId = regionLoad.getName();
-
- if (tableRegions.contains(regionId)) {
-
- long regionSizeBytes = regionLoad.getStorefileSizeMB() * megaByte;
- sizeMap.put(regionId, regionSizeBytes);
-
- // logger.info("Region " + regionLoad.getNameAsString()
- // + " has size " + regionSizeBytes);
- }
- }
- }
- } finally {
- hBaseAdmin.close();
- }
-
- }
-
- boolean enabled(Configuration configuration) {
- return configuration.getBoolean(ENABLE_REGIONSIZECALCULATOR, true);
- }
-
- /**
- * Returns size of given region in bytes. Returns 0 if region was not found.
- **/
- public long getRegionSize(byte[] regionId) {
- Long size = sizeMap.get(regionId);
- if (size == null) {
- logger.info("Unknown region:" + Arrays.toString(regionId));
- return 0;
- } else {
- return size;
- }
- }
-
- public Map<byte[], Long> getRegionSizeMap() {
- return Collections.unmodifiableMap(sizeMap);
- }
-}
diff --git a/common/src/main/java/com/kylinolap/common/util/HadoopUtil.java b/common/src/main/java/com/kylinolap/common/util/HadoopUtil.java
deleted file mode 100644
index 77767fc6bb..0000000000
--- a/common/src/main/java/com/kylinolap/common/util/HadoopUtil.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.common.util;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.UnknownHostException;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HConstants;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class HadoopUtil {
- private static final Logger logger = LoggerFactory.getLogger(HadoopUtil.class);
-
- private static Configuration hadoopConfig;
-
- public static Configuration getDefaultConfiguration() {
- if (hadoopConfig == null) {
- hadoopConfig = new Configuration();
- }
- return hadoopConfig;
- }
-
- public static FileSystem getFileSystem(String path) throws IOException {
- return FileSystem.get(makeURI(path), getDefaultConfiguration());
- }
-
- public static URI makeURI(String filePath) {
- try {
- return new URI(filePath);
- } catch (URISyntaxException e) {
- throw new IllegalArgumentException("Cannot create FileSystem from URI: " + filePath, e);
- }
- }
-
- /**
- * e.g. "hbase:kylin-local.corp.ebay.com:2181:/hbase-unsecure"
- */
- public static Configuration newHBaseConfiguration(String url) {
- Configuration conf = HBaseConfiguration.create();
- if (StringUtils.isEmpty(url))
- return conf;
-
- // chop off "hbase:"
- if (url.startsWith("hbase:") == false)
- throw new IllegalArgumentException("hbase url must start with 'hbase:' -- " + url);
-
- url = StringUtils.substringAfter(url, "hbase:");
- if (StringUtils.isEmpty(url))
- return conf;
-
- // case of "hbase:domain.com:2181:/hbase-unsecure"
- Pattern urlPattern = Pattern.compile("([\\w\\d\\-.]+)[:](\\d+)(?:[:](.*))?");
- Matcher m = urlPattern.matcher(url);
- if (m.matches() == false)
- throw new IllegalArgumentException("HBase URL '" + url + "' is invalid, expected url is like '" + "hbase:domain.com:2181:/hbase-unsecure" + "'");
-
- logger.debug("Creating hbase conf by parsing -- " + url);
-
- String quorum = m.group(1);
- try {
- InetAddress.getByName(quorum);
- } catch (UnknownHostException e) {
- throw new IllegalArgumentException("Zookeeper quorum is invalid: " + quorum + "; urlString=" + url, e);
- }
- conf.set(HConstants.ZOOKEEPER_QUORUM, quorum);
-
- String port = m.group(2);
- conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, port);
-
- String znodePath = m.group(3) == null ? "" : m.group(3);
- if (StringUtils.isEmpty(znodePath) == false)
- conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, znodePath);
-
- // reduce rpc retry
- conf.set(HConstants.HBASE_CLIENT_PAUSE, "3000");
- conf.set(HConstants.HBASE_CLIENT_RETRIES_NUMBER, "5");
- conf.set(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, "60000");
- // conf.set(ScannerCallable.LOG_SCANNER_ACTIVITY, "true");
-
- return conf;
- }
-
-}
diff --git a/common/src/main/java/com/kylinolap/common/util/JsonUtil.java b/common/src/main/java/com/kylinolap/common/util/JsonUtil.java
deleted file mode 100644
index 1ca8a904a6..0000000000
--- a/common/src/main/java/com/kylinolap/common/util/JsonUtil.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.common.util;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.Reader;
-
-import com.fasterxml.jackson.core.JsonGenerationException;
-import com.fasterxml.jackson.core.JsonParseException;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.JsonMappingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
-
-public class JsonUtil {
-
- // reuse the object mapper to save memory footprint
- private static final ObjectMapper mapper = new ObjectMapper();
- private static final ObjectMapper indentMapper = new ObjectMapper();
-
- static {
- mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
- indentMapper.configure(SerializationFeature.INDENT_OUTPUT, true);
- }
-
- public static <T> T readValue(File src, Class<T> valueType) throws IOException, JsonParseException, JsonMappingException {
- return mapper.readValue(src, valueType);
- }
-
- public static <T> T readValue(String content, Class<T> valueType) throws IOException, JsonParseException, JsonMappingException {
- return mapper.readValue(content, valueType);
- }
-
- public static <T> T readValue(Reader src, Class<T> valueType) throws IOException, JsonParseException, JsonMappingException {
- return mapper.readValue(src, valueType);
- }
-
- public static <T> T readValue(InputStream src, Class<T> valueType) throws IOException, JsonParseException, JsonMappingException {
- return mapper.readValue(src, valueType);
- }
-
- public static <T> T readValue(byte[] src, Class<T> valueType) throws IOException, JsonParseException, JsonMappingException {
- return mapper.readValue(src, valueType);
- }
-
- public static void writeValueIndent(OutputStream out, Object value) throws IOException, JsonGenerationException, JsonMappingException {
- indentMapper.writeValue(out, value);
- }
-
- public static void writeValue(OutputStream out, Object value) throws IOException, JsonGenerationException, JsonMappingException {
- mapper.writeValue(out, value);
- }
-
- public static String writeValueAsString(Object value) throws JsonProcessingException {
- return mapper.writeValueAsString(value);
- }
-
- public static byte[] writeValueAsBytes(Object value) throws JsonProcessingException {
- return mapper.writeValueAsBytes(value);
- }
-
- public static String writeValueAsIndentString(Object value) throws JsonProcessingException {
- return indentMapper.writeValueAsString(value);
- }
-
-}
diff --git a/common/src/main/java/com/kylinolap/common/util/LocalFileMetadataTestCase.java b/common/src/main/java/com/kylinolap/common/util/LocalFileMetadataTestCase.java
deleted file mode 100644
index f176c719a4..0000000000
--- a/common/src/main/java/com/kylinolap/common/util/LocalFileMetadataTestCase.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.common.util;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.commons.io.FileUtils;
-
-import com.kylinolap.common.KylinConfig;
-import com.kylinolap.common.persistence.ResourceStore;
-
-/**
- * @author ysong1
- */
-public class LocalFileMetadataTestCase extends AbstractKylinTestCase {
- private String tempTestMetadataUrl = null;
-
- @Override
- public void createTestMetadata() {
- KylinConfig.destoryInstance();
-
- this.tempTestMetadataUrl = "../examples/test_metadata";
- try {
- FileUtils.deleteDirectory(new File(tempTestMetadataUrl));
- FileUtils.copyDirectory(new File(LOCALMETA_TEST_DATA), new File(tempTestMetadataUrl));
- } catch (IOException e) {
- e.printStackTrace();
- }
-
- if (System.getProperty(KylinConfig.KYLIN_CONF) == null && System.getenv(KylinConfig.KYLIN_CONF) == null)
- System.setProperty(KylinConfig.KYLIN_CONF, tempTestMetadataUrl);
-
- KylinConfig.getInstanceFromEnv().setMetadataUrl(tempTestMetadataUrl);
-
- }
-
- @Override
- public void cleanupTestMetadata() {
- try {
- FileUtils.deleteDirectory(new File(tempTestMetadataUrl));
- } catch (IOException e) {
- throw new IllegalStateException("Can't delete directory " + tempTestMetadataUrl, e);
- }
- System.clearProperty(KylinConfig.KYLIN_CONF);
- KylinConfig.destoryInstance();
- this.tempTestMetadataUrl = null;
- }
-
- protected ResourceStore getStore() {
- return ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
- }
-}
diff --git a/common/src/main/java/com/kylinolap/common/util/LongAsFloat.java b/common/src/main/java/com/kylinolap/common/util/LongAsFloat.java
deleted file mode 100644
index 3d38ec246b..0000000000
--- a/common/src/main/java/com/kylinolap/common/util/LongAsFloat.java
+++ /dev/null
@@ -1,5 +0,0 @@
-package com.kylinolap.common.util;
-
-public class LongAsFloat {
-
-}
diff --git a/common/src/main/java/com/kylinolap/common/util/MailService.java b/common/src/main/java/com/kylinolap/common/util/MailService.java
deleted file mode 100644
index 608806c19d..0000000000
--- a/common/src/main/java/com/kylinolap/common/util/MailService.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.common.util;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.commons.mail.Email;
-import org.apache.commons.mail.EmailException;
-import org.apache.commons.mail.HtmlEmail;
-
-import com.kylinolap.common.KylinConfig;
-
-/**
- * @author xduo
- *
- */
-public class MailService {
-
- private Boolean enabled = Boolean.TRUE;
- private String host;
- private String username;
- private String password;
- private String sender;
-
- private static final Log logger = LogFactory.getLog(MailService.class);
-
- public MailService() {
- this(KylinConfig.getInstanceFromEnv());
- }
-
- public MailService(KylinConfig config) {
- enabled = "true".equalsIgnoreCase(config.getProperty(KylinConfig.MAIL_ENABLED, "true"));
- host = config.getProperty(KylinConfig.MAIL_HOST, "");
- username = config.getProperty(KylinConfig.MAIL_USERNAME, "");
- password = config.getProperty(KylinConfig.MAIL_PASSWORD, "");
- sender = config.getProperty(KylinConfig.MAIL_SENDER, "");
-
- if (enabled) {
- assert !host.isEmpty();
- }
- }
-
- /**
- *
- * @param receivers
- * @param subject
- * @param content
- * @return true or false indicating whether the email was delivered successfully
- * @throws IOException
- */
- public boolean sendMail(List<String> receivers, String subject, String content) throws IOException {
-
- if (!enabled) {
- logger.info("Email service is disabled; this mail will not be delivered: " + subject);
- logger.info("To enable mail service, set 'mail.enabled=true' in kylin.properties");
- return false;
- }
-
- Email email = new HtmlEmail();
- email.setHostName(host);
- if (username != null && username.trim().length() > 0) {
- email.setAuthentication(username, password);
- }
-
- //email.setDebug(true);
- try {
- for (String receiver : receivers) {
- email.addTo(receiver);
- }
-
- email.setFrom(sender);
- email.setSubject(subject);
- email.setCharset("UTF-8");
- ((HtmlEmail) email).setHtmlMsg(content);
- email.send();
- email.getMailSession();
-
- } catch (EmailException e) {
- logger.error(e.getLocalizedMessage(),e);
- return false;
- }
-
- return true;
- }
-}
diff --git a/common/src/main/java/com/kylinolap/common/util/MyLogFormatter.java b/common/src/main/java/com/kylinolap/common/util/MyLogFormatter.java
deleted file mode 100644
index b7e618043e..0000000000
--- a/common/src/main/java/com/kylinolap/common/util/MyLogFormatter.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.common.util;
-
-import java.io.PrintWriter;
-import java.io.StringWriter;
-import java.sql.Timestamp;
-import java.util.Date;
-import java.util.logging.Formatter;
-import java.util.logging.LogRecord;
-
-public class MyLogFormatter extends Formatter {
-
- Date dat = new Date();
-
- // Line separator string. This is the value of the line.separator
- // property at the moment that the SimpleFormatter was created.
- private String lineSeparator = "\n";
-
- /**
- * Format the given LogRecord.
- *
- * @param record
- * the log record to be formatted.
- * @return a formatted log record
- */
- public synchronized String format(LogRecord record) {
- StringBuffer sb = new StringBuffer();
- // Minimize memory allocations here.
- Timestamp ts = new Timestamp(record.getMillis());
- String text = ts.toString();
- sb.append("JUL ");
- sb.append(text);
- sb.append(" ");
- if (record.getSourceClassName() != null) {
- sb.append(record.getSourceClassName());
- } else {
- sb.append(record.getLoggerName());
- }
- if (record.getSourceMethodName() != null) {
- sb.append(" ");
- sb.append(record.getSourceMethodName());
- }
- sb.append(lineSeparator);
- String message = formatMessage(record);
- sb.append(record.getLevel().getLocalizedName());
- sb.append(": ");
- sb.append(message);
- sb.append(lineSeparator);
- if (record.getThrown() != null) {
- try {
- StringWriter sw = new StringWriter();
- PrintWriter pw = new PrintWriter(sw);
- record.getThrown().printStackTrace(pw);
- pw.close();
- sb.append(sw.toString());
- } catch (Exception ex) {
- }
- }
- return sb.toString();
- }
-}
diff --git a/common/src/main/java/com/kylinolap/common/util/RandomSampler.java b/common/src/main/java/com/kylinolap/common/util/RandomSampler.java
deleted file mode 100644
index 214d187edd..0000000000
--- a/common/src/main/java/com/kylinolap/common/util/RandomSampler.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.common.util;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-
-/**
- * @author ysong1
- *
- */
-public class RandomSampler<T> {
-
- private Random rdm = new Random();
-
- public List<T> sample(List<T> data, int sampleNumber) {
- if (data == null) {
- throw new IllegalArgumentException("Input list is null");
- }
- if (data.size() < sampleNumber) {
- return data;
- }
-
- List<T> result = new ArrayList<T>(sampleNumber);
- int n = data.size();
- for (int i = 0; i < n; i++) {
- if (i < sampleNumber) {
- result.add(data.get(i));
- } else {
- int j = rdm.nextInt(i);
- if (j < sampleNumber) {
- result.set(j, data.get(i));
- }
- }
- }
- return result;
- }
-}
diff --git a/common/src/main/java/com/kylinolap/common/util/SSHClient.java b/common/src/main/java/com/kylinolap/common/util/SSHClient.java
deleted file mode 100644
index aadef76e97..0000000000
--- a/common/src/main/java/com/kylinolap/common/util/SSHClient.java
+++ /dev/null
@@ -1,268 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.common.util;
-
-/**
- * @author George Song (ysong1)
- *
- */
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.jcraft.jsch.Channel;
-import com.jcraft.jsch.ChannelExec;
-import com.jcraft.jsch.JSch;
-import com.jcraft.jsch.JSchException;
-import com.jcraft.jsch.Session;
-
-public class SSHClient {
- protected static final Logger logger = LoggerFactory.getLogger(SSHClient.class);
-
- private String hostname;
- private String username;
- private String password;
- private String identityPath;
-
- private SSHLogger sshLogger;
-
- public SSHClient(String hostname, String username, String password, SSHLogger sshLogger) {
- this.hostname = hostname;
- this.username = username;
- if (password != null && new File(password).exists()) {
- this.identityPath = new File(password).getAbsolutePath();
- this.password = null;
- } else {
- this.password = password;
- this.identityPath = null;
- }
- this.sshLogger = sshLogger;
- }
-
- public void scpFileToRemote(String localFile, String remoteTargetDirectory) throws Exception {
- FileInputStream fis = null;
- try {
- System.out.println("SCP file " + localFile + " to " + remoteTargetDirectory);
-
- Session session = newJSchSession();
- session.connect();
-
- boolean ptimestamp = false;
-
- // exec 'scp -t rfile' remotely
- String command = "scp " + (ptimestamp ? "-p" : "") + " -t " + remoteTargetDirectory;
- Channel channel = session.openChannel("exec");
- ((ChannelExec) channel).setCommand(command);
-
- // get I/O streams for remote scp
- OutputStream out = channel.getOutputStream();
- InputStream in = channel.getInputStream();
-
- channel.connect();
-
- if (checkAck(in) != 0) {
- System.exit(0);
- }
-
- File _lfile = new File(localFile);
-
- if (ptimestamp) {
- command = "T " + (_lfile.lastModified() / 1000) + " 0";
- // The access time should be sent here,
- // but it is not accessible with JavaAPI ;-<
- command += (" " + (_lfile.lastModified() / 1000) + " 0\n");
- out.write(command.getBytes());
- out.flush();
- if (checkAck(in) != 0) {
- throw new Exception("Error in checkAck()");
- }
- }
-
- // send "C0644 filesize filename", where filename should not include '/'
- long filesize = _lfile.length();
- command = "C0644 " + filesize + " ";
- if (localFile.lastIndexOf("/") > 0) {
- command += localFile.substring(localFile.lastIndexOf("/") + 1);
- } else if (localFile.lastIndexOf(File.separator) > 0) {
- command += localFile.substring(localFile.lastIndexOf(File.separator) + 1);
- } else {
- command += localFile;
- }
- command += "\n";
- out.write(command.getBytes());
- out.flush();
- if (checkAck(in) != 0) {
- throw new Exception("Error in checkAck()");
- }
-
- // send a content of lfile
- fis = new FileInputStream(localFile);
- byte[] buf = new byte[1024];
- while (true) {
- int len = fis.read(buf, 0, buf.length);
- if (len <= 0)
- break;
- out.write(buf, 0, len); // out.flush();
- }
- fis.close();
- fis = null;
- // send '\0'
- buf[0] = 0;
- out.write(buf, 0, 1);
- out.flush();
- if (checkAck(in) != 0) {
- throw new Exception("Error in checkAck()");
- }
- out.close();
-
- channel.disconnect();
- session.disconnect();
- } catch (Exception e) {
- throw e;
- } finally {
- try {
- if (fis != null)
- fis.close();
- } catch (Exception ee) {
- }
- }
- }
-
- public SSHClientOutput execCommand(String command) throws Exception {
- return execCommand(command, 7200);
- }
-
- public SSHClientOutput execCommand(String command, int timeoutSeconds) throws Exception {
- try {
- System.out.println("[" + username + "@" + hostname + "] Execute command: " + command);
-
- StringBuffer text = new StringBuffer();
- int exitCode = -1;
-
- Session session = newJSchSession();
- session.connect();
-
- Channel channel = session.openChannel("exec");
- ((ChannelExec) channel).setCommand(command);
-
- channel.setInputStream(null);
-
- // channel.setOutputStream(System.out);
-
- ((ChannelExec) channel).setErrStream(System.err);
-
- InputStream in = channel.getInputStream();
- InputStream err = ((ChannelExec) channel).getErrStream();
-
- channel.connect();
-
- int timeout = timeoutSeconds;
- byte[] tmp = new byte[1024];
- while (true) {
- timeout--;
- while (in.available() > 0) {
- int i = in.read(tmp, 0, 1024);
- if (i < 0)
- break;
-
- String line = new String(tmp, 0, i);
- text.append(line);
- if (this.sshLogger != null) {
- this.sshLogger.log(line);
- }
- }
- while (err.available() > 0) {
- int i = err.read(tmp, 0, 1024);
- if (i < 0)
- break;
-
- String line = new String(tmp, 0, i);
- text.append(line);
- if (this.sshLogger != null) {
- this.sshLogger.log(line);
- }
- }
- if (channel.isClosed()) {
- if (in.available() > 0)
- continue;
- exitCode = channel.getExitStatus();
- System.out.println("[" + username + "@" + hostname + "] Command exit-status: " + exitCode);
-
- break;
- }
- try {
- Thread.sleep(1000);
- } catch (Exception ee) {
- throw ee;
- }
- if (timeout < 0)
- throw new Exception("Remote commmand not finished within " + timeoutSeconds + " seconds.");
- }
- channel.disconnect();
- session.disconnect();
- return new SSHClientOutput(exitCode, text.toString());
- } catch (Exception e) {
- throw e;
- }
- }
-
- private Session newJSchSession() throws JSchException {
- JSch jsch = new JSch();
- if (identityPath != null) {
- jsch.addIdentity(identityPath);
- }
-
- Session session = jsch.getSession(username, hostname, 22);
- if (password != null) {
- session.setPassword(password);
- }
- session.setConfig("StrictHostKeyChecking", "no");
- return session;
- }
-
- private int checkAck(InputStream in) throws IOException {
- int b = in.read();
- // b may be 0 for success,
- // 1 for error,
- // 2 for fatal error,
- // -1
- if (b == 0)
- return b;
- if (b == -1)
- return b;
-
- if (b == 1 || b == 2) {
- StringBuffer sb = new StringBuffer();
- int c;
- do {
- c = in.read();
- sb.append((char) c);
- } while (c != '\n');
- if (b == 1) { // error
- System.out.print(sb.toString());
- }
- if (b == 2) { // fatal error
- System.out.print(sb.toString());
- }
- }
- return b;
- }
-}
diff --git a/common/src/main/java/com/kylinolap/common/util/SSHClientOutput.java b/common/src/main/java/com/kylinolap/common/util/SSHClientOutput.java
deleted file mode 100644
index 40f2d2d7a0..0000000000
--- a/common/src/main/java/com/kylinolap/common/util/SSHClientOutput.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.common.util;
-
-/**
- * @author George Song (ysong1)
- *
- */
-
-public class SSHClientOutput {
- private String text;
- private int exitCode = -1;
-
- /**
- * @param text
- * @param exitCode
- */
- public SSHClientOutput(int exitCode, String text) {
- this.text = text;
- this.exitCode = exitCode;
- }
-
- /**
- * @return the text
- */
- public String getText() {
- return text.toString();
- }
-
- /**
- * @return the exitCode
- */
- public int getExitCode() {
- return exitCode;
- }
-
-}
diff --git a/common/src/main/java/com/kylinolap/common/util/SSHLogger.java b/common/src/main/java/com/kylinolap/common/util/SSHLogger.java
deleted file mode 100644
index ebf025c6aa..0000000000
--- a/common/src/main/java/com/kylinolap/common/util/SSHLogger.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.common.util;
-
-/**
- * @author ysong1
- *
- */
-public interface SSHLogger {
- public void log(String message);
-}
diff --git a/common/src/main/java/com/kylinolap/common/util/StringSplitter.java b/common/src/main/java/com/kylinolap/common/util/StringSplitter.java
deleted file mode 100644
index fc7b6ea94b..0000000000
--- a/common/src/main/java/com/kylinolap/common/util/StringSplitter.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.common.util;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * @author George Song (ysong1)
- *
- */
-public class StringSplitter {
- public static String[] split(String str, String delimiter) {
- // The optimized split function
- List<String> list = new ArrayList<String>();
- int index = 0, offset = 0;
- int l = delimiter.length();
- if (str.startsWith(delimiter)) {
- // in case the first field is empty
- list.add("");
- offset = offset + l;
- }
- while ((index = str.indexOf(delimiter, index + 1)) != -1) {
- list.add(str.substring(offset, index));
- offset = index + l;
- }
- // add the last field, or the str doesn't contain delimiter at all
- list.add(str.substring(offset));
- return list.toArray(new String[0]);
- }
-}
diff --git a/common/src/main/java/com/kylinolap/common/util/StringUtil.java b/common/src/main/java/com/kylinolap/common/util/StringUtil.java
deleted file mode 100644
index 6a3eaa3692..0000000000
--- a/common/src/main/java/com/kylinolap/common/util/StringUtil.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.common.util;
-
-import java.util.ArrayList;
-import java.util.Collection;
-
-/**
- * Created with IntelliJ IDEA. User: lukhan Date: 12/2/13 Time: 11:43 AM To
- * change this template use File | Settings | File Templates.
- */
-public class StringUtil {
-
- public static String[] filterSystemArgs(String args[]) {
- ArrayList<String> whatsLeft = new ArrayList<String>();
- for (String a : args) {
- if (a.startsWith("-D")) {
- String key;
- String value;
- int cut = a.indexOf('=');
- if (cut < 0) {
- key = a.substring(2);
- value = "";
- } else {
- key = a.substring(2, cut);
- value = a.substring(cut + 1);
- }
- System.setProperty(key, value);
- } else {
- whatsLeft.add(a);
- }
- }
- return (String[]) whatsLeft.toArray(new String[whatsLeft.size()]);
- }
-
- public static void toUpperCaseArray(String[] source, String[] target) {
- for (int i = 0; i < source.length; i++) {
- if (source[i] != null) {
- target[i] = source[i].toUpperCase();
- }
- }
- }
-
- public static String dropSuffix(String str, String suffix) {
- if (str.endsWith(suffix))
- return str.substring(0, str.length() - suffix.length());
- else
- return str;
- }
-
- public static String min(Collection<String> strs) {
- String min = null;
- for (String s : strs) {
- if (min == null || min.compareTo(s) > 0)
- min = s;
- }
- return min;
- }
-
- public static String max(Collection<String> strs) {
- String max = null;
- for (String s : strs) {
- if (max == null || max.compareTo(s) < 0)
- max = s;
- }
- return max;
- }
-
- public static String min(String s1, String s2) {
- if (s1 == null)
- return s2;
- else if (s2 == null)
- return s1;
- else
- return s1.compareTo(s2) < 0 ? s1 : s2;
- }
-
- public static String max(String s1, String s2) {
- if (s1 == null)
- return s2;
- else if (s2 == null)
- return s1;
- else
- return s1.compareTo(s2) > 0 ? s1 : s2;
- }
-}
diff --git a/common/src/main/java/org/apache/kylin/common/KylinConfig.java b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
new file mode 100644
index 0000000000..3a17aacec9
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -0,0 +1,765 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.SortedSet;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.restclient.RestClient;
+import org.apache.kylin.common.util.CliCommandExecutor;
+import org.apache.kylin.common.util.Log4jConfigurer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Sets;
+
+/**
+ * @author yangli9
+ */
+public class KylinConfig {
+
+ /*
+ * NO NEED TO DEFINE PUBLIC CONSTANTS FOR KEY NAMES!
+ *
+ * Kylin configuration is exposed as public methods. A client never need to access key names directly.
+ * Feel free to hard code key names, as long as they are encapsulated in this class. It reads better!
+ */
+
+ public static final String KYLIN_STORAGE_URL = "kylin.storage.url";
+
+ public static final String KYLIN_ES_CLUSTER_URL = "kylin.es.cluster.url";
+
+ public static final String KYLIN_METADATA_URL = "kylin.metadata.url";
+
+ public static final String KYLIN_REST_SERVERS = "kylin.rest.servers";
+
+ public static final String KYLIN_REST_TIMEZONE = "kylin.rest.timezone";
+
+ public static final String KYLIN_JOB_CONCURRENT_MAX_LIMIT = "kylin.job.concurrent.max.limit";
+
+ public static final String KYLIN_JOB_YARN_APP_REST_CHECK_URL = "kylin.job.yarn.app.rest.check.status.url";
+
+ public static final String KYLIN_JOB_YARN_APP_REST_CHECK_INTERVAL_SECONDS = "kylin.job.yarn.app.rest.check.interval.seconds";
+
+ public static final String KYLIN_TMP_HDFS_DIR = "kylin.tmp.hdfs.dir";
+
+ public static final String HIVE_DATABASE_FOR_INTERMEDIATE_TABLE = "kylin.job.hive.database.for.intermediatetable";
+
+ public static final String HIVE_TABLE_LOCATION_PREFIX = "hive.table.location.";
+
+ public static final String KYLIN_JOB_REMOTE_CLI_PASSWORD = "kylin.job.remote.cli.password";
+
+ public static final String KYLIN_JOB_REMOTE_CLI_USERNAME = "kylin.job.remote.cli.username";
+
+ public static final String KYLIN_JOB_REMOTE_CLI_HOSTNAME = "kylin.job.remote.cli.hostname";
+
+ public static final String KYLIN_JOB_REMOTE_CLI_WORKING_DIR = "kylin.job.remote.cli.working.dir";
+
+ public static final String KYLIN_JOB_CMD_EXTRA_ARGS = "kylin.job.cmd.extra.args";
+ /**
+ * Toggle to indicate whether to use hive for table flattening. Default
+ * true.
+ */
+ public static final String KYLIN_JOB_RUN_AS_REMOTE_CMD = "kylin.job.run.as.remote.cmd";
+
+ public static final String KYLIN_JOB_MAPREDUCE_DEFAULT_REDUCE_COUNT_RATIO = "kylin.job.mapreduce.default.reduce.count.ratio";
+
+ public static final String KYLIN_JOB_MAPREDUCE_DEFAULT_REDUCE_INPUT_MB = "kylin.job.mapreduce.default.reduce.input.mb";
+
+ public static final String KYLIN_JOB_MAPREDUCE_MAX_REDUCER_NUMBER = "kylin.job.mapreduce.max.reducer.number";
+
+ public static final String KYLIN_JOB_JAR = "kylin.job.jar";
+
+ public static final String COPROCESSOR_LOCAL_JAR = "kylin.coprocessor.local.jar";
+
+ public static final String KYLIN_JOB_LOG_DIR = "kylin.job.log.dir";
+
+ public static final String KYLIN_HDFS_WORKING_DIR = "kylin.hdfs.working.dir";
+
+ public static final String KYLIN_HBASE_CLUSTER_FS = "kylin.hbase.cluster.fs";
+
+ /**
+ * Kylin properties file
+ */
+ public static final String KYLIN_CONF_PROPERTIES_FILE = "kylin.properties";
+
+ public static final String MAIL_ENABLED = "mail.enabled";
+
+ public static final String MAIL_HOST = "mail.host";
+
+ public static final String MAIL_USERNAME = "mail.username";
+
+ public static final String MAIL_PASSWORD = "mail.password";
+
+ public static final String MAIL_SENDER = "mail.sender";
+
+ public static final String KYLIN_HOME = "KYLIN_HOME";
+
+ public static final String KYLIN_CONF = "KYLIN_CONF";
+
+ private static final Logger logger = LoggerFactory.getLogger(KylinConfig.class);
+
+ public static final String VERSION = "${project.version}";
+
+ public static final String HTABLE_DEFAULT_COMPRESSION_CODEC = "kylin.hbase.default.compression.codec";
+
+ public static final String HBASE_REGION_CUT_SMALL = "kylin.hbase.region.cut.small";
+ public static final String HBASE_REGION_CUT_MEDIUM = "kylin.hbase.region.cut.medium";
+ public static final String HBASE_REGION_CUT_LARGE = "kylin.hbase.region.cut.large";
+
+ public static final String HBASE_REGION_COUNT_MIN = "kylin.hbase.region.count.min";
+ public static final String HBASE_REGION_COUNT_MAX = "kylin.hbase.region.count.max";
+
+ // static cached instances
+ private static KylinConfig ENV_INSTANCE = null;
+
+ public static KylinConfig getInstanceFromEnv() {
+ if (ENV_INSTANCE == null) {
+ try {
+ KylinConfig config = loadKylinConfig();
+ ENV_INSTANCE = config;
+ } catch (IllegalArgumentException e) {
+ throw new IllegalStateException("Failed to find KylinConfig ", e);
+ }
+ }
+ return ENV_INSTANCE;
+ }
+
+ public static void destoryInstance() {
+ ENV_INSTANCE = null;
+ }
+
+ public static enum UriType {
+ PROPERTIES_FILE, REST_ADDR, LOCAL_FOLDER
+ }
+
+ private static UriType decideUriType(String metaUri) {
+
+ try {
+ File file = new File(metaUri);
+ if (file.exists() || metaUri.contains("/")) {
+ if (file.exists() == false) {
+ file.mkdirs();
+ }
+ if (file.isDirectory()) {
+ return UriType.LOCAL_FOLDER;
+ } else if (file.isFile()) {
+ if (file.getName().equalsIgnoreCase(KYLIN_CONF_PROPERTIES_FILE)) {
+ return UriType.PROPERTIES_FILE;
+ } else {
+ throw new IllegalStateException("Metadata uri : " + metaUri + " is a local file but not kylin.properties");
+ }
+ } else {
+ throw new IllegalStateException("Metadata uri : " + metaUri + " looks like a file but it's neither a file nor a directory");
+ }
+ } else {
+ if (RestClient.matchFullRestPattern(metaUri))
+ return UriType.REST_ADDR;
+ else
+ throw new IllegalStateException("Metadata uri : " + metaUri + " is not a valid REST URI address");
+ }
+ } catch (Exception e) {
+ logger.info(e.getLocalizedMessage());
+ throw new IllegalStateException("Metadata uri : " + metaUri + " is not recognized");
+ }
+ }
+
+ public static KylinConfig createInstanceFromUri(String uri) {
+ /**
+ * --hbase: 1. PROPERTIES_FILE: path to kylin.properties 2. REST_ADDR:
+ * rest service resource, format: user:password@host:port --local: 1.
+ * LOCAL_FOLDER: path to resource folder
+ */
+ UriType uriType = decideUriType(uri);
+ logger.info("The URI " + uri + " is recognized as " + uriType);
+
+ if (uriType == UriType.LOCAL_FOLDER) {
+ KylinConfig config = new KylinConfig();
+ config.setMetadataUrl(uri);
+ return config;
+ } else if (uriType == UriType.PROPERTIES_FILE) {
+ KylinConfig config;
+ try {
+ config = new KylinConfig();
+ InputStream is = new FileInputStream(uri);
+ config.reloadKylinConfig(is);
+ is.close();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return config;
+ } else {// rest_addr
+ try {
+ KylinConfig config = new KylinConfig();
+ RestClient client = new RestClient(uri);
+ String propertyText = client.getKylinProperties();
+ InputStream is = IOUtils.toInputStream(propertyText);
+ config.reloadKylinConfig(is);
+ is.close();
+ return config;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ public static KylinConfig getKylinConfigFromInputStream(InputStream is) {
+ KylinConfig config = new KylinConfig();
+ config.reloadKylinConfig(is);
+ return config;
+ }
+
+ // ============================================================================
+
+ /**
+ * Find config from environment. The Search process: 1. Check the
+ * $KYLIN_CONF/kylin.properties 2. Check the $KYLIN_HOME/conf/kylin.properties
+ *
+ * @return
+ */
+ private static KylinConfig loadKylinConfig() {
+ Log4jConfigurer.initLogger();
+
+ InputStream is = getKylinPropertiesAsInputSteam();
+ if (is == null) {
+ throw new IllegalArgumentException("Failed to load kylin config");
+ }
+ KylinConfig config = new KylinConfig();
+ config.reloadKylinConfig(is);
+
+ return config;
+ }
+
+ private PropertiesConfiguration kylinConfig = new PropertiesConfiguration();
+
+ private String metadataUrl;
+ private String storageUrl;
+ private String esClusterUrl;
+ public CliCommandExecutor getCliCommandExecutor() throws IOException {
+ CliCommandExecutor exec = new CliCommandExecutor();
+ if (getRunAsRemoteCommand()) {
+ exec.setRunAtRemote(getRemoteHadoopCliHostname(), getRemoteHadoopCliUsername(), getRemoteHadoopCliPassword());
+ }
+ return exec;
+ }
+
+ // ============================================================================
+
+ public boolean isHiveReroutingEnabled() {
+ return Boolean.parseBoolean(getOptional("kylin.route.hive.enabled", "false"));
+ }
+
+ public String getHiveRerouteUrl() {
+ return getOptional("kylin.route.hive.url", "jdbc:hive2://");
+ }
+
+ public String getHiveRerouteUsername() {
+ return getOptional("kylin.route.hive.username", "");
+ }
+
+ public String getHiveReroutePassword() {
+ return getOptional("kylin.route.hive.password", "");
+ }
+
+ public String getStorageUrl() {
+ return storageUrl;
+ }
+ public String getKylinEsClusterUrl(){
+ return esClusterUrl;
+ }
+ /** Use the hive reroute feature instead */
+ @Deprecated
+ public String getHiveUrl() {
+ return getOptional("hive.url", "");
+ }
+
+ /** Use the hive reroute feature instead */
+ @Deprecated
+ public String getHiveUser() {
+ return getOptional("hive.user", "");
+ }
+
+ /** Use the hive reroute feature instead */
+ @Deprecated
+ public String getHivePassword() {
+ return getOptional("hive.password", "");
+ }
+
+ public String getHdfsWorkingDirectory() {
+ String root = getRequired(KYLIN_HDFS_WORKING_DIR);
+ if (!root.endsWith("/")) {
+ root += "/";
+ }
+ return root + getMetadataUrlPrefix() + "/";
+ }
+
+ public String getHBaseClusterFs() {
+ return getOptional(KYLIN_HBASE_CLUSTER_FS, "");
+ }
+
+ public String getKylinJobLogDir() {
+ return getOptional(KYLIN_JOB_LOG_DIR, "/tmp/kylin/logs");
+ }
+
+ public String getKylinJobJarPath() {
+ final String jobJar = getOptional(KYLIN_JOB_JAR);
+ if (StringUtils.isNotEmpty(jobJar)) {
+ return jobJar;
+ }
+ String kylinHome = getKylinHome();
+ if (StringUtils.isEmpty(kylinHome)) {
+ return "";
+ }
+ return getFileName(kylinHome + File.separator + "lib", JOB_JAR_NAME_PATTERN);
+ }
+
+ public String getKylinJobMRLibDir() {
+ return getOptional("kylin.job.mr.lib.dir", "");
+ }
+
+ public void overrideKylinJobJarPath(String path) {
+ logger.info("override " + KYLIN_JOB_JAR + " to " + path);
+ System.setProperty(KYLIN_JOB_JAR, path);
+ }
+
+ private static final Pattern COPROCESSOR_JAR_NAME_PATTERN = Pattern.compile("kylin-coprocessor-(.+)\\.jar");
+ private static final Pattern JOB_JAR_NAME_PATTERN = Pattern.compile("kylin-job-(.+)\\.jar");
+
+ public String getCoprocessorLocalJar() {
+ final String coprocessorJar = getOptional(COPROCESSOR_LOCAL_JAR);
+ if (StringUtils.isNotEmpty(coprocessorJar)) {
+ return coprocessorJar;
+ }
+ String kylinHome = getKylinHome();
+ if (StringUtils.isEmpty(kylinHome)) {
+ throw new RuntimeException("getCoprocessorLocalJar needs KYLIN_HOME");
+ }
+ return getFileName(kylinHome + File.separator + "lib", COPROCESSOR_JAR_NAME_PATTERN);
+ }
+
+ private static String getFileName(String homePath, Pattern pattern) {
+ File home = new File(homePath);
+ SortedSet<String> files = Sets.newTreeSet();
+ if (home.exists() && home.isDirectory()) {
+ for (File file : home.listFiles()) {
+ final Matcher matcher = pattern.matcher(file.getName());
+ if (matcher.matches()) {
+ files.add(file.getAbsolutePath());
+ }
+ }
+ }
+ if (files.isEmpty()) {
+ throw new RuntimeException("cannot find " + pattern.toString() + " in " + homePath);
+ } else {
+ return files.last();
+ }
+ }
+
+ public void overrideCoprocessorLocalJar(String path) {
+ logger.info("override " + COPROCESSOR_LOCAL_JAR + " to " + path);
+ System.setProperty(COPROCESSOR_LOCAL_JAR, path);
+ }
+
+ public double getDefaultHadoopJobReducerInputMB() {
+ return Double.parseDouble(getOptional(KYLIN_JOB_MAPREDUCE_DEFAULT_REDUCE_INPUT_MB, "500"));
+ }
+
+ public double getDefaultHadoopJobReducerCountRatio() {
+ return Double.parseDouble(getOptional(KYLIN_JOB_MAPREDUCE_DEFAULT_REDUCE_COUNT_RATIO, "1.0"));
+ }
+
+ public int getHadoopJobMaxReducerNumber() {
+ return Integer.parseInt(getOptional(KYLIN_JOB_MAPREDUCE_MAX_REDUCER_NUMBER, "5000"));
+ }
+
+ public boolean getRunAsRemoteCommand() {
+ return Boolean.parseBoolean(getOptional(KYLIN_JOB_RUN_AS_REMOTE_CMD));
+ }
+
+ public String getRemoteHadoopCliHostname() {
+ return getOptional(KYLIN_JOB_REMOTE_CLI_HOSTNAME);
+ }
+
+ public String getRemoteHadoopCliUsername() {
+ return getOptional(KYLIN_JOB_REMOTE_CLI_USERNAME);
+ }
+
+ public String getRemoteHadoopCliPassword() {
+ return getOptional(KYLIN_JOB_REMOTE_CLI_PASSWORD);
+ }
+
+ public String getCliWorkingDir() {
+ return getOptional(KYLIN_JOB_REMOTE_CLI_WORKING_DIR);
+ }
+
+ public String getMapReduceCmdExtraArgs() {
+ return getOptional(KYLIN_JOB_CMD_EXTRA_ARGS);
+ }
+
+ public String getOverrideHiveTableLocation(String table) {
+ return getOptional(HIVE_TABLE_LOCATION_PREFIX + table.toUpperCase());
+ }
+
+ public String getTempHDFSDir() {
+ return getOptional(KYLIN_TMP_HDFS_DIR, "/tmp/kylin");
+ }
+
+ public String getYarnStatusCheckUrl() {
+ return getOptional(KYLIN_JOB_YARN_APP_REST_CHECK_URL, null);
+ }
+
+ public int getYarnStatusCheckIntervalSeconds() {
+ return Integer.parseInt(getOptional(KYLIN_JOB_YARN_APP_REST_CHECK_INTERVAL_SECONDS, "60"));
+ }
+
+ public int getMaxConcurrentJobLimit() {
+ return Integer.parseInt(getOptional(KYLIN_JOB_CONCURRENT_MAX_LIMIT, "10"));
+ }
+
+ public String getTimeZone() {
+ return getOptional(KYLIN_REST_TIMEZONE, "PST");
+ }
+
+ public String[] getRestServers() {
+ return getOptionalStringArray(KYLIN_REST_SERVERS);
+ }
+
+ public String getAdminDls() {
+ return getOptional("kylin.job.admin.dls", null);
+ }
+
+ public long getJobStepTimeout() {
+ return Long.parseLong(getOptional("kylin.job.step.timeout", String.valueOf(2 * 60 * 60)));
+ }
+
+ public String getServerMode() {
+ return this.getOptional("kylin.server.mode", "all");
+ }
+
+ public int getDictionaryMaxCardinality() {
+ return Integer.parseInt(getOptional("kylin.dictionary.max.cardinality", "5000000"));
+ }
+
+ public int getTableSnapshotMaxMB() {
+ return Integer.parseInt(getOptional("kylin.table.snapshot.max_mb", "300"));
+ }
+
+ public int getScanThreshold() {
+ return Integer.parseInt(getOptional("kylin.query.scan.threshold", "10000000"));
+ }
+
+ public boolean getQueryRunLocalCoprocessor() {
+ return Boolean.parseBoolean(getOptional("kylin.query.run.local.coprocessor", "false"));
+ }
+
+ public Long getQueryDurationCacheThreshold() {
+ return Long.parseLong(this.getOptional("kylin.query.cache.threshold.duration", String.valueOf(2000)));
+ }
+
+ public Long getQueryScanCountCacheThreshold() {
+ return Long.parseLong(this.getOptional("kylin.query.cache.threshold.scancount", String.valueOf(10 * 1024)));
+ }
+
+ public boolean isQuerySecureEnabled() {
+ return Boolean.parseBoolean(this.getOptional("kylin.query.security.enabled", "false"));
+ }
+
+ public boolean isQueryCacheEnabled() {
+ return Boolean.parseBoolean(this.getOptional("kylin.query.cache.enabled", "true"));
+ }
+
+ public long getQueryMemBudget() {
+ return Long.parseLong(this.getOptional("kylin.query.mem.budget", String.valueOf(3L * 1024 * 1024 * 1024)));
+ }
+
+ public int getHBaseKeyValueSize() {
+ return Integer.parseInt(this.getOptional("kylin.hbase.client.keyvalue.maxsize", "10485760"));
+ }
+
+ public int getHBaseScanCacheRows() {
+ return Integer.parseInt(this.getOptional("kylin.hbase.scan.cache_rows", "1024"));
+ }
+
+ public int getHBaseScanMaxResultSize() {
+ return Integer.parseInt(this.getOptional("kylin.hbase.scan.max_result_size", "" + (5 * 1024 * 1024))); // 5 MB
+ }
+
+ public String getHbaseDefaultCompressionCodec() {
+ return getOptional(HTABLE_DEFAULT_COMPRESSION_CODEC, "");
+
+ }
+
+ private String getOptional(String prop) {
+ final String property = System.getProperty(prop);
+ return property != null ? property : kylinConfig.getString(prop);
+ }
+
+ private String[] getOptionalStringArray(String prop) {
+ final String property = System.getProperty(prop);
+ if (!StringUtils.isBlank(property))
+ return property.split("\\s*,\\s*");
+
+ return kylinConfig.getStringArray(prop);
+ }
+
+ private String getOptional(String prop, String dft) {
+ final String property = System.getProperty(prop);
+ return property != null ? property : kylinConfig.getString(prop, dft);
+ }
+
+ private String getRequired(String prop) {
+ final String property = System.getProperty(prop);
+ if (property != null) {
+ return property;
+ }
+ String r = kylinConfig.getString(prop);
+ if (StringUtils.isEmpty(r)) {
+ throw new IllegalArgumentException("missing '" + prop + "' in conf/kylin_instance.properties");
+ }
+ return r;
+ }
+
+ void reloadKylinConfig(InputStream is) {
+ PropertiesConfiguration config = new PropertiesConfiguration();
+ try {
+ config.load(is);
+ } catch (ConfigurationException e) {
+ throw new RuntimeException("Cannot load kylin config.", e);
+ } finally {
+ try {
+ is.close();
+ } catch (IOException e) {
+ logger.error("Failed to close inputstream.", e);
+ }
+ }
+ this.kylinConfig = config;
+ this.metadataUrl = getOptional(KYLIN_METADATA_URL);
+ this.storageUrl = getOptional(KYLIN_STORAGE_URL);
+ this.esClusterUrl = getOptional(KYLIN_ES_CLUSTER_URL);
+ }
+
+ public void writeProperties(File file) throws IOException {
+ try {
+ kylinConfig.save(file);
+ } catch (ConfigurationException ex) {
+ throw new IOException("Error writing KylinConfig to " + file, ex);
+ }
+ }
+
+ public static String getKylinHome() {
+ String kylinHome = System.getenv(KYLIN_HOME);
+ if (StringUtils.isEmpty(kylinHome)) {
+ logger.warn("KYLIN_HOME was not set");
+ return kylinHome;
+ }
+ return kylinHome;
+ }
+
+ public void printProperties() throws IOException {
+ try {
+ kylinConfig.save(System.out);
+ } catch (ConfigurationException ex) {
+ throw new IOException("Error printing KylinConfig", ex);
+ }
+ }
+
+ private static File getKylinProperties() {
+ String kylinConfHome = System.getProperty(KYLIN_CONF);
+ if (!StringUtils.isEmpty(kylinConfHome)) {
+ logger.info("Use KYLIN_CONF=" + kylinConfHome);
+ return getKylinPropertiesFile(kylinConfHome);
+ }
+
+ logger.warn("KYLIN_CONF property was not set, will seek KYLIN_HOME env variable");
+
+ String kylinHome = getKylinHome();
+ if (StringUtils.isEmpty(kylinHome))
+ throw new RuntimeException("Didn't find KYLIN_CONF or KYLIN_HOME, please set one of them");
+
+ String path = kylinHome + File.separator + "conf";
+ return getKylinPropertiesFile(path);
+
+ }
+
+ public static InputStream getKylinPropertiesAsInputSteam() {
+ File propFile = getKylinProperties();
+ if (propFile == null || !propFile.exists()) {
+ logger.error("fail to locate kylin.properties");
+ throw new RuntimeException("fail to locate kylin.properties");
+ }
+
+ File overrideFile = new File(propFile.getParentFile(), propFile.getName() + ".override");
+ if (overrideFile.exists()) {
+ try {
+ PropertiesConfiguration conf = new PropertiesConfiguration();
+ conf.load(propFile);
+ PropertiesConfiguration override = new PropertiesConfiguration();
+ override.load(overrideFile);
+ conf.copy(override);
+ ByteArrayOutputStream bout = new ByteArrayOutputStream();
+ conf.save(bout);
+ return new ByteArrayInputStream(bout.toByteArray());
+ } catch (ConfigurationException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ try {
+ return new FileInputStream(propFile);
+ } catch (FileNotFoundException e) {
+ logger.error("this should not happen");
+ throw new RuntimeException(e);
+ }
+
+ }
+
+ /**
+ * Check if there is kylin.properties exist
+ *
+ * @param path
+ * @return the properties file
+ */
+ private static File getKylinPropertiesFile(String path) {
+ if (path == null) {
+ return null;
+ }
+
+ return new File(path, KYLIN_CONF_PROPERTIES_FILE);
+ }
+
+ public String getMetadataUrl() {
+ return metadataUrl;
+ }
+
+ public String getMetadataUrlPrefix() {
+ String hbaseMetadataUrl = getMetadataUrl();
+ String defaultPrefix = "kylin_metadata";
+
+ if (hbaseMetadataUrl.indexOf("@hbase") > 0) {
+ int cut = hbaseMetadataUrl.indexOf('@');
+ String tmp = cut < 0 ? defaultPrefix : hbaseMetadataUrl.substring(0, cut);
+ return tmp;
+ } else {
+ return defaultPrefix;
+ }
+ }
+
+ public void setMetadataUrl(String metadataUrl) {
+ kylinConfig.setProperty(KYLIN_METADATA_URL, metadataUrl);
+ this.metadataUrl = metadataUrl;
+ }
+
+ public void setStorageUrl(String storageUrl) {
+ kylinConfig.setProperty(KYLIN_STORAGE_URL, storageUrl);
+ this.storageUrl = storageUrl;
+ }
+ public void setKylinEsClusterUrl(String esClusterUrl) {
+ kylinConfig.setProperty(KYLIN_ES_CLUSTER_URL, esClusterUrl);
+ this.esClusterUrl = esClusterUrl;
+ }
+ public String getHiveDatabaseForIntermediateTable() {
+ return this.getOptional(HIVE_DATABASE_FOR_INTERMEDIATE_TABLE, "default");
+ }
+
+ public void setRunAsRemoteCommand(String v) {
+ kylinConfig.setProperty(KYLIN_JOB_RUN_AS_REMOTE_CMD, v);
+ }
+
+ public void setRemoteHadoopCliHostname(String v) {
+ kylinConfig.setProperty(KYLIN_JOB_REMOTE_CLI_HOSTNAME, v);
+ }
+
+ public void setRemoteHadoopCliUsername(String v) {
+ kylinConfig.setProperty(KYLIN_JOB_REMOTE_CLI_USERNAME, v);
+ }
+
+ public void setRemoteHadoopCliPassword(String v) {
+ kylinConfig.setProperty(KYLIN_JOB_REMOTE_CLI_PASSWORD, v);
+ }
+
+ public int getHBaseRegionCountMin() {
+ return Integer.parseInt(getOptional(HBASE_REGION_COUNT_MIN, "1"));
+ }
+
+ public int getHBaseRegionCountMax() {
+ return Integer.parseInt(getOptional(HBASE_REGION_COUNT_MAX, "500"));
+ }
+
+ public int getHBaseRegionCut(String capacity) {
+ String cut;
+ switch (capacity) {
+ case "SMALL":
+ cut = getProperty(HBASE_REGION_CUT_SMALL, "10");
+ break;
+ case "MEDIUM":
+ cut = getProperty(HBASE_REGION_CUT_MEDIUM, "20");
+ break;
+ case "LARGE":
+ cut = getProperty(HBASE_REGION_CUT_LARGE, "100");
+ break;
+ default:
+ throw new IllegalArgumentException("Capacity not recognized: " + capacity);
+ }
+
+ return Integer.valueOf(cut);
+ }
+
+ public String getProperty(String key, String defaultValue) {
+ return kylinConfig.getString(key, defaultValue);
+ }
+
+ /**
+ * Set a new key:value into the kylin config.
+ *
+ * @param key
+ * @param value
+ */
+ public void setProperty(String key, String value) {
+ logger.info("Kylin Config was updated with " + key + " : " + value);
+ kylinConfig.setProperty(key, value);
+ }
+
+ public String getConfigAsString() throws IOException {
+ try {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ kylinConfig.save(baos);
+ String content = baos.toString();
+ return content;
+ } catch (ConfigurationException ex) {
+ throw new IOException("Error writing KylinConfig to String", ex);
+ }
+ }
+
+ public String toString() {
+ return getMetadataUrl();
+ }
+
+}
diff --git a/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java b/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
new file mode 100644
index 0000000000..609529517c
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.debug;
+
+import java.util.Map;
+
+/**
+ */
+public class BackdoorToggles {
+
+ private static final ThreadLocal<Map<String, String>> _backdoorToggles = new ThreadLocal<Map<String, String>>();
+
+ public static void setToggles(Map<String, String> toggles) {
+ _backdoorToggles.set(toggles);
+ }
+
+ public static String getObserverBehavior() {
+ return getString(DEBUG_TOGGLE_OBSERVER_BEHAVIOR);
+ }
+
+ public static boolean getDisableFuzzyKey() {
+ return getBoolean(DEBUG_TOGGLE_DISABLE_FUZZY_KEY);
+ }
+
+ public static boolean getRunLocalCoprocessor() {
+ return getBoolean(DEBUG_TOGGLE_LOCAL_COPROCESSOR);
+ }
+
+ private static String getString(String key) {
+ Map<String, String> toggles = _backdoorToggles.get();
+ if (toggles == null) {
+ return null;
+ } else {
+ return toggles.get(key);
+ }
+ }
+
+ private static boolean getBoolean(String key) {
+ return "true".equals(getString(key));
+ }
+
+ public static void cleanToggles() {
+ _backdoorToggles.remove();
+ }
+
+ /**
+ * set DEBUG_TOGGLE_DISABLE_FUZZY_KEY=true to disable fuzzy key for debug/profile usage
+ *
+ *
+ *
+ example:
+
+ "backdoorToggles": {
+ "DEBUG_TOGGLE_DISABLE_FUZZY_KEY": "true"
+ }
+
+ */
+ public final static String DEBUG_TOGGLE_DISABLE_FUZZY_KEY = "DEBUG_TOGGLE_DISABLE_FUZZY_KEY";
+
+ /**
+ * set DEBUG_TOGGLE_OBSERVER_BEHAVIOR=SCAN/SCAN_FILTER/SCAN_FILTER_AGGR to control observer behavior for debug/profile usage
+ *
+ example:
+
+ "backdoorToggles": {
+ "DEBUG_TOGGLE_OBSERVER_BEHAVIOR": "SCAN"
+ }
+
+ */
+ public final static String DEBUG_TOGGLE_OBSERVER_BEHAVIOR = "DEBUG_TOGGLE_OBSERVER_BEHAVIOR";
+
+ /**
+ * set DEBUG_TOGGLE_LOCAL_COPROCESSOR=true to run coprocessor at client side (not in HBase region server)
+ *
+ example:
+
+ "backdoorToggles": {
+ "DEBUG_TOGGLE_LOCAL_COPROCESSOR": "true"
+ }
+
+ */
+ public final static String DEBUG_TOGGLE_LOCAL_COPROCESSOR = "DEBUG_TOGGLE_LOCAL_COPROCESSOR";
+}
diff --git a/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java b/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
new file mode 100644
index 0000000000..5ee323ecda
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
@@ -0,0 +1,291 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.hll;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.util.Arrays;
+
+import org.apache.kylin.common.util.BytesUtil;
+
+import com.google.common.hash.HashFunction;
+import com.google.common.hash.Hashing;
+
+/**
+ * About compression, test on HLLC data shows
+ *
+ * - LZF compression ratio is around 65%-80%, fast
+ * - GZIP compression ratio is around 41%-46%, very slow
+ *
+ * @author yangli9
+ */
+public class HyperLogLogPlusCounter implements Comparable<HyperLogLogPlusCounter> {
+
+ private final int p;
+ private final int m;
+ private final HashFunction hashFunc;
+ byte[] registers;
+
+ public HyperLogLogPlusCounter() {
+ this(10);
+ }
+
+ public HyperLogLogPlusCounter(int p) {
+ this(p, Hashing.murmur3_128());
+ }
+
+ public HyperLogLogPlusCounter(HyperLogLogPlusCounter another) {
+ this(another.p, another.hashFunc);
+ merge(another);
+ }
+
+ /** The larger p is, the more storage (2^p bytes), the better accuracy */
+ private HyperLogLogPlusCounter(int p, HashFunction hashFunc) {
+ this.p = p;
+ this.m = 1 << p;//(int) Math.pow(2, p);
+ this.hashFunc = hashFunc;
+ this.registers = new byte[m];
+ }
+
+ public void clear() {
+ byte zero = (byte) 0;
+ Arrays.fill(registers, zero);
+ }
+
+ public void add(int value) {
+ add(hashFunc.hashInt(value).asLong());
+ }
+
+ public void add(String value) {
+ add(hashFunc.hashString(value, Charset.defaultCharset()).asLong());
+ }
+
+ public void add(byte[] value) {
+ add(hashFunc.hashBytes(value).asLong());
+ }
+
+ public void add(byte[] value, int offset, int length) {
+ add(hashFunc.hashBytes(value, offset, length).asLong());
+ }
+
+ protected void add(long hash) {
+ int bucketMask = m - 1;
+ int bucket = (int) (hash & bucketMask);
+ int firstOnePos = Long.numberOfLeadingZeros(hash | bucketMask) + 1;
+
+ if (firstOnePos > registers[bucket])
+ registers[bucket] = (byte) firstOnePos;
+ }
+
+ public void merge(HyperLogLogPlusCounter another) {
+ assert this.p == another.p;
+ assert this.hashFunc == another.hashFunc;
+
+ for (int i = 0; i < m; i++) {
+ if (registers[i] < another.registers[i])
+ registers[i] = another.registers[i];
+ }
+ }
+
+ public long getCountEstimate() {
+ return new HLLCSnapshot(this).getCountEstimate();
+ }
+
+ public int getMemBytes() {
+ return 12 + m;
+ }
+
+ public double getErrorRate() {
+ return 1.04 / Math.sqrt(m);
+ }
+
+ private int size() {
+ int size = 0;
+ for (int i = 0; i < m; i++) {
+ if (registers[i] > 0)
+ size++;
+ }
+ return size;
+ }
+
+ // ============================================================================
+
+ // a memory efficient snapshot of HLL registers which can yield count
+ // estimate later
+ public static class HLLCSnapshot {
+ byte p;
+ double registerSum;
+ int zeroBuckets;
+
+ public HLLCSnapshot(HyperLogLogPlusCounter hllc) {
+ p = (byte) hllc.p;
+ registerSum = 0;
+ zeroBuckets = 0;
+
+ byte[] registers = hllc.registers;
+ for (int i = 0; i < hllc.m; i++) {
+ if (registers[i] == 0) {
+ registerSum++;
+ zeroBuckets++;
+ } else {
+ registerSum += 1.0 / (1 << registers[i]);
+ }
+ }
+ }
+
+ public long getCountEstimate() {
+ int m = (int) Math.pow(2, p);
+ double alpha = 1 / (2 * Math.log(2) * (1 + (3 * Math.log(2) - 1) / m));
+ double alphaMM = alpha * m * m;
+ double estimate = alphaMM / registerSum;
+
+ // small cardinality adjustment
+ if (zeroBuckets >= m * 0.07) { // (reference presto's HLL impl)
+ estimate = m * Math.log(m * 1.0 / zeroBuckets);
+ } else if (HyperLogLogPlusTable.isBiasCorrection(m, estimate)) {
+ estimate = HyperLogLogPlusTable.biasCorrection(p, estimate);
+ }
+
+ return Math.round(estimate);
+ }
+ }
+
+ // ============================================================================
+
+ public void writeRegisters(final ByteBuffer out) throws IOException {
+
+ final int indexLen = getRegisterIndexSize();
+ int size = size();
+
+ // decide output scheme -- map (3*size bytes) or array (2^p bytes)
+ byte scheme;
+ if ((indexLen + 1) * size < m)
+ scheme = 0; // map
+ else
+ scheme = 1; // array
+ out.put(scheme);
+
+ if (scheme == 0) { // map scheme
+ BytesUtil.writeVInt(size, out);
+ for (int i = 0; i < m; i++) {
+ if (registers[i] > 0) {
+ BytesUtil.writeUnsigned(i, indexLen, out);
+ out.put(registers[i]);
+ }
+ }
+ } else { // array scheme
+ for (int i = 0; i < m; i++) {
+ out.put(registers[i]);
+ }
+ }
+ }
+
+ public void readRegisters(ByteBuffer in) throws IOException {
+ byte scheme = in.get();
+
+ if (scheme == 0) { // map scheme
+ clear();
+ int size = BytesUtil.readVInt(in);
+ if (size > m)
+ throw new IllegalArgumentException("register size (" + size + ") cannot be larger than m (" + m + ")");
+ int indexLen = getRegisterIndexSize();
+ for (int i = 0; i < size; i++) {
+ int key = BytesUtil.readUnsigned(in, indexLen);
+ registers[key] = in.get();
+ }
+ } else { // array scheme
+ in.get(registers);
+ }
+ }
+
+ public void writeRegistersArray(final ByteBuffer out) {
+ out.put(this.registers);
+ }
+
+ public void readRegistersArray(ByteBuffer in) {
+ in.get(registers, 0, m);
+ }
+
+ private int getRegisterIndexSize() {
+ return (p - 1) / 8 + 1; // 2 when p=16, 3 when p=17
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((hashFunc == null) ? 0 : hashFunc.hashCode());
+ result = prime * result + p;
+ result = prime * result + Arrays.hashCode(registers);
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ HyperLogLogPlusCounter other = (HyperLogLogPlusCounter) obj;
+ if (hashFunc == null) {
+ if (other.hashFunc != null)
+ return false;
+ } else if (!hashFunc.equals(other.hashFunc))
+ return false;
+ if (p != other.p)
+ return false;
+ if (!Arrays.equals(registers, other.registers))
+ return false;
+ return true;
+ }
+
+ @Override
+ public int compareTo(HyperLogLogPlusCounter o) {
+ if (o == null)
+ return 1;
+
+ long e1 = this.getCountEstimate();
+ long e2 = o.getCountEstimate();
+
+ if (e1 == e2)
+ return 0;
+ else if (e1 > e2)
+ return 1;
+ else
+ return -1;
+ }
+
+ public static void main(String[] args) throws IOException {
+ dumpErrorRates();
+ }
+
+ static void dumpErrorRates() {
+ for (int p = 10; p <= 18; p++) {
+ double rate = new HyperLogLogPlusCounter(p).getErrorRate();
+ double er = Math.round(rate * 10000) / 100D;
+ double er2 = Math.round(rate * 2 * 10000) / 100D;
+ double er3 = Math.round(rate * 3 * 10000) / 100D;
+ long size = Math.round(Math.pow(2, p));
+ System.out.println("HLLC" + p + ",\t" + size + " bytes,\t68% err<" + er + "%" + ",\t95% err<" + er2 + "%" + ",\t99.7% err<" + er3 + "%");
+ }
+ }
+}
diff --git a/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusTable.java b/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusTable.java
new file mode 100644
index 0000000000..0061e9315c
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusTable.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.hll;
+
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+/**
+ * @author xjiang
+ *
+ */
+public class HyperLogLogPlusTable {
+
+ // threshold and bias data taken from google's bias correction data set:
+ // https://docs.google.com/document/d/1gyjfMHy43U9OWBXxfaeG-3MjGzejW1dlpyMwEYAAWEI/view?fullscreen#
+ public static final double[] thresholdData = { 10, 20, 40, 80, 220, 400, 900, 1800, 3100, 6500, 15500, 20000, 50000, 120000, 350000 };
+
+ public static final double[][] rawEstimateData = {
+ // precision 4
+ { 11, 11.717, 12.207, 12.7896, 13.2882, 13.8204, 14.3772, 14.9342, 15.5202, 16.161, 16.7722, 17.4636, 18.0396, 18.6766, 19.3566, 20.0454, 20.7936, 21.4856, 22.2666, 22.9946, 23.766, 24.4692, 25.3638, 26.0764, 26.7864, 27.7602, 28.4814, 29.433, 30.2926, 31.0664, 31.9996, 32.7956, 33.5366, 34.5894, 35.5738, 36.2698, 37.3682, 38.0544, 39.2342, 40.0108, 40.7966, 41.9298, 42.8704, 43.6358, 44.5194, 45.773, 46.6772, 47.6174, 48.4888, 49.3304, 50.2506, 51.4996, 52.3824, 53.3078, 54.3984, 55.5838, 56.6618, 57.2174, 58.3514, 59.0802, 60.1482, 61.0376, 62.3598, 62.8078, 63.9744, 64.914, 65.781, 67.1806, 68.0594, 68.8446, 69.7928, 70.8248, 71.8324, 72.8598, 73.6246, 74.7014, 75.393, 76.6708, 77.2394, },
+ // precision 5
+ { 23, 23.1194, 23.8208, 24.2318, 24.77, 25.2436, 25.7774, 26.2848, 26.8224, 27.3742, 27.9336, 28.503, 29.0494, 29.6292, 30.2124, 30.798, 31.367, 31.9728, 32.5944, 33.217, 33.8438, 34.3696, 35.0956, 35.7044, 36.324, 37.0668, 37.6698, 38.3644, 39.049, 39.6918, 40.4146, 41.082, 41.687, 42.5398, 43.2462, 43.857, 44.6606, 45.4168, 46.1248, 46.9222, 47.6804, 48.447, 49.3454, 49.9594, 50.7636, 51.5776, 52.331, 53.19, 53.9676, 54.7564, 55.5314, 56.4442, 57.3708, 57.9774, 58.9624, 59.8796, 60.755, 61.472, 62.2076, 63.1024, 63.8908, 64.7338, 65.7728, 66.629, 67.413, 68.3266, 69.1524, 70.2642, 71.1806, 72.0566, 72.9192, 73.7598, 74.3516, 75.5802, 76.4386, 77.4916, 78.1524, 79.1892, 79.8414, 80.8798, 81.8376, 82.4698, 83.7656, 84.331, 85.5914, 86.6012, 87.7016, 88.5582, 89.3394, 90.3544, 91.4912, 92.308, 93.3552, 93.9746, 95.2052, 95.727, 97.1322, 98.3944, 98.7588, 100.242, 101.1914, 102.2538, 102.8776, 103.6292, 105.1932, 105.9152, 107.0868, 107.6728, 108.7144, 110.3114, 110.8716,
+ 111.245, 112.7908, 113.7064, 114.636, 115.7464, 116.1788, 117.7464, 118.4896, 119.6166, 120.5082, 121.7798, 122.9028, 123.4426, 124.8854, 125.705, 126.4652, 128.3464, 128.3462, 130.0398, 131.0342, 131.0042, 132.4766, 133.511, 134.7252, 135.425, 136.5172, 138.0572, 138.6694, 139.3712, 140.8598, 141.4594, 142.554, 143.4006, 144.7374, 146.1634, 146.8994, 147.605, 147.9304, 149.1636, 150.2468, 151.5876, 152.2096, 153.7032, 154.7146, 155.807, 156.9228, 157.0372, 158.5852, },
+ // precision 6
+ { 46, 46.1902, 47.271, 47.8358, 48.8142, 49.2854, 50.317, 51.354, 51.8924, 52.9436, 53.4596, 54.5262, 55.6248, 56.1574, 57.2822, 57.837, 58.9636, 60.074, 60.7042, 61.7976, 62.4772, 63.6564, 64.7942, 65.5004, 66.686, 67.291, 68.5672, 69.8556, 70.4982, 71.8204, 72.4252, 73.7744, 75.0786, 75.8344, 77.0294, 77.8098, 79.0794, 80.5732, 81.1878, 82.5648, 83.2902, 84.6784, 85.3352, 86.8946, 88.3712, 89.0852, 90.499, 91.2686, 92.6844, 94.2234, 94.9732, 96.3356, 97.2286, 98.7262, 100.3284, 101.1048, 102.5962, 103.3562, 105.1272, 106.4184, 107.4974, 109.0822, 109.856, 111.48, 113.2834, 114.0208, 115.637, 116.5174, 118.0576, 119.7476, 120.427, 122.1326, 123.2372, 125.2788, 126.6776, 127.7926, 129.1952, 129.9564, 131.6454, 133.87, 134.5428, 136.2, 137.0294, 138.6278, 139.6782, 141.792, 143.3516, 144.2832, 146.0394, 147.0748, 148.4912, 150.849, 151.696, 153.5404, 154.073, 156.3714, 157.7216, 158.7328, 160.4208, 161.4184, 163.9424, 165.2772, 166.411, 168.1308, 168.769, 170.9258,
+ 172.6828, 173.7502, 175.706, 176.3886, 179.0186, 180.4518, 181.927, 183.4172, 184.4114, 186.033, 188.5124, 189.5564, 191.6008, 192.4172, 193.8044, 194.997, 197.4548, 198.8948, 200.2346, 202.3086, 203.1548, 204.8842, 206.6508, 206.6772, 209.7254, 210.4752, 212.7228, 214.6614, 215.1676, 217.793, 218.0006, 219.9052, 221.66, 223.5588, 225.1636, 225.6882, 227.7126, 229.4502, 231.1978, 232.9756, 233.1654, 236.727, 238.1974, 237.7474, 241.1346, 242.3048, 244.1948, 245.3134, 246.879, 249.1204, 249.853, 252.6792, 253.857, 254.4486, 257.2362, 257.9534, 260.0286, 260.5632, 262.663, 264.723, 265.7566, 267.2566, 267.1624, 270.62, 272.8216, 273.2166, 275.2056, 276.2202, 278.3726, 280.3344, 281.9284, 283.9728, 284.1924, 286.4872, 287.587, 289.807, 291.1206, 292.769, 294.8708, 296.665, 297.1182, 299.4012, 300.6352, 302.1354, 304.1756, 306.1606, 307.3462, 308.5214, 309.4134, 310.8352, 313.9684, 315.837, 316.7796, 318.9858, },
+ // precision 7
+ { 92, 93.4934, 94.9758, 96.4574, 97.9718, 99.4954, 101.5302, 103.0756, 104.6374, 106.1782, 107.7888, 109.9522, 111.592, 113.2532, 114.9086, 116.5938, 118.9474, 120.6796, 122.4394, 124.2176, 125.9768, 128.4214, 130.2528, 132.0102, 133.8658, 135.7278, 138.3044, 140.1316, 142.093, 144.0032, 145.9092, 148.6306, 150.5294, 152.5756, 154.6508, 156.662, 159.552, 161.3724, 163.617, 165.5754, 167.7872, 169.8444, 172.7988, 174.8606, 177.2118, 179.3566, 181.4476, 184.5882, 186.6816, 189.0824, 191.0258, 193.6048, 196.4436, 198.7274, 200.957, 203.147, 205.4364, 208.7592, 211.3386, 213.781, 215.8028, 218.656, 221.6544, 223.996, 226.4718, 229.1544, 231.6098, 234.5956, 237.0616, 239.5758, 242.4878, 244.5244, 248.2146, 250.724, 252.8722, 255.5198, 258.0414, 261.941, 264.9048, 266.87, 269.4304, 272.028, 274.4708, 278.37, 281.0624, 283.4668, 286.5532, 289.4352, 293.2564, 295.2744, 298.2118, 300.7472, 304.1456, 307.2928, 309.7504, 312.5528, 315.979, 318.2102, 322.1834, 324.3494, 327.325,
+ 330.6614, 332.903, 337.2544, 339.9042, 343.215, 345.2864, 348.0814, 352.6764, 355.301, 357.139, 360.658, 363.1732, 366.5902, 369.9538, 373.0828, 375.922, 378.9902, 382.7328, 386.4538, 388.1136, 391.2234, 394.0878, 396.708, 401.1556, 404.1852, 406.6372, 409.6822, 412.7796, 416.6078, 418.4916, 422.131, 424.5376, 428.1988, 432.211, 434.4502, 438.5282, 440.912, 444.0448, 447.7432, 450.8524, 453.7988, 456.7858, 458.8868, 463.9886, 466.5064, 468.9124, 472.6616, 475.4682, 478.582, 481.304, 485.2738, 488.6894, 490.329, 496.106, 497.6908, 501.1374, 504.5322, 506.8848, 510.3324, 513.4512, 516.179, 520.4412, 522.6066, 526.167, 528.7794, 533.379, 536.067, 538.46, 542.9116, 545.692, 547.9546, 552.493, 555.2722, 557.335, 562.449, 564.2014, 569.0738, 571.0974, 574.8564, 578.2996, 581.409, 583.9704, 585.8098, 589.6528, 594.5998, 595.958, 600.068, 603.3278, 608.2016, 609.9632, 612.864, 615.43, 620.7794, 621.272, 625.8644, 629.206, 633.219, 634.5154, 638.6102, },
+ // precision 8
+ { 184.2152, 187.2454, 190.2096, 193.6652, 196.6312, 199.6822, 203.249, 206.3296, 210.0038, 213.2074, 216.4612, 220.27, 223.5178, 227.4412, 230.8032, 234.1634, 238.1688, 241.6074, 245.6946, 249.2664, 252.8228, 257.0432, 260.6824, 264.9464, 268.6268, 272.2626, 276.8376, 280.4034, 284.8956, 288.8522, 292.7638, 297.3552, 301.3556, 305.7526, 309.9292, 313.8954, 318.8198, 322.7668, 327.298, 331.6688, 335.9466, 340.9746, 345.1672, 349.3474, 354.3028, 358.8912, 364.114, 368.4646, 372.9744, 378.4092, 382.6022, 387.843, 392.5684, 397.1652, 402.5426, 407.4152, 412.5388, 417.3592, 422.1366, 427.486, 432.3918, 437.5076, 442.509, 447.3834, 453.3498, 458.0668, 463.7346, 469.1228, 473.4528, 479.7, 484.644, 491.0518, 495.5774, 500.9068, 506.432, 512.1666, 517.434, 522.6644, 527.4894, 533.6312, 538.3804, 544.292, 550.5496, 556.0234, 562.8206, 566.6146, 572.4188, 579.117, 583.6762, 590.6576, 595.7864, 601.509, 607.5334, 612.9204, 619.772, 624.2924, 630.8654, 636.1836, 642.745, 649.1316,
+ 655.0386, 660.0136, 666.6342, 671.6196, 678.1866, 684.4282, 689.3324, 695.4794, 702.5038, 708.129, 713.528, 720.3204, 726.463, 732.7928, 739.123, 744.7418, 751.2192, 756.5102, 762.6066, 769.0184, 775.2224, 781.4014, 787.7618, 794.1436, 798.6506, 805.6378, 811.766, 819.7514, 824.5776, 828.7322, 837.8048, 843.6302, 849.9336, 854.4798, 861.3388, 867.9894, 873.8196, 880.3136, 886.2308, 892.4588, 899.0816, 905.4076, 912.0064, 917.3878, 923.619, 929.998, 937.3482, 943.9506, 947.991, 955.1144, 962.203, 968.8222, 975.7324, 981.7826, 988.7666, 994.2648, 1000.3128, 1007.4082, 1013.7536, 1020.3376, 1026.7156, 1031.7478, 1037.4292, 1045.393, 1051.2278, 1058.3434, 1062.8726, 1071.884, 1076.806, 1082.9176, 1089.1678, 1095.5032, 1102.525, 1107.2264, 1115.315, 1120.93, 1127.252, 1134.1496, 1139.0408, 1147.5448, 1153.3296, 1158.1974, 1166.5262, 1174.3328, 1175.657, 1184.4222, 1190.9172, 1197.1292, 1204.4606, 1210.4578, 1218.8728, 1225.3336, 1226.6592, 1236.5768, 1241.363,
+ 1249.4074, 1254.6566, 1260.8014, 1266.5454, 1274.5192, },
+ // precision 9
+ { 369, 374.8294, 381.2452, 387.6698, 394.1464, 400.2024, 406.8782, 413.6598, 420.462, 427.2826, 433.7102, 440.7416, 447.9366, 455.1046, 462.285, 469.0668, 476.306, 483.8448, 491.301, 498.9886, 506.2422, 513.8138, 521.7074, 529.7428, 537.8402, 545.1664, 553.3534, 561.594, 569.6886, 577.7876, 585.65, 594.228, 602.8036, 611.1666, 620.0818, 628.0824, 637.2574, 646.302, 655.1644, 664.0056, 672.3802, 681.7192, 690.5234, 700.2084, 708.831, 718.485, 728.1112, 737.4764, 746.76, 756.3368, 766.5538, 775.5058, 785.2646, 795.5902, 804.3818, 814.8998, 824.9532, 835.2062, 845.2798, 854.4728, 864.9582, 875.3292, 886.171, 896.781, 906.5716, 916.7048, 927.5322, 937.875, 949.3972, 958.3464, 969.7274, 980.2834, 992.1444, 1003.4264, 1013.0166, 1024.018, 1035.0438, 1046.34, 1057.6856, 1068.9836, 1079.0312, 1091.677, 1102.3188, 1113.4846, 1124.4424, 1135.739, 1147.1488, 1158.9202, 1169.406, 1181.5342, 1193.2834, 1203.8954, 1216.3286, 1226.2146, 1239.6684, 1251.9946, 1262.123, 1275.4338,
+ 1285.7378, 1296.076, 1308.9692, 1320.4964, 1333.0998, 1343.9864, 1357.7754, 1368.3208, 1380.4838, 1392.7388, 1406.0758, 1416.9098, 1428.9728, 1440.9228, 1453.9292, 1462.617, 1476.05, 1490.2996, 1500.6128, 1513.7392, 1524.5174, 1536.6322, 1548.2584, 1562.3766, 1572.423, 1587.1232, 1596.5164, 1610.5938, 1622.5972, 1633.1222, 1647.7674, 1658.5044, 1671.57, 1683.7044, 1695.4142, 1708.7102, 1720.6094, 1732.6522, 1747.841, 1756.4072, 1769.9786, 1782.3276, 1797.5216, 1808.3186, 1819.0694, 1834.354, 1844.575, 1856.2808, 1871.1288, 1880.7852, 1893.9622, 1906.3418, 1920.6548, 1932.9302, 1945.8584, 1955.473, 1968.8248, 1980.6446, 1995.9598, 2008.349, 2019.8556, 2033.0334, 2044.0206, 2059.3956, 2069.9174, 2082.6084, 2093.7036, 2106.6108, 2118.9124, 2132.301, 2144.7628, 2159.8422, 2171.0212, 2183.101, 2193.5112, 2208.052, 2221.3194, 2233.3282, 2247.295, 2257.7222, 2273.342, 2286.5638, 2299.6786, 2310.8114, 2322.3312, 2335.516, 2349.874, 2363.5968, 2373.865, 2387.1918,
+ 2401.8328, 2414.8496, 2424.544, 2436.7592, 2447.1682, 2464.1958, 2474.3438, 2489.0006, 2497.4526, 2513.6586, 2527.19, 2540.7028, 2553.768, },
+ // precision 10
+ { 738.1256, 750.4234, 763.1064, 775.4732, 788.4636, 801.0644, 814.488, 827.9654, 841.0832, 854.7864, 868.1992, 882.2176, 896.5228, 910.1716, 924.7752, 938.899, 953.6126, 968.6492, 982.9474, 998.5214, 1013.1064, 1028.6364, 1044.2468, 1059.4588, 1075.3832, 1091.0584, 1106.8606, 1123.3868, 1139.5062, 1156.1862, 1172.463, 1189.339, 1206.1936, 1223.1292, 1240.1854, 1257.2908, 1275.3324, 1292.8518, 1310.5204, 1328.4854, 1345.9318, 1364.552, 1381.4658, 1400.4256, 1419.849, 1438.152, 1456.8956, 1474.8792, 1494.118, 1513.62, 1532.5132, 1551.9322, 1570.7726, 1590.6086, 1610.5332, 1630.5918, 1650.4294, 1669.7662, 1690.4106, 1710.7338, 1730.9012, 1750.4486, 1770.1556, 1791.6338, 1812.7312, 1833.6264, 1853.9526, 1874.8742, 1896.8326, 1918.1966, 1939.5594, 1961.07, 1983.037, 2003.1804, 2026.071, 2047.4884, 2070.0848, 2091.2944, 2114.333, 2135.9626, 2158.2902, 2181.0814, 2202.0334, 2224.4832, 2246.39, 2269.7202, 2292.1714, 2314.2358, 2338.9346, 2360.891, 2384.0264, 2408.3834, 2430.1544,
+ 2454.8684, 2476.9896, 2501.4368, 2522.8702, 2548.0408, 2570.6738, 2593.5208, 2617.0158, 2640.2302, 2664.0962, 2687.4986, 2714.2588, 2735.3914, 2759.6244, 2781.8378, 2808.0072, 2830.6516, 2856.2454, 2877.2136, 2903.4546, 2926.785, 2951.2294, 2976.468, 3000.867, 3023.6508, 3049.91, 3073.5984, 3098.162, 3121.5564, 3146.2328, 3170.9484, 3195.5902, 3221.3346, 3242.7032, 3271.6112, 3296.5546, 3317.7376, 3345.072, 3369.9518, 3394.326, 3418.1818, 3444.6926, 3469.086, 3494.2754, 3517.8698, 3544.248, 3565.3768, 3588.7234, 3616.979, 3643.7504, 3668.6812, 3695.72, 3719.7392, 3742.6224, 3770.4456, 3795.6602, 3819.9058, 3844.002, 3869.517, 3895.6824, 3920.8622, 3947.1364, 3973.985, 3995.4772, 4021.62, 4046.628, 4074.65, 4096.2256, 4121.831, 4146.6406, 4173.276, 4195.0744, 4223.9696, 4251.3708, 4272.9966, 4300.8046, 4326.302, 4353.1248, 4374.312, 4403.0322, 4426.819, 4450.0598, 4478.5206, 4504.8116, 4528.8928, 4553.9584, 4578.8712, 4603.8384, 4632.3872, 4655.5128, 4675.821,
+ 4704.6222, 4731.9862, 4755.4174, 4781.2628, 4804.332, 4832.3048, 4862.8752, 4883.4148, 4906.9544, 4935.3516, 4954.3532, 4984.0248, 5011.217, 5035.3258, 5057.3672, 5084.1828, },
+ // precision 11
+ { 1477, 1501.6014, 1526.5802, 1551.7942, 1577.3042, 1603.2062, 1629.8402, 1656.2292, 1682.9462, 1709.9926, 1737.3026, 1765.4252, 1793.0578, 1821.6092, 1849.626, 1878.5568, 1908.527, 1937.5154, 1967.1874, 1997.3878, 2027.37, 2058.1972, 2089.5728, 2120.1012, 2151.9668, 2183.292, 2216.0772, 2247.8578, 2280.6562, 2313.041, 2345.714, 2380.3112, 2414.1806, 2447.9854, 2481.656, 2516.346, 2551.5154, 2586.8378, 2621.7448, 2656.6722, 2693.5722, 2729.1462, 2765.4124, 2802.8728, 2838.898, 2876.408, 2913.4926, 2951.4938, 2989.6776, 3026.282, 3065.7704, 3104.1012, 3143.7388, 3181.6876, 3221.1872, 3261.5048, 3300.0214, 3339.806, 3381.409, 3421.4144, 3461.4294, 3502.2286, 3544.651, 3586.6156, 3627.337, 3670.083, 3711.1538, 3753.5094, 3797.01, 3838.6686, 3882.1678, 3922.8116, 3967.9978, 4009.9204, 4054.3286, 4097.5706, 4140.6014, 4185.544, 4229.5976, 4274.583, 4316.9438, 4361.672, 4406.2786, 4451.8628, 4496.1834, 4543.505, 4589.1816, 4632.5188, 4678.2294, 4724.8908, 4769.0194, 4817.052,
+ 4861.4588, 4910.1596, 4956.4344, 5002.5238, 5048.13, 5093.6374, 5142.8162, 5187.7894, 5237.3984, 5285.6078, 5331.0858, 5379.1036, 5428.6258, 5474.6018, 5522.7618, 5571.5822, 5618.59, 5667.9992, 5714.88, 5763.454, 5808.6982, 5860.3644, 5910.2914, 5953.571, 6005.9232, 6055.1914, 6104.5882, 6154.5702, 6199.7036, 6251.1764, 6298.7596, 6350.0302, 6398.061, 6448.4694, 6495.933, 6548.0474, 6597.7166, 6646.9416, 6695.9208, 6742.6328, 6793.5276, 6842.1934, 6894.2372, 6945.3864, 6996.9228, 7044.2372, 7094.1374, 7142.2272, 7192.2942, 7238.8338, 7288.9006, 7344.0908, 7394.8544, 7443.5176, 7490.4148, 7542.9314, 7595.6738, 7641.9878, 7694.3688, 7743.0448, 7797.522, 7845.53, 7899.594, 7950.3132, 7996.455, 8050.9442, 8092.9114, 8153.1374, 8197.4472, 8252.8278, 8301.8728, 8348.6776, 8401.4698, 8453.551, 8504.6598, 8553.8944, 8604.1276, 8657.6514, 8710.3062, 8758.908, 8807.8706, 8862.1702, 8910.4668, 8960.77, 9007.2766, 9063.164, 9121.0534, 9164.1354, 9218.1594, 9267.767,
+ 9319.0594, 9372.155, 9419.7126, 9474.3722, 9520.1338, 9572.368, 9622.7702, 9675.8448, 9726.5396, 9778.7378, 9827.6554, 9878.1922, 9928.7782, 9978.3984, 10026.578, 10076.5626, 10137.1618, 10177.5244, 10229.9176, },
+ // precision 12
+ { 2954, 3003.4782, 3053.3568, 3104.3666, 3155.324, 3206.9598, 3259.648, 3312.539, 3366.1474, 3420.2576, 3474.8376, 3530.6076, 3586.451, 3643.38, 3700.4104, 3757.5638, 3815.9676, 3875.193, 3934.838, 3994.8548, 4055.018, 4117.1742, 4178.4482, 4241.1294, 4304.4776, 4367.4044, 4431.8724, 4496.3732, 4561.4304, 4627.5326, 4693.949, 4761.5532, 4828.7256, 4897.6182, 4965.5186, 5034.4528, 5104.865, 5174.7164, 5244.6828, 5316.6708, 5387.8312, 5459.9036, 5532.476, 5604.8652, 5679.6718, 5753.757, 5830.2072, 5905.2828, 5980.0434, 6056.6264, 6134.3192, 6211.5746, 6290.0816, 6367.1176, 6447.9796, 6526.5576, 6606.1858, 6686.9144, 6766.1142, 6847.0818, 6927.9664, 7010.9096, 7091.0816, 7175.3962, 7260.3454, 7344.018, 7426.4214, 7511.3106, 7596.0686, 7679.8094, 7765.818, 7852.4248, 7936.834, 8022.363, 8109.5066, 8200.4554, 8288.5832, 8373.366, 8463.4808, 8549.7682, 8642.0522, 8728.3288, 8820.9528, 8907.727, 9001.0794, 9091.2522, 9179.988, 9269.852, 9362.6394, 9453.642, 9546.9024, 9640.6616,
+ 9732.6622, 9824.3254, 9917.7484, 10007.9392, 10106.7508, 10196.2152, 10289.8114, 10383.5494, 10482.3064, 10576.8734, 10668.7872, 10764.7156, 10862.0196, 10952.793, 11049.9748, 11146.0702, 11241.4492, 11339.2772, 11434.2336, 11530.741, 11627.6136, 11726.311, 11821.5964, 11918.837, 12015.3724, 12113.0162, 12213.0424, 12306.9804, 12408.4518, 12504.8968, 12604.586, 12700.9332, 12798.705, 12898.5142, 12997.0488, 13094.788, 13198.475, 13292.7764, 13392.9698, 13486.8574, 13590.1616, 13686.5838, 13783.6264, 13887.2638, 13992.0978, 14081.0844, 14189.9956, 14280.0912, 14382.4956, 14486.4384, 14588.1082, 14686.2392, 14782.276, 14888.0284, 14985.1864, 15088.8596, 15187.0998, 15285.027, 15383.6694, 15495.8266, 15591.3736, 15694.2008, 15790.3246, 15898.4116, 15997.4522, 16095.5014, 16198.8514, 16291.7492, 16402.6424, 16499.1266, 16606.2436, 16697.7186, 16796.3946, 16902.3376, 17005.7672, 17100.814, 17206.8282, 17305.8262, 17416.0744, 17508.4092, 17617.0178, 17715.4554,
+ 17816.758, 17920.1748, 18012.9236, 18119.7984, 18223.2248, 18324.2482, 18426.6276, 18525.0932, 18629.8976, 18733.2588, 18831.0466, 18940.1366, 19032.2696, 19131.729, 19243.4864, 19349.6932, 19442.866, 19547.9448, 19653.2798, 19754.4034, 19854.0692, 19965.1224, 20065.1774, 20158.2212, 20253.353, 20366.3264, 20463.22, },
+ // precision 13
+ { 5908.5052, 6007.2672, 6107.347, 6208.5794, 6311.2622, 6414.5514, 6519.3376, 6625.6952, 6732.5988, 6841.3552, 6950.5972, 7061.3082, 7173.5646, 7287.109, 7401.8216, 7516.4344, 7633.3802, 7751.2962, 7870.3784, 7990.292, 8110.79, 8233.4574, 8356.6036, 8482.2712, 8607.7708, 8735.099, 8863.1858, 8993.4746, 9123.8496, 9255.6794, 9388.5448, 9522.7516, 9657.3106, 9792.6094, 9930.5642, 10068.794, 10206.7256, 10347.81, 10490.3196, 10632.0778, 10775.9916, 10920.4662, 11066.124, 11213.073, 11358.0362, 11508.1006, 11659.1716, 11808.7514, 11959.4884, 12112.1314, 12265.037, 12420.3756, 12578.933, 12734.311, 12890.0006, 13047.2144, 13207.3096, 13368.5144, 13528.024, 13689.847, 13852.7528, 14018.3168, 14180.5372, 14346.9668, 14513.5074, 14677.867, 14846.2186, 15017.4186, 15184.9716, 15356.339, 15529.2972, 15697.3578, 15871.8686, 16042.187, 16216.4094, 16389.4188, 16565.9126, 16742.3272, 16919.0042, 17094.7592, 17273.965, 17451.8342, 17634.4254, 17810.5984, 17988.9242, 18171.051,
+ 18354.7938, 18539.466, 18721.0408, 18904.9972, 19081.867, 19271.9118, 19451.8694, 19637.9816, 19821.2922, 20013.1292, 20199.3858, 20387.8726, 20572.9514, 20770.7764, 20955.1714, 21144.751, 21329.9952, 21520.709, 21712.7016, 21906.3868, 22096.2626, 22286.0524, 22475.051, 22665.5098, 22862.8492, 23055.5294, 23249.6138, 23437.848, 23636.273, 23826.093, 24020.3296, 24213.3896, 24411.7392, 24602.9614, 24805.7952, 24998.1552, 25193.9588, 25389.0166, 25585.8392, 25780.6976, 25981.2728, 26175.977, 26376.5252, 26570.1964, 26773.387, 26962.9812, 27163.0586, 27368.164, 27565.0534, 27758.7428, 27961.1276, 28163.2324, 28362.3816, 28565.7668, 28758.644, 28956.9768, 29163.4722, 29354.7026, 29561.1186, 29767.9948, 29959.9986, 30164.0492, 30366.9818, 30562.5338, 30762.9928, 30976.1592, 31166.274, 31376.722, 31570.3734, 31770.809, 31974.8934, 32179.5286, 32387.5442, 32582.3504, 32794.076, 32989.9528, 33191.842, 33392.4684, 33595.659, 33801.8672, 34000.3414, 34200.0922, 34402.6792,
+ 34610.0638, 34804.0084, 35011.13, 35218.669, 35418.6634, 35619.0792, 35830.6534, 36028.4966, 36229.7902, 36438.6422, 36630.7764, 36833.3102, 37048.6728, 37247.3916, 37453.5904, 37669.3614, 37854.5526, 38059.305, 38268.0936, 38470.2516, 38674.7064, 38876.167, 39068.3794, 39281.9144, 39492.8566, 39684.8628, 39898.4108, 40093.1836, 40297.6858, 40489.7086, 40717.2424, },
+ // precision 14
+ { 11817.475, 12015.0046, 12215.3792, 12417.7504, 12623.1814, 12830.0086, 13040.0072, 13252.503, 13466.178, 13683.2738, 13902.0344, 14123.9798, 14347.394, 14573.7784, 14802.6894, 15033.6824, 15266.9134, 15502.8624, 15741.4944, 15980.7956, 16223.8916, 16468.6316, 16715.733, 16965.5726, 17217.204, 17470.666, 17727.8516, 17986.7886, 18247.6902, 18510.9632, 18775.304, 19044.7486, 19314.4408, 19587.202, 19862.2576, 20135.924, 20417.0324, 20697.9788, 20979.6112, 21265.0274, 21550.723, 21841.6906, 22132.162, 22428.1406, 22722.127, 23020.5606, 23319.7394, 23620.4014, 23925.2728, 24226.9224, 24535.581, 24845.505, 25155.9618, 25470.3828, 25785.9702, 26103.7764, 26420.4132, 26742.0186, 27062.8852, 27388.415, 27714.6024, 28042.296, 28365.4494, 28701.1526, 29031.8008, 29364.2156, 29704.497, 30037.1458, 30380.111, 30723.8168, 31059.5114, 31404.9498, 31751.6752, 32095.2686, 32444.7792, 32794.767, 33145.204, 33498.4226, 33847.6502, 34209.006, 34560.849, 34919.4838, 35274.9778, 35635.1322,
+ 35996.3266, 36359.1394, 36722.8266, 37082.8516, 37447.7354, 37815.9606, 38191.0692, 38559.4106, 38924.8112, 39294.6726, 39663.973, 40042.261, 40416.2036, 40779.2036, 41161.6436, 41540.9014, 41921.1998, 42294.7698, 42678.5264, 43061.3464, 43432.375, 43818.432, 44198.6598, 44583.0138, 44970.4794, 45353.924, 45729.858, 46118.2224, 46511.5724, 46900.7386, 47280.6964, 47668.1472, 48055.6796, 48446.9436, 48838.7146, 49217.7296, 49613.7796, 50010.7508, 50410.0208, 50793.7886, 51190.2456, 51583.1882, 51971.0796, 52376.5338, 52763.319, 53165.5534, 53556.5594, 53948.2702, 54346.352, 54748.7914, 55138.577, 55543.4824, 55941.1748, 56333.7746, 56745.1552, 57142.7944, 57545.2236, 57935.9956, 58348.5268, 58737.5474, 59158.5962, 59542.6896, 59958.8004, 60349.3788, 60755.0212, 61147.6144, 61548.194, 61946.0696, 62348.6042, 62763.603, 63162.781, 63560.635, 63974.3482, 64366.4908, 64771.5876, 65176.7346, 65597.3916, 65995.915, 66394.0384, 66822.9396, 67203.6336, 67612.2032,
+ 68019.0078, 68420.0388, 68821.22, 69235.8388, 69640.0724, 70055.155, 70466.357, 70863.4266, 71276.2482, 71677.0306, 72080.2006, 72493.0214, 72893.5952, 73314.5856, 73714.9852, 74125.3022, 74521.2122, 74933.6814, 75341.5904, 75743.0244, 76166.0278, 76572.1322, 76973.1028, 77381.6284, 77800.6092, 78189.328, 78607.0962, 79012.2508, 79407.8358, 79825.725, 80238.701, 80646.891, 81035.6436, 81460.0448, 81876.3884, },
+ // precision 15
+ { 23635.0036, 24030.8034, 24431.4744, 24837.1524, 25246.7928, 25661.326, 26081.3532, 26505.2806, 26933.9892, 27367.7098, 27805.318, 28248.799, 28696.4382, 29148.8244, 29605.5138, 30066.8668, 30534.2344, 31006.32, 31480.778, 31962.2418, 32447.3324, 32938.0232, 33432.731, 33930.728, 34433.9896, 34944.1402, 35457.5588, 35974.5958, 36497.3296, 37021.9096, 37554.326, 38088.0826, 38628.8816, 39171.3192, 39723.2326, 40274.5554, 40832.3142, 41390.613, 41959.5908, 42532.5466, 43102.0344, 43683.5072, 44266.694, 44851.2822, 45440.7862, 46038.0586, 46640.3164, 47241.064, 47846.155, 48454.7396, 49076.9168, 49692.542, 50317.4778, 50939.65, 51572.5596, 52210.2906, 52843.7396, 53481.3996, 54127.236, 54770.406, 55422.6598, 56078.7958, 56736.7174, 57397.6784, 58064.5784, 58730.308, 59404.9784, 60077.0864, 60751.9158, 61444.1386, 62115.817, 62808.7742, 63501.4774, 64187.5454, 64883.6622, 65582.7468, 66274.5318, 66976.9276, 67688.7764, 68402.138, 69109.6274, 69822.9706, 70543.6108,
+ 71265.5202, 71983.3848, 72708.4656, 73433.384, 74158.4664, 74896.4868, 75620.9564, 76362.1434, 77098.3204, 77835.7662, 78582.6114, 79323.9902, 80067.8658, 80814.9246, 81567.0136, 82310.8536, 83061.9952, 83821.4096, 84580.8608, 85335.547, 86092.5802, 86851.6506, 87612.311, 88381.2016, 89146.3296, 89907.8974, 90676.846, 91451.4152, 92224.5518, 92995.8686, 93763.5066, 94551.2796, 95315.1944, 96096.1806, 96881.0918, 97665.679, 98442.68, 99229.3002, 100011.0994, 100790.6386, 101580.1564, 102377.7484, 103152.1392, 103944.2712, 104730.216, 105528.6336, 106324.9398, 107117.6706, 107890.3988, 108695.2266, 109485.238, 110294.7876, 111075.0958, 111878.0496, 112695.2864, 113464.5486, 114270.0474, 115068.608, 115884.3626, 116673.2588, 117483.3716, 118275.097, 119085.4092, 119879.2808, 120687.5868, 121499.9944, 122284.916, 123095.9254, 123912.5038, 124709.0454, 125503.7182, 126323.259, 127138.9412, 127943.8294, 128755.646, 129556.5354, 130375.3298, 131161.4734, 131971.1962,
+ 132787.5458, 133588.1056, 134431.351, 135220.2906, 136023.398, 136846.6558, 137667.0004, 138463.663, 139283.7154, 140074.6146, 140901.3072, 141721.8548, 142543.2322, 143356.1096, 144173.7412, 144973.0948, 145794.3162, 146609.5714, 147420.003, 148237.9784, 149050.5696, 149854.761, 150663.1966, 151494.0754, 152313.1416, 153112.6902, 153935.7206, 154746.9262, 155559.547, 156401.9746, 157228.7036, 158008.7254, 158820.75, 159646.9184, 160470.4458, 161279.5348, 162093.3114, 162918.542, 163729.2842, },
+ // precision 16
+ { 47271, 48062.3584, 48862.7074, 49673.152, 50492.8416, 51322.9514, 52161.03, 53009.407, 53867.6348, 54734.206, 55610.5144, 56496.2096, 57390.795, 58297.268, 59210.6448, 60134.665, 61068.0248, 62010.4472, 62962.5204, 63923.5742, 64895.0194, 65876.4182, 66862.6136, 67862.6968, 68868.8908, 69882.8544, 70911.271, 71944.0924, 72990.0326, 74040.692, 75100.6336, 76174.7826, 77252.5998, 78340.2974, 79438.2572, 80545.4976, 81657.2796, 82784.6336, 83915.515, 85059.7362, 86205.9368, 87364.4424, 88530.3358, 89707.3744, 90885.9638, 92080.197, 93275.5738, 94479.391, 95695.918, 96919.2236, 98148.4602, 99382.3474, 100625.6974, 101878.0284, 103141.6278, 104409.4588, 105686.2882, 106967.5402, 108261.6032, 109548.1578, 110852.0728, 112162.231, 113479.0072, 114806.2626, 116137.9072, 117469.5048, 118813.5186, 120165.4876, 121516.2556, 122875.766, 124250.5444, 125621.2222, 127003.2352, 128387.848, 129775.2644, 131181.7776, 132577.3086, 133979.9458, 135394.1132, 136800.9078, 138233.217,
+ 139668.5308, 141085.212, 142535.2122, 143969.0684, 145420.2872, 146878.1542, 148332.7572, 149800.3202, 151269.66, 152743.6104, 154213.0948, 155690.288, 157169.4246, 158672.1756, 160160.059, 161650.6854, 163145.7772, 164645.6726, 166159.1952, 167682.1578, 169177.3328, 170700.0118, 172228.8964, 173732.6664, 175265.5556, 176787.799, 178317.111, 179856.6914, 181400.865, 182943.4612, 184486.742, 186033.4698, 187583.7886, 189148.1868, 190688.4526, 192250.1926, 193810.9042, 195354.2972, 196938.7682, 198493.5898, 200079.2824, 201618.912, 203205.5492, 204765.5798, 206356.1124, 207929.3064, 209498.7196, 211086.229, 212675.1324, 214256.7892, 215826.2392, 217412.8474, 218995.6724, 220618.6038, 222207.1166, 223781.0364, 225387.4332, 227005.7928, 228590.4336, 230217.8738, 231805.1054, 233408.9, 234995.3432, 236601.4956, 238190.7904, 239817.2548, 241411.2832, 243002.4066, 244640.1884, 246255.3128, 247849.3508, 249479.9734, 251106.8822, 252705.027, 254332.9242, 255935.129,
+ 257526.9014, 259154.772, 260777.625, 262390.253, 264004.4906, 265643.59, 267255.4076, 268873.426, 270470.7252, 272106.4804, 273722.4456, 275337.794, 276945.7038, 278592.9154, 280204.3726, 281841.1606, 283489.171, 285130.1716, 286735.3362, 288364.7164, 289961.1814, 291595.5524, 293285.683, 294899.6668, 296499.3434, 298128.0462, 299761.8946, 301394.2424, 302997.6748, 304615.1478, 306269.7724, 307886.114, 309543.1028, 311153.2862, 312782.8546, 314421.2008, 316033.2438, 317692.9636, 319305.2648, 320948.7406, 322566.3364, 324228.4224, 325847.1542, },
+ // precision 17
+ { 94542, 96125.811, 97728.019, 99348.558, 100987.9705, 102646.7565, 104324.5125, 106021.7435, 107736.7865, 109469.272, 111223.9465, 112995.219, 114787.432, 116593.152, 118422.71, 120267.2345, 122134.6765, 124020.937, 125927.2705, 127851.255, 129788.9485, 131751.016, 133726.8225, 135722.592, 137736.789, 139770.568, 141821.518, 143891.343, 145982.1415, 148095.387, 150207.526, 152355.649, 154515.6415, 156696.05, 158887.7575, 161098.159, 163329.852, 165569.053, 167837.4005, 170121.6165, 172420.4595, 174732.6265, 177062.77, 179412.502, 181774.035, 184151.939, 186551.6895, 188965.691, 191402.8095, 193857.949, 196305.0775, 198774.6715, 201271.2585, 203764.78, 206299.3695, 208818.1365, 211373.115, 213946.7465, 216532.076, 219105.541, 221714.5375, 224337.5135, 226977.5125, 229613.0655, 232270.2685, 234952.2065, 237645.3555, 240331.1925, 243034.517, 245756.0725, 248517.6865, 251232.737, 254011.3955, 256785.995, 259556.44, 262368.335, 265156.911, 267965.266, 270785.583, 273616.0495,
+ 276487.4835, 279346.639, 282202.509, 285074.3885, 287942.2855, 290856.018, 293774.0345, 296678.5145, 299603.6355, 302552.6575, 305492.9785, 308466.8605, 311392.581, 314347.538, 317319.4295, 320285.9785, 323301.7325, 326298.3235, 329301.3105, 332301.987, 335309.791, 338370.762, 341382.923, 344431.1265, 347464.1545, 350507.28, 353619.2345, 356631.2005, 359685.203, 362776.7845, 365886.488, 368958.2255, 372060.6825, 375165.4335, 378237.935, 381328.311, 384430.5225, 387576.425, 390683.242, 393839.648, 396977.8425, 400101.9805, 403271.296, 406409.8425, 409529.5485, 412678.7, 415847.423, 419020.8035, 422157.081, 425337.749, 428479.6165, 431700.902, 434893.1915, 438049.582, 441210.5415, 444379.2545, 447577.356, 450741.931, 453959.548, 457137.0935, 460329.846, 463537.4815, 466732.3345, 469960.5615, 473164.681, 476347.6345, 479496.173, 482813.1645, 486025.6995, 489249.4885, 492460.1945, 495675.8805, 498908.0075, 502131.802, 505374.3855, 508550.9915, 511806.7305, 515026.776,
+ 518217.0005, 521523.9855, 524705.9855, 527950.997, 531210.0265, 534472.497, 537750.7315, 540926.922, 544207.094, 547429.4345, 550666.3745, 553975.3475, 557150.7185, 560399.6165, 563662.697, 566916.7395, 570146.1215, 573447.425, 576689.6245, 579874.5745, 583202.337, 586503.0255, 589715.635, 592910.161, 596214.3885, 599488.035, 602740.92, 605983.0685, 609248.67, 612491.3605, 615787.912, 619107.5245, 622307.9555, 625577.333, 628840.4385, 632085.2155, 635317.6135, 638691.7195, 641887.467, 645139.9405, 648441.546, 651666.252, 654941.845, },
+ // precision 18
+ { 189084, 192250.913, 195456.774, 198696.946, 201977.762, 205294.444, 208651.754, 212042.099, 215472.269, 218941.91, 222443.912, 225996.845, 229568.199, 233193.568, 236844.457, 240543.233, 244279.475, 248044.27, 251854.588, 255693.2, 259583.619, 263494.621, 267445.385, 271454.061, 275468.769, 279549.456, 283646.446, 287788.198, 291966.099, 296181.164, 300431.469, 304718.618, 309024.004, 313393.508, 317760.803, 322209.731, 326675.061, 331160.627, 335654.47, 340241.442, 344841.833, 349467.132, 354130.629, 358819.432, 363574.626, 368296.587, 373118.482, 377914.93, 382782.301, 387680.669, 392601.981, 397544.323, 402529.115, 407546.018, 412593.658, 417638.657, 422762.865, 427886.169, 433017.167, 438213.273, 443441.254, 448692.421, 453937.533, 459239.049, 464529.569, 469910.083, 475274.03, 480684.473, 486070.26, 491515.237, 496995.651, 502476.617, 507973.609, 513497.19, 519083.233, 524726.509, 530305.505, 535945.728, 541584.404, 547274.055, 552967.236, 558667.862, 564360.216,
+ 570128.148, 575965.08, 581701.952, 587532.523, 593361.144, 599246.128, 605033.418, 610958.779, 616837.117, 622772.818, 628672.04, 634675.369, 640574.831, 646585.739, 652574.547, 658611.217, 664642.684, 670713.914, 676737.681, 682797.313, 688837.897, 694917.874, 701009.882, 707173.648, 713257.254, 719415.392, 725636.761, 731710.697, 737906.209, 744103.074, 750313.39, 756504.185, 762712.579, 768876.985, 775167.859, 781359, 787615.959, 793863.597, 800245.477, 806464.582, 812785.294, 819005.925, 825403.057, 831676.197, 837936.284, 844266.968, 850642.711, 856959.756, 863322.774, 869699.931, 876102.478, 882355.787, 888694.463, 895159.952, 901536.143, 907872.631, 914293.672, 920615.14, 927130.974, 933409.404, 939922.178, 946331.47, 952745.93, 959209.264, 965590.224, 972077.284, 978501.961, 984953.19, 991413.271, 997817.479, 1004222.658, 1010725.676, 1017177.138, 1023612.529, 1030098.236, 1036493.719, 1043112.207, 1049537.036, 1056008.096, 1062476.184, 1068942.337,
+ 1075524.95, 1081932.864, 1088426.025, 1094776.005, 1101327.448, 1107901.673, 1114423.639, 1120884.602, 1127324.923, 1133794.24, 1140328.886, 1146849.376, 1153346.682, 1159836.502, 1166478.703, 1172953.304, 1179391.502, 1185950.982, 1192544.052, 1198913.41, 1205430.994, 1212015.525, 1218674.042, 1225121.683, 1231551.101, 1238126.379, 1244673.795, 1251260.649, 1257697.86, 1264320.983, 1270736.319, 1277274.694, 1283804.95, 1290211.514, 1296858.568, 1303455.691, } };
+
+ public static final double[][] biasData = {
+ // precision 4
+ { 10, 9.717, 9.207, 8.7896, 8.2882, 7.8204, 7.3772, 6.9342, 6.5202, 6.161, 5.7722, 5.4636, 5.0396, 4.6766, 4.3566, 4.0454, 3.7936, 3.4856, 3.2666, 2.9946, 2.766, 2.4692, 2.3638, 2.0764, 1.7864, 1.7602, 1.4814, 1.433, 1.2926, 1.0664, 0.999600000000001, 0.7956, 0.5366, 0.589399999999998, 0.573799999999999, 0.269799999999996, 0.368200000000002, 0.0544000000000011, 0.234200000000001, 0.0108000000000033, -0.203400000000002, -0.0701999999999998, -0.129600000000003, -0.364199999999997, -0.480600000000003, -0.226999999999997, -0.322800000000001, -0.382599999999996, -0.511200000000002, -0.669600000000003, -0.749400000000001, -0.500399999999999, -0.617600000000003, -0.6922, -0.601599999999998, -0.416200000000003, -0.338200000000001, -0.782600000000002, -0.648600000000002, -0.919800000000002, -0.851799999999997, -0.962400000000002, -0.6402, -1.1922, -1.0256, -1.086, -1.21899999999999, -0.819400000000002, -0.940600000000003, -1.1554, -1.2072, -1.1752, -1.16759999999999,
+ -1.14019999999999, -1.3754, -1.29859999999999, -1.607, -1.3292, -1.7606, },
+ // precision 5
+ { 22, 21.1194, 20.8208, 20.2318, 19.77, 19.2436, 18.7774, 18.2848, 17.8224, 17.3742, 16.9336, 16.503, 16.0494, 15.6292, 15.2124, 14.798, 14.367, 13.9728, 13.5944, 13.217, 12.8438, 12.3696, 12.0956, 11.7044, 11.324, 11.0668, 10.6698, 10.3644, 10.049, 9.6918, 9.4146, 9.082, 8.687, 8.5398, 8.2462, 7.857, 7.6606, 7.4168, 7.1248, 6.9222, 6.6804, 6.447, 6.3454, 5.9594, 5.7636, 5.5776, 5.331, 5.19, 4.9676, 4.7564, 4.5314, 4.4442, 4.3708, 3.9774, 3.9624, 3.8796, 3.755, 3.472, 3.2076, 3.1024, 2.8908, 2.7338, 2.7728, 2.629, 2.413, 2.3266, 2.1524, 2.2642, 2.1806, 2.0566, 1.9192, 1.7598, 1.3516, 1.5802, 1.43859999999999, 1.49160000000001, 1.1524, 1.1892, 0.841399999999993, 0.879800000000003, 0.837599999999995, 0.469800000000006, 0.765600000000006, 0.331000000000003, 0.591399999999993, 0.601200000000006, 0.701599999999999, 0.558199999999999, 0.339399999999998, 0.354399999999998, 0.491200000000006, 0.308000000000007, 0.355199999999996, -0.0254000000000048, 0.205200000000005,
+ -0.272999999999996, 0.132199999999997, 0.394400000000005, -0.241200000000006, 0.242000000000004, 0.191400000000002, 0.253799999999998, -0.122399999999999, -0.370800000000003, 0.193200000000004, -0.0848000000000013, 0.0867999999999967, -0.327200000000005, -0.285600000000002, 0.311400000000006, -0.128399999999999, -0.754999999999995, -0.209199999999996, -0.293599999999998, -0.364000000000004, -0.253600000000006, -0.821200000000005, -0.253600000000006, -0.510400000000004, -0.383399999999995, -0.491799999999998, -0.220200000000006, -0.0972000000000008, -0.557400000000001, -0.114599999999996, -0.295000000000002, -0.534800000000004, 0.346399999999988, -0.65379999999999, 0.0398000000000138, 0.0341999999999985, -0.995800000000003, -0.523400000000009, -0.489000000000004, -0.274799999999999, -0.574999999999989, -0.482799999999997, 0.0571999999999946, -0.330600000000004, -0.628800000000012, -0.140199999999993, -0.540600000000012, -0.445999999999998, -0.599400000000003,
+ -0.262599999999992, 0.163399999999996, -0.100599999999986, -0.39500000000001, -1.06960000000001, -0.836399999999998, -0.753199999999993, -0.412399999999991, -0.790400000000005, -0.29679999999999, -0.28540000000001, -0.193000000000012, -0.0772000000000048, -0.962799999999987, -0.414800000000014, },
+ // precision 6
+ { 45, 44.1902, 43.271, 42.8358, 41.8142, 41.2854, 40.317, 39.354, 38.8924, 37.9436, 37.4596, 36.5262, 35.6248, 35.1574, 34.2822, 33.837, 32.9636, 32.074, 31.7042, 30.7976, 30.4772, 29.6564, 28.7942, 28.5004, 27.686, 27.291, 26.5672, 25.8556, 25.4982, 24.8204, 24.4252, 23.7744, 23.0786, 22.8344, 22.0294, 21.8098, 21.0794, 20.5732, 20.1878, 19.5648, 19.2902, 18.6784, 18.3352, 17.8946, 17.3712, 17.0852, 16.499, 16.2686, 15.6844, 15.2234, 14.9732, 14.3356, 14.2286, 13.7262, 13.3284, 13.1048, 12.5962, 12.3562, 12.1272, 11.4184, 11.4974, 11.0822, 10.856, 10.48, 10.2834, 10.0208, 9.637, 9.51739999999999, 9.05759999999999, 8.74760000000001, 8.42700000000001, 8.1326, 8.2372, 8.2788, 7.6776, 7.79259999999999, 7.1952, 6.9564, 6.6454, 6.87, 6.5428, 6.19999999999999, 6.02940000000001, 5.62780000000001, 5.6782, 5.792, 5.35159999999999, 5.28319999999999, 5.0394, 5.07480000000001, 4.49119999999999, 4.84899999999999, 4.696, 4.54040000000001, 4.07300000000001, 4.37139999999999, 3.7216,
+ 3.7328, 3.42080000000001, 3.41839999999999, 3.94239999999999, 3.27719999999999, 3.411, 3.13079999999999, 2.76900000000001, 2.92580000000001, 2.68279999999999, 2.75020000000001, 2.70599999999999, 2.3886, 3.01859999999999, 2.45179999999999, 2.92699999999999, 2.41720000000001, 2.41139999999999, 2.03299999999999, 2.51240000000001, 2.5564, 2.60079999999999, 2.41720000000001, 1.80439999999999, 1.99700000000001, 2.45480000000001, 1.8948, 2.2346, 2.30860000000001, 2.15479999999999, 1.88419999999999, 1.6508, 0.677199999999999, 1.72540000000001, 1.4752, 1.72280000000001, 1.66139999999999, 1.16759999999999, 1.79300000000001, 1.00059999999999, 0.905200000000008, 0.659999999999997, 1.55879999999999, 1.1636, 0.688199999999995, 0.712600000000009, 0.450199999999995, 1.1978, 0.975599999999986, 0.165400000000005, 1.727, 1.19739999999999, -0.252600000000001, 1.13460000000001, 1.3048, 1.19479999999999, 0.313400000000001, 0.878999999999991, 1.12039999999999, 0.853000000000009,
+ 1.67920000000001, 0.856999999999999, 0.448599999999999, 1.2362, 0.953399999999988, 1.02859999999998, 0.563199999999995, 0.663000000000011, 0.723000000000013, 0.756599999999992, 0.256599999999992, -0.837600000000009, 0.620000000000005, 0.821599999999989, 0.216600000000028, 0.205600000000004, 0.220199999999977, 0.372599999999977, 0.334400000000016, 0.928400000000011, 0.972800000000007, 0.192400000000021, 0.487199999999973, -0.413000000000011, 0.807000000000016, 0.120600000000024, 0.769000000000005, 0.870799999999974, 0.66500000000002, 0.118200000000002, 0.401200000000017, 0.635199999999998, 0.135400000000004, 0.175599999999974, 1.16059999999999, 0.34620000000001, 0.521400000000028, -0.586599999999976, -1.16480000000001, 0.968399999999974, 0.836999999999989, 0.779600000000016, 0.985799999999983, },
+ // precision 7
+ { 91, 89.4934, 87.9758, 86.4574, 84.9718, 83.4954, 81.5302, 80.0756, 78.6374, 77.1782, 75.7888, 73.9522, 72.592, 71.2532, 69.9086, 68.5938, 66.9474, 65.6796, 64.4394, 63.2176, 61.9768, 60.4214, 59.2528, 58.0102, 56.8658, 55.7278, 54.3044, 53.1316, 52.093, 51.0032, 49.9092, 48.6306, 47.5294, 46.5756, 45.6508, 44.662, 43.552, 42.3724, 41.617, 40.5754, 39.7872, 38.8444, 37.7988, 36.8606, 36.2118, 35.3566, 34.4476, 33.5882, 32.6816, 32.0824, 31.0258, 30.6048, 29.4436, 28.7274, 27.957, 27.147, 26.4364, 25.7592, 25.3386, 24.781, 23.8028, 23.656, 22.6544, 21.996, 21.4718, 21.1544, 20.6098, 19.5956, 19.0616, 18.5758, 18.4878, 17.5244, 17.2146, 16.724, 15.8722, 15.5198, 15.0414, 14.941, 14.9048, 13.87, 13.4304, 13.028, 12.4708, 12.37, 12.0624, 11.4668, 11.5532, 11.4352, 11.2564, 10.2744, 10.2118, 9.74720000000002, 10.1456, 9.2928, 8.75040000000001, 8.55279999999999, 8.97899999999998, 8.21019999999999, 8.18340000000001, 7.3494, 7.32499999999999, 7.66140000000001, 6.90300000000002,
+ 7.25439999999998, 6.9042, 7.21499999999997, 6.28640000000001, 6.08139999999997, 6.6764, 6.30099999999999, 5.13900000000001, 5.65800000000002, 5.17320000000001, 4.59019999999998, 4.9538, 5.08280000000002, 4.92200000000003, 4.99020000000002, 4.7328, 5.4538, 4.11360000000002, 4.22340000000003, 4.08780000000002, 3.70800000000003, 4.15559999999999, 4.18520000000001, 3.63720000000001, 3.68220000000002, 3.77960000000002, 3.6078, 2.49160000000001, 3.13099999999997, 2.5376, 3.19880000000001, 3.21100000000001, 2.4502, 3.52820000000003, 2.91199999999998, 3.04480000000001, 2.7432, 2.85239999999999, 2.79880000000003, 2.78579999999999, 1.88679999999999, 2.98860000000002, 2.50639999999999, 1.91239999999999, 2.66160000000002, 2.46820000000002, 1.58199999999999, 1.30399999999997, 2.27379999999999, 2.68939999999998, 1.32900000000001, 3.10599999999999, 1.69080000000002, 2.13740000000001, 2.53219999999999, 1.88479999999998, 1.33240000000001, 1.45119999999997, 1.17899999999997,
+ 2.44119999999998, 1.60659999999996, 2.16700000000003, 0.77940000000001, 2.37900000000002, 2.06700000000001, 1.46000000000004, 2.91160000000002, 1.69200000000001, 0.954600000000028, 2.49300000000005, 2.2722, 1.33500000000004, 2.44899999999996, 1.20140000000004, 3.07380000000001, 2.09739999999999, 2.85640000000001, 2.29960000000005, 2.40899999999999, 1.97040000000004, 0.809799999999996, 1.65279999999996, 2.59979999999996, 0.95799999999997, 2.06799999999998, 2.32780000000002, 4.20159999999998, 1.96320000000003, 1.86400000000003, 1.42999999999995, 3.77940000000001, 1.27200000000005, 1.86440000000005, 2.20600000000002, 3.21900000000005, 1.5154, 2.61019999999996, },
+ // precision 8
+ { 183.2152, 180.2454, 177.2096, 173.6652, 170.6312, 167.6822, 164.249, 161.3296, 158.0038, 155.2074, 152.4612, 149.27, 146.5178, 143.4412, 140.8032, 138.1634, 135.1688, 132.6074, 129.6946, 127.2664, 124.8228, 122.0432, 119.6824, 116.9464, 114.6268, 112.2626, 109.8376, 107.4034, 104.8956, 102.8522, 100.7638, 98.3552, 96.3556, 93.7526, 91.9292, 89.8954, 87.8198, 85.7668, 83.298, 81.6688, 79.9466, 77.9746, 76.1672, 74.3474, 72.3028, 70.8912, 69.114, 67.4646, 65.9744, 64.4092, 62.6022, 60.843, 59.5684, 58.1652, 56.5426, 55.4152, 53.5388, 52.3592, 51.1366, 49.486, 48.3918, 46.5076, 45.509, 44.3834, 43.3498, 42.0668, 40.7346, 40.1228, 38.4528, 37.7, 36.644, 36.0518, 34.5774, 33.9068, 32.432, 32.1666, 30.434, 29.6644, 28.4894, 27.6312, 26.3804, 26.292, 25.5496000000001, 25.0234, 24.8206, 22.6146, 22.4188, 22.117, 20.6762, 20.6576, 19.7864, 19.509, 18.5334, 17.9204, 17.772, 16.2924, 16.8654, 15.1836, 15.745, 15.1316, 15.0386, 14.0136, 13.6342, 12.6196, 12.1866, 12.4281999999999,
+ 11.3324, 10.4794000000001, 11.5038, 10.129, 9.52800000000002, 10.3203999999999, 9.46299999999997, 9.79280000000006, 9.12300000000005, 8.74180000000001, 9.2192, 7.51020000000005, 7.60659999999996, 7.01840000000004, 7.22239999999999, 7.40139999999997, 6.76179999999999, 7.14359999999999, 5.65060000000005, 5.63779999999997, 5.76599999999996, 6.75139999999999, 5.57759999999996, 3.73220000000003, 5.8048, 5.63019999999995, 4.93359999999996, 3.47979999999995, 4.33879999999999, 3.98940000000005, 3.81960000000004, 3.31359999999995, 3.23080000000004, 3.4588, 3.08159999999998, 3.4076, 3.00639999999999, 2.38779999999997, 2.61900000000003, 1.99800000000005, 3.34820000000002, 2.95060000000001, 0.990999999999985, 2.11440000000005, 2.20299999999997, 2.82219999999995, 2.73239999999998, 2.7826, 3.76660000000004, 2.26480000000004, 2.31280000000004, 2.40819999999997, 2.75360000000001, 3.33759999999995, 2.71559999999999, 1.7478000000001, 1.42920000000004, 2.39300000000003,
+ 2.22779999999989, 2.34339999999997, 0.87259999999992, 3.88400000000001, 1.80600000000004, 1.91759999999999, 1.16779999999994, 1.50320000000011, 2.52500000000009, 0.226400000000012, 2.31500000000005, 0.930000000000064, 1.25199999999995, 2.14959999999996, 0.0407999999999902, 2.5447999999999, 1.32960000000003, 0.197400000000016, 2.52620000000002, 3.33279999999991, -1.34300000000007, 0.422199999999975, 0.917200000000093, 1.12920000000008, 1.46060000000011, 1.45779999999991, 2.8728000000001, 3.33359999999993, -1.34079999999994, 1.57680000000005, 0.363000000000056, 1.40740000000005, 0.656600000000026, 0.801400000000058, -0.454600000000028, 1.51919999999996, },
+ // precision 9
+ { 368, 361.8294, 355.2452, 348.6698, 342.1464, 336.2024, 329.8782, 323.6598, 317.462, 311.2826, 305.7102, 299.7416, 293.9366, 288.1046, 282.285, 277.0668, 271.306, 265.8448, 260.301, 254.9886, 250.2422, 244.8138, 239.7074, 234.7428, 229.8402, 225.1664, 220.3534, 215.594, 210.6886, 205.7876, 201.65, 197.228, 192.8036, 188.1666, 184.0818, 180.0824, 176.2574, 172.302, 168.1644, 164.0056, 160.3802, 156.7192, 152.5234, 149.2084, 145.831, 142.485, 139.1112, 135.4764, 131.76, 129.3368, 126.5538, 122.5058, 119.2646, 116.5902, 113.3818, 110.8998, 107.9532, 105.2062, 102.2798, 99.4728, 96.9582, 94.3292, 92.171, 89.7809999999999, 87.5716, 84.7048, 82.5322, 79.875, 78.3972, 75.3464, 73.7274, 71.2834, 70.1444, 68.4263999999999, 66.0166, 64.018, 62.0437999999999, 60.3399999999999, 58.6856, 57.9836, 55.0311999999999, 54.6769999999999, 52.3188, 51.4846, 49.4423999999999, 47.739, 46.1487999999999, 44.9202, 43.4059999999999, 42.5342000000001, 41.2834, 38.8954000000001, 38.3286000000001,
+ 36.2146, 36.6684, 35.9946, 33.123, 33.4338, 31.7378000000001, 29.076, 28.9692, 27.4964, 27.0998, 25.9864, 26.7754, 24.3208, 23.4838, 22.7388000000001, 24.0758000000001, 21.9097999999999, 20.9728, 19.9228000000001, 19.9292, 16.617, 17.05, 18.2996000000001, 15.6128000000001, 15.7392, 14.5174, 13.6322, 12.2583999999999, 13.3766000000001, 11.423, 13.1232, 9.51639999999998, 10.5938000000001, 9.59719999999993, 8.12220000000002, 9.76739999999995, 7.50440000000003, 7.56999999999994, 6.70440000000008, 6.41419999999994, 6.71019999999999, 5.60940000000005, 4.65219999999999, 6.84099999999989, 3.4072000000001, 3.97859999999991, 3.32760000000007, 5.52160000000003, 3.31860000000006, 2.06940000000009, 4.35400000000004, 1.57500000000005, 0.280799999999999, 2.12879999999996, -0.214799999999968, -0.0378000000000611, -0.658200000000079, 0.654800000000023, -0.0697999999999865, 0.858400000000074, -2.52700000000004, -2.1751999999999, -3.35539999999992, -1.04019999999991,
+ -0.651000000000067, -2.14439999999991, -1.96659999999997, -3.97939999999994, -0.604400000000169, -3.08260000000018, -3.39159999999993, -5.29640000000018, -5.38920000000007, -5.08759999999984, -4.69900000000007, -5.23720000000003, -3.15779999999995, -4.97879999999986, -4.89899999999989, -7.48880000000008, -5.94799999999987, -5.68060000000014, -6.67180000000008, -4.70499999999993, -7.27779999999984, -4.6579999999999, -4.4362000000001, -4.32139999999981, -5.18859999999995, -6.66879999999992, -6.48399999999992, -5.1260000000002, -4.4032000000002, -6.13500000000022, -5.80819999999994, -4.16719999999987, -4.15039999999999, -7.45600000000013, -7.24080000000004, -9.83179999999993, -5.80420000000004, -8.6561999999999, -6.99940000000015, -10.5473999999999, -7.34139999999979, -6.80999999999995, -6.29719999999998, -6.23199999999997, },
+ // precision 10
+ { 737.1256, 724.4234, 711.1064, 698.4732, 685.4636, 673.0644, 660.488, 647.9654, 636.0832, 623.7864, 612.1992, 600.2176, 588.5228, 577.1716, 565.7752, 554.899, 543.6126, 532.6492, 521.9474, 511.5214, 501.1064, 490.6364, 480.2468, 470.4588, 460.3832, 451.0584, 440.8606, 431.3868, 422.5062, 413.1862, 404.463, 395.339, 386.1936, 378.1292, 369.1854, 361.2908, 353.3324, 344.8518, 337.5204, 329.4854, 321.9318, 314.552, 306.4658, 299.4256, 292.849, 286.152, 278.8956, 271.8792, 265.118, 258.62, 252.5132, 245.9322, 239.7726, 233.6086, 227.5332, 222.5918, 216.4294, 210.7662, 205.4106, 199.7338, 194.9012, 188.4486, 183.1556, 178.6338, 173.7312, 169.6264, 163.9526, 159.8742, 155.8326, 151.1966, 147.5594, 143.07, 140.037, 134.1804, 131.071, 127.4884, 124.0848, 120.2944, 117.333, 112.9626, 110.2902, 107.0814, 103.0334, 99.4832000000001, 96.3899999999999, 93.7202000000002, 90.1714000000002, 87.2357999999999, 85.9346, 82.8910000000001, 80.0264000000002, 78.3834000000002,
+ 75.1543999999999, 73.8683999999998, 70.9895999999999, 69.4367999999999, 64.8701999999998, 65.0408000000002, 61.6738, 59.5207999999998, 57.0158000000001, 54.2302, 53.0962, 50.4985999999999, 52.2588000000001, 47.3914, 45.6244000000002, 42.8377999999998, 43.0072, 40.6516000000001, 40.2453999999998, 35.2136, 36.4546, 33.7849999999999, 33.2294000000002, 32.4679999999998, 30.8670000000002, 28.6507999999999, 28.9099999999999, 27.5983999999999, 26.1619999999998, 24.5563999999999, 23.2328000000002, 21.9484000000002, 21.5902000000001, 21.3346000000001, 17.7031999999999, 20.6111999999998, 19.5545999999999, 15.7375999999999, 17.0720000000001, 16.9517999999998, 15.326, 13.1817999999998, 14.6925999999999, 13.0859999999998, 13.2754, 10.8697999999999, 11.248, 7.3768, 4.72339999999986, 7.97899999999981, 8.7503999999999, 7.68119999999999, 9.7199999999998, 7.73919999999998, 5.6224000000002, 7.44560000000001, 6.6601999999998, 5.9058, 4.00199999999995, 4.51699999999983,
+ 4.68240000000014, 3.86220000000003, 5.13639999999987, 5.98500000000013, 2.47719999999981, 2.61999999999989, 1.62800000000016, 4.65000000000009, 0.225599999999758, 0.831000000000131, -0.359400000000278, 1.27599999999984, -2.92559999999958, -0.0303999999996449, 2.37079999999969, -2.0033999999996, 0.804600000000391, 0.30199999999968, 1.1247999999996, -2.6880000000001, 0.0321999999996478, -1.18099999999959, -3.9402, -1.47940000000017, -0.188400000000001, -2.10720000000038, -2.04159999999956, -3.12880000000041, -4.16160000000036, -0.612799999999879, -3.48719999999958, -8.17900000000009, -5.37780000000021, -4.01379999999972, -5.58259999999973, -5.73719999999958, -7.66799999999967, -5.69520000000011, -1.1247999999996, -5.58520000000044, -8.04560000000038, -4.64840000000004, -11.6468000000004, -7.97519999999986, -5.78300000000036, -7.67420000000038, -10.6328000000003, -9.81720000000041, },
+ // precision 11
+ { 1476, 1449.6014, 1423.5802, 1397.7942, 1372.3042, 1347.2062, 1321.8402, 1297.2292, 1272.9462, 1248.9926, 1225.3026, 1201.4252, 1178.0578, 1155.6092, 1132.626, 1110.5568, 1088.527, 1066.5154, 1045.1874, 1024.3878, 1003.37, 982.1972, 962.5728, 942.1012, 922.9668, 903.292, 884.0772, 864.8578, 846.6562, 828.041, 809.714, 792.3112, 775.1806, 757.9854, 740.656, 724.346, 707.5154, 691.8378, 675.7448, 659.6722, 645.5722, 630.1462, 614.4124, 600.8728, 585.898, 572.408, 558.4926, 544.4938, 531.6776, 517.282, 505.7704, 493.1012, 480.7388, 467.6876, 456.1872, 445.5048, 433.0214, 420.806, 411.409, 400.4144, 389.4294, 379.2286, 369.651, 360.6156, 350.337, 342.083, 332.1538, 322.5094, 315.01, 305.6686, 298.1678, 287.8116, 280.9978, 271.9204, 265.3286, 257.5706, 249.6014, 242.544, 235.5976, 229.583, 220.9438, 214.672, 208.2786, 201.8628, 195.1834, 191.505, 186.1816, 178.5188, 172.2294, 167.8908, 161.0194, 158.052, 151.4588, 148.1596, 143.4344, 138.5238, 133.13, 127.6374, 124.8162,
+ 118.7894, 117.3984, 114.6078, 109.0858, 105.1036, 103.6258, 98.6018000000004, 95.7618000000002, 93.5821999999998, 88.5900000000001, 86.9992000000002, 82.8800000000001, 80.4539999999997, 74.6981999999998, 74.3644000000004, 73.2914000000001, 65.5709999999999, 66.9232000000002, 65.1913999999997, 62.5882000000001, 61.5702000000001, 55.7035999999998, 56.1764000000003, 52.7596000000003, 53.0302000000001, 49.0609999999997, 48.4694, 44.933, 46.0474000000004, 44.7165999999997, 41.9416000000001, 39.9207999999999, 35.6328000000003, 35.5276000000003, 33.1934000000001, 33.2371999999996, 33.3864000000003, 33.9228000000003, 30.2371999999996, 29.1373999999996, 25.2272000000003, 24.2942000000003, 19.8338000000003, 18.9005999999999, 23.0907999999999, 21.8544000000002, 19.5176000000001, 15.4147999999996, 16.9314000000004, 18.6737999999996, 12.9877999999999, 14.3688000000002, 12.0447999999997, 15.5219999999999, 12.5299999999997, 14.5940000000001, 14.3131999999996, 9.45499999999993,
+ 12.9441999999999, 3.91139999999996, 13.1373999999996, 5.44720000000052, 9.82779999999912, 7.87279999999919, 3.67760000000089, 5.46980000000076, 5.55099999999948, 5.65979999999945, 3.89439999999922, 3.1275999999998, 5.65140000000065, 6.3062000000009, 3.90799999999945, 1.87060000000019, 5.17020000000048, 2.46680000000015, 0.770000000000437, -3.72340000000077, 1.16400000000067, 8.05340000000069, 0.135399999999208, 2.15940000000046, 0.766999999999825, 1.0594000000001, 3.15500000000065, -0.287399999999252, 2.37219999999979, -2.86620000000039, -1.63199999999961, -2.22979999999916, -0.15519999999924, -1.46039999999994, -0.262199999999211, -2.34460000000036, -2.8078000000005, -3.22179999999935, -5.60159999999996, -8.42200000000048, -9.43740000000071, 0.161799999999857, -10.4755999999998, -10.0823999999993, },
+ // precision 12
+ { 2953, 2900.4782, 2848.3568, 2796.3666, 2745.324, 2694.9598, 2644.648, 2595.539, 2546.1474, 2498.2576, 2450.8376, 2403.6076, 2357.451, 2311.38, 2266.4104, 2221.5638, 2176.9676, 2134.193, 2090.838, 2048.8548, 2007.018, 1966.1742, 1925.4482, 1885.1294, 1846.4776, 1807.4044, 1768.8724, 1731.3732, 1693.4304, 1657.5326, 1621.949, 1586.5532, 1551.7256, 1517.6182, 1483.5186, 1450.4528, 1417.865, 1385.7164, 1352.6828, 1322.6708, 1291.8312, 1260.9036, 1231.476, 1201.8652, 1173.6718, 1145.757, 1119.2072, 1092.2828, 1065.0434, 1038.6264, 1014.3192, 988.5746, 965.0816, 940.1176, 917.9796, 894.5576, 871.1858, 849.9144, 827.1142, 805.0818, 783.9664, 763.9096, 742.0816, 724.3962, 706.3454, 688.018, 667.4214, 650.3106, 633.0686, 613.8094, 597.818, 581.4248, 563.834, 547.363, 531.5066, 520.455400000001, 505.583199999999, 488.366, 476.480799999999, 459.7682, 450.0522, 434.328799999999, 423.952799999999, 408.727000000001, 399.079400000001, 387.252200000001, 373.987999999999,
+ 360.852000000001, 351.6394, 339.642, 330.902400000001, 322.661599999999, 311.662200000001, 301.3254, 291.7484, 279.939200000001, 276.7508, 263.215200000001, 254.811400000001, 245.5494, 242.306399999999, 234.8734, 223.787200000001, 217.7156, 212.0196, 200.793, 195.9748, 189.0702, 182.449199999999, 177.2772, 170.2336, 164.741, 158.613600000001, 155.311, 147.5964, 142.837, 137.3724, 132.0162, 130.0424, 121.9804, 120.451800000001, 114.8968, 111.585999999999, 105.933199999999, 101.705, 98.5141999999996, 95.0488000000005, 89.7880000000005, 91.4750000000004, 83.7764000000006, 80.9698000000008, 72.8574000000008, 73.1615999999995, 67.5838000000003, 62.6263999999992, 63.2638000000006, 66.0977999999996, 52.0843999999997, 58.9956000000002, 47.0912000000008, 46.4956000000002, 48.4383999999991, 47.1082000000006, 43.2392, 37.2759999999998, 40.0283999999992, 35.1864000000005, 35.8595999999998, 32.0998, 28.027, 23.6694000000007, 33.8266000000003, 26.3736000000008,
+ 27.2008000000005, 21.3245999999999, 26.4115999999995, 23.4521999999997, 19.5013999999992, 19.8513999999996, 10.7492000000002, 18.6424000000006, 13.1265999999996, 18.2436000000016, 6.71860000000015, 3.39459999999963, 6.33759999999893, 7.76719999999841, 0.813999999998487, 3.82819999999992, 0.826199999999517, 8.07440000000133, -1.59080000000176, 5.01780000000144, 0.455399999998917, -0.24199999999837, 0.174800000000687, -9.07640000000174, -4.20160000000033, -3.77520000000004, -4.75179999999818, -5.3724000000002, -8.90680000000066, -6.10239999999976, -5.74120000000039, -9.95339999999851, -3.86339999999836, -13.7304000000004, -16.2710000000006, -7.51359999999841, -3.30679999999847, -13.1339999999982, -10.0551999999989, -6.72019999999975, -8.59660000000076, -10.9307999999983, -1.8775999999998, -4.82259999999951, -13.7788, -21.6470000000008, -10.6735999999983, -15.7799999999988, },
+ // precision 13
+ { 5907.5052, 5802.2672, 5697.347, 5593.5794, 5491.2622, 5390.5514, 5290.3376, 5191.6952, 5093.5988, 4997.3552, 4902.5972, 4808.3082, 4715.5646, 4624.109, 4533.8216, 4444.4344, 4356.3802, 4269.2962, 4183.3784, 4098.292, 4014.79, 3932.4574, 3850.6036, 3771.2712, 3691.7708, 3615.099, 3538.1858, 3463.4746, 3388.8496, 3315.6794, 3244.5448, 3173.7516, 3103.3106, 3033.6094, 2966.5642, 2900.794, 2833.7256, 2769.81, 2707.3196, 2644.0778, 2583.9916, 2523.4662, 2464.124, 2406.073, 2347.0362, 2292.1006, 2238.1716, 2182.7514, 2128.4884, 2077.1314, 2025.037, 1975.3756, 1928.933, 1879.311, 1831.0006, 1783.2144, 1738.3096, 1694.5144, 1649.024, 1606.847, 1564.7528, 1525.3168, 1482.5372, 1443.9668, 1406.5074, 1365.867, 1329.2186, 1295.4186, 1257.9716, 1225.339, 1193.2972, 1156.3578, 1125.8686, 1091.187, 1061.4094, 1029.4188, 1000.9126, 972.3272, 944.004199999999, 915.7592, 889.965, 862.834200000001, 840.4254, 812.598399999999, 785.924200000001, 763.050999999999, 741.793799999999, 721.466,
+ 699.040799999999, 677.997200000002, 649.866999999998, 634.911800000002, 609.8694, 591.981599999999, 570.2922, 557.129199999999, 538.3858, 521.872599999999, 502.951400000002, 495.776399999999, 475.171399999999, 459.751, 439.995200000001, 426.708999999999, 413.7016, 402.3868, 387.262599999998, 372.0524, 357.050999999999, 342.5098, 334.849200000001, 322.529399999999, 311.613799999999, 295.848000000002, 289.273000000001, 274.093000000001, 263.329600000001, 251.389599999999, 245.7392, 231.9614, 229.7952, 217.155200000001, 208.9588, 199.016599999999, 190.839199999999, 180.6976, 176.272799999999, 166.976999999999, 162.5252, 151.196400000001, 149.386999999999, 133.981199999998, 130.0586, 130.164000000001, 122.053400000001, 110.7428, 108.1276, 106.232400000001, 100.381600000001, 98.7668000000012, 86.6440000000002, 79.9768000000004, 82.4722000000002, 68.7026000000005, 70.1186000000016, 71.9948000000004, 58.998599999999, 59.0492000000013, 56.9818000000014, 47.5338000000011,
+ 42.9928, 51.1591999999982, 37.2740000000013, 42.7220000000016, 31.3734000000004, 26.8090000000011, 25.8934000000008, 26.5286000000015, 29.5442000000003, 19.3503999999994, 26.0760000000009, 17.9527999999991, 14.8419999999969, 10.4683999999979, 8.65899999999965, 9.86720000000059, 4.34139999999752, -0.907800000000861, -3.32080000000133, -0.936199999996461, -11.9916000000012, -8.87000000000262, -6.33099999999831, -11.3366000000024, -15.9207999999999, -9.34659999999712, -15.5034000000014, -19.2097999999969, -15.357799999998, -28.2235999999975, -30.6898000000001, -19.3271999999997, -25.6083999999973, -24.409599999999, -13.6385999999984, -33.4473999999973, -32.6949999999997, -28.9063999999998, -31.7483999999968, -32.2935999999972, -35.8329999999987, -47.620600000002, -39.0855999999985, -33.1434000000008, -46.1371999999974, -37.5892000000022, -46.8164000000033, -47.3142000000007, -60.2914000000019, -37.7575999999972, },
+ // precision 14
+ { 11816.475, 11605.0046, 11395.3792, 11188.7504, 10984.1814, 10782.0086, 10582.0072, 10384.503, 10189.178, 9996.2738, 9806.0344, 9617.9798, 9431.394, 9248.7784, 9067.6894, 8889.6824, 8712.9134, 8538.8624, 8368.4944, 8197.7956, 8031.8916, 7866.6316, 7703.733, 7544.5726, 7386.204, 7230.666, 7077.8516, 6926.7886, 6778.6902, 6631.9632, 6487.304, 6346.7486, 6206.4408, 6070.202, 5935.2576, 5799.924, 5671.0324, 5541.9788, 5414.6112, 5290.0274, 5166.723, 5047.6906, 4929.162, 4815.1406, 4699.127, 4588.5606, 4477.7394, 4369.4014, 4264.2728, 4155.9224, 4055.581, 3955.505, 3856.9618, 3761.3828, 3666.9702, 3575.7764, 3482.4132, 3395.0186, 3305.8852, 3221.415, 3138.6024, 3056.296, 2970.4494, 2896.1526, 2816.8008, 2740.2156, 2670.497, 2594.1458, 2527.111, 2460.8168, 2387.5114, 2322.9498, 2260.6752, 2194.2686, 2133.7792, 2074.767, 2015.204, 1959.4226, 1898.6502, 1850.006, 1792.849, 1741.4838, 1687.9778, 1638.1322, 1589.3266, 1543.1394, 1496.8266, 1447.8516, 1402.7354, 1361.9606,
+ 1327.0692, 1285.4106, 1241.8112, 1201.6726, 1161.973, 1130.261, 1094.2036, 1048.2036, 1020.6436, 990.901400000002, 961.199800000002, 924.769800000002, 899.526400000002, 872.346400000002, 834.375, 810.432000000001, 780.659800000001, 756.013800000001, 733.479399999997, 707.923999999999, 673.858, 652.222399999999, 636.572399999997, 615.738599999997, 586.696400000001, 564.147199999999, 541.679600000003, 523.943599999999, 505.714599999999, 475.729599999999, 461.779600000002, 449.750800000002, 439.020799999998, 412.7886, 400.245600000002, 383.188199999997, 362.079599999997, 357.533799999997, 334.319000000003, 327.553399999997, 308.559399999998, 291.270199999999, 279.351999999999, 271.791400000002, 252.576999999997, 247.482400000001, 236.174800000001, 218.774599999997, 220.155200000001, 208.794399999999, 201.223599999998, 182.995600000002, 185.5268, 164.547400000003, 176.5962, 150.689599999998, 157.8004, 138.378799999999, 134.021200000003, 117.614399999999,
+ 108.194000000003, 97.0696000000025, 89.6042000000016, 95.6030000000028, 84.7810000000027, 72.635000000002, 77.3482000000004, 59.4907999999996, 55.5875999999989, 50.7346000000034, 61.3916000000027, 50.9149999999936, 39.0384000000049, 58.9395999999979, 29.633600000001, 28.2032000000036, 26.0078000000067, 17.0387999999948, 9.22000000000116, 13.8387999999977, 8.07240000000456, 14.1549999999988, 15.3570000000036, 3.42660000000615, 6.24820000000182, -2.96940000000177, -8.79940000000352, -5.97860000000219, -14.4048000000039, -3.4143999999942, -13.0148000000045, -11.6977999999945, -25.7878000000055, -22.3185999999987, -24.409599999999, -31.9756000000052, -18.9722000000038, -22.8678000000073, -30.8972000000067, -32.3715999999986, -22.3907999999938, -43.6720000000059, -35.9038, -39.7492000000057, -54.1641999999993, -45.2749999999942, -42.2989999999991, -44.1089999999967, -64.3564000000042, -49.9551999999967, -42.6116000000038, },
+ // precision 15
+ { 23634.0036, 23210.8034, 22792.4744, 22379.1524, 21969.7928, 21565.326, 21165.3532, 20770.2806, 20379.9892, 19994.7098, 19613.318, 19236.799, 18865.4382, 18498.8244, 18136.5138, 17778.8668, 17426.2344, 17079.32, 16734.778, 16397.2418, 16063.3324, 15734.0232, 15409.731, 15088.728, 14772.9896, 14464.1402, 14157.5588, 13855.5958, 13559.3296, 13264.9096, 12978.326, 12692.0826, 12413.8816, 12137.3192, 11870.2326, 11602.5554, 11340.3142, 11079.613, 10829.5908, 10583.5466, 10334.0344, 10095.5072, 9859.694, 9625.2822, 9395.7862, 9174.0586, 8957.3164, 8738.064, 8524.155, 8313.7396, 8116.9168, 7913.542, 7718.4778, 7521.65, 7335.5596, 7154.2906, 6968.7396, 6786.3996, 6613.236, 6437.406, 6270.6598, 6107.7958, 5945.7174, 5787.6784, 5635.5784, 5482.308, 5337.9784, 5190.0864, 5045.9158, 4919.1386, 4771.817, 4645.7742, 4518.4774, 4385.5454, 4262.6622, 4142.74679999999, 4015.5318, 3897.9276, 3790.7764, 3685.13800000001, 3573.6274, 3467.9706, 3368.61079999999, 3271.5202, 3170.3848,
+ 3076.4656, 2982.38400000001, 2888.4664, 2806.4868, 2711.9564, 2634.1434, 2551.3204, 2469.7662, 2396.61139999999, 2318.9902, 2243.8658, 2171.9246, 2105.01360000001, 2028.8536, 1960.9952, 1901.4096, 1841.86079999999, 1777.54700000001, 1714.5802, 1654.65059999999, 1596.311, 1546.2016, 1492.3296, 1433.8974, 1383.84600000001, 1339.4152, 1293.5518, 1245.8686, 1193.50659999999, 1162.27959999999, 1107.19439999999, 1069.18060000001, 1035.09179999999, 999.679000000004, 957.679999999993, 925.300199999998, 888.099400000006, 848.638600000006, 818.156400000007, 796.748399999997, 752.139200000005, 725.271200000003, 692.216, 671.633600000001, 647.939799999993, 621.670599999998, 575.398799999995, 561.226599999995, 532.237999999998, 521.787599999996, 483.095799999996, 467.049599999998, 465.286399999997, 415.548599999995, 401.047399999996, 380.607999999993, 377.362599999993, 347.258799999996, 338.371599999999, 310.096999999994, 301.409199999995, 276.280799999993, 265.586800000005,
+ 258.994399999996, 223.915999999997, 215.925399999993, 213.503800000006, 191.045400000003, 166.718200000003, 166.259000000005, 162.941200000001, 148.829400000002, 141.645999999993, 123.535399999993, 122.329800000007, 89.473399999988, 80.1962000000058, 77.5457999999926, 59.1056000000099, 83.3509999999951, 52.2906000000075, 36.3979999999865, 40.6558000000077, 42.0003999999899, 19.6630000000005, 19.7153999999864, -8.38539999999921, -0.692799999989802, 0.854800000000978, 3.23219999999856, -3.89040000000386, -5.25880000001052, -24.9052000000083, -22.6837999999989, -26.4286000000138, -34.997000000003, -37.0216000000073, -43.430400000012, -58.2390000000014, -68.8034000000043, -56.9245999999985, -57.8583999999973, -77.3097999999882, -73.2793999999994, -81.0738000000129, -87.4530000000086, -65.0254000000132, -57.296399999992, -96.2746000000043, -103.25, -96.081600000005, -91.5542000000132, -102.465200000006, -107.688599999994, -101.458000000013, -109.715800000005, },
+ // precision 16
+ { 47270, 46423.3584, 45585.7074, 44757.152, 43938.8416, 43130.9514, 42330.03, 41540.407, 40759.6348, 39988.206, 39226.5144, 38473.2096, 37729.795, 36997.268, 36272.6448, 35558.665, 34853.0248, 34157.4472, 33470.5204, 32793.5742, 32127.0194, 31469.4182, 30817.6136, 30178.6968, 29546.8908, 28922.8544, 28312.271, 27707.0924, 27114.0326, 26526.692, 25948.6336, 25383.7826, 24823.5998, 24272.2974, 23732.2572, 23201.4976, 22674.2796, 22163.6336, 21656.515, 21161.7362, 20669.9368, 20189.4424, 19717.3358, 19256.3744, 18795.9638, 18352.197, 17908.5738, 17474.391, 17052.918, 16637.2236, 16228.4602, 15823.3474, 15428.6974, 15043.0284, 14667.6278, 14297.4588, 13935.2882, 13578.5402, 13234.6032, 12882.1578, 12548.0728, 12219.231, 11898.0072, 11587.2626, 11279.9072, 10973.5048, 10678.5186, 10392.4876, 10105.2556, 9825.766, 9562.5444, 9294.2222, 9038.2352, 8784.848, 8533.2644, 8301.7776, 8058.30859999999, 7822.94579999999, 7599.11319999999, 7366.90779999999, 7161.217, 6957.53080000001,
+ 6736.212, 6548.21220000001, 6343.06839999999, 6156.28719999999, 5975.15419999999, 5791.75719999999, 5621.32019999999, 5451.66, 5287.61040000001, 5118.09479999999, 4957.288, 4798.4246, 4662.17559999999, 4512.05900000001, 4364.68539999999, 4220.77720000001, 4082.67259999999, 3957.19519999999, 3842.15779999999, 3699.3328, 3583.01180000001, 3473.8964, 3338.66639999999, 3233.55559999999, 3117.799, 3008.111, 2909.69140000001, 2814.86499999999, 2719.46119999999, 2624.742, 2532.46979999999, 2444.7886, 2370.1868, 2272.45259999999, 2196.19260000001, 2117.90419999999, 2023.2972, 1969.76819999999, 1885.58979999999, 1833.2824, 1733.91200000001, 1682.54920000001, 1604.57980000001, 1556.11240000001, 1491.3064, 1421.71960000001, 1371.22899999999, 1322.1324, 1264.7892, 1196.23920000001, 1143.8474, 1088.67240000001, 1073.60380000001, 1023.11660000001, 959.036400000012, 927.433199999999, 906.792799999996, 853.433599999989, 841.873800000001, 791.1054, 756.899999999994,
+ 704.343200000003, 672.495599999995, 622.790399999998, 611.254799999995, 567.283200000005, 519.406599999988, 519.188400000014, 495.312800000014, 451.350799999986, 443.973399999988, 431.882199999993, 392.027000000002, 380.924200000009, 345.128999999986, 298.901400000002, 287.771999999997, 272.625, 247.253000000026, 222.490600000019, 223.590000000026, 196.407599999977, 176.425999999978, 134.725199999986, 132.4804, 110.445599999977, 86.7939999999944, 56.7038000000175, 64.915399999998, 38.3726000000024, 37.1606000000029, 46.170999999973, 49.1716000000015, 15.3362000000197, 6.71639999997569, -34.8185999999987, -39.4476000000141, 12.6830000000191, -12.3331999999937, -50.6565999999875, -59.9538000000175, -65.1054000000004, -70.7576000000117, -106.325200000021, -126.852200000023, -110.227599999984, -132.885999999999, -113.897200000007, -142.713800000027, -151.145399999979, -150.799200000009, -177.756200000003, -156.036399999983, -182.735199999996, -177.259399999981,
+ -198.663600000029, -174.577600000019, -193.84580000001, },
+ // precision 17
+ { 94541, 92848.811, 91174.019, 89517.558, 87879.9705, 86262.7565, 84663.5125, 83083.7435, 81521.7865, 79977.272, 78455.9465, 76950.219, 75465.432, 73994.152, 72546.71, 71115.2345, 69705.6765, 68314.937, 66944.2705, 65591.255, 64252.9485, 62938.016, 61636.8225, 60355.592, 59092.789, 57850.568, 56624.518, 55417.343, 54231.1415, 53067.387, 51903.526, 50774.649, 49657.6415, 48561.05, 47475.7575, 46410.159, 45364.852, 44327.053, 43318.4005, 42325.6165, 41348.4595, 40383.6265, 39436.77, 38509.502, 37594.035, 36695.939, 35818.6895, 34955.691, 34115.8095, 33293.949, 32465.0775, 31657.6715, 30877.2585, 30093.78, 29351.3695, 28594.1365, 27872.115, 27168.7465, 26477.076, 25774.541, 25106.5375, 24452.5135, 23815.5125, 23174.0655, 22555.2685, 21960.2065, 21376.3555, 20785.1925, 20211.517, 19657.0725, 19141.6865, 18579.737, 18081.3955, 17578.995, 17073.44, 16608.335, 16119.911, 15651.266, 15194.583, 14749.0495, 14343.4835, 13925.639, 13504.509, 13099.3885, 12691.2855, 12328.018,
+ 11969.0345, 11596.5145, 11245.6355, 10917.6575, 10580.9785, 10277.8605, 9926.58100000001, 9605.538, 9300.42950000003, 8989.97850000003, 8728.73249999998, 8448.3235, 8175.31050000002, 7898.98700000002, 7629.79100000003, 7413.76199999999, 7149.92300000001, 6921.12650000001, 6677.1545, 6443.28000000003, 6278.23450000002, 6014.20049999998, 5791.20299999998, 5605.78450000001, 5438.48800000001, 5234.2255, 5059.6825, 4887.43349999998, 4682.935, 4496.31099999999, 4322.52250000002, 4191.42499999999, 4021.24200000003, 3900.64799999999, 3762.84250000003, 3609.98050000001, 3502.29599999997, 3363.84250000003, 3206.54849999998, 3079.70000000001, 2971.42300000001, 2867.80349999998, 2727.08100000001, 2630.74900000001, 2496.6165, 2440.902, 2356.19150000002, 2235.58199999999, 2120.54149999999, 2012.25449999998, 1933.35600000003, 1820.93099999998, 1761.54800000001, 1663.09350000002, 1578.84600000002, 1509.48149999999, 1427.3345, 1379.56150000001, 1306.68099999998, 1212.63449999999,
+ 1084.17300000001, 1124.16450000001, 1060.69949999999, 1007.48849999998, 941.194499999983, 879.880500000028, 836.007500000007, 782.802000000025, 748.385499999975, 647.991500000004, 626.730500000005, 570.776000000013, 484.000500000024, 513.98550000001, 418.985499999952, 386.996999999974, 370.026500000036, 355.496999999974, 356.731499999994, 255.92200000002, 259.094000000041, 205.434499999974, 165.374500000034, 197.347500000033, 95.718499999959, 67.6165000000037, 54.6970000000438, 31.7395000000251, -15.8784999999916, 8.42500000004657, -26.3754999999655, -118.425500000012, -66.6629999999423, -42.9745000000112, -107.364999999991, -189.839000000036, -162.611499999999, -164.964999999967, -189.079999999958, -223.931499999948, -235.329999999958, -269.639500000048, -249.087999999989, -206.475499999942, -283.04449999996, -290.667000000016, -304.561499999953, -336.784499999951, -380.386500000022, -283.280499999993, -364.533000000054, -389.059499999974, -364.454000000027,
+ -415.748000000021, -417.155000000028, },
+ // precision 18
+ { 189083, 185696.913, 182348.774, 179035.946, 175762.762, 172526.444, 169329.754, 166166.099, 163043.269, 159958.91, 156907.912, 153906.845, 150924.199, 147996.568, 145093.457, 142239.233, 139421.475, 136632.27, 133889.588, 131174.2, 128511.619, 125868.621, 123265.385, 120721.061, 118181.769, 115709.456, 113252.446, 110840.198, 108465.099, 106126.164, 103823.469, 101556.618, 99308.004, 97124.508, 94937.803, 92833.731, 90745.061, 88677.627, 86617.47, 84650.442, 82697.833, 80769.132, 78879.629, 77014.432, 75215.626, 73384.587, 71652.482, 69895.93, 68209.301, 66553.669, 64921.981, 63310.323, 61742.115, 60205.018, 58698.658, 57190.657, 55760.865, 54331.169, 52908.167, 51550.273, 50225.254, 48922.421, 47614.533, 46362.049, 45098.569, 43926.083, 42736.03, 41593.473, 40425.26, 39316.237, 38243.651, 37170.617, 36114.609, 35084.19, 34117.233, 33206.509, 32231.505, 31318.728, 30403.404, 29540.0550000001, 28679.236, 27825.862, 26965.216, 26179.148, 25462.08, 24645.952, 23922.523,
+ 23198.144, 22529.128, 21762.4179999999, 21134.779, 20459.117, 19840.818, 19187.04, 18636.3689999999, 17982.831, 17439.7389999999, 16874.547, 16358.2169999999, 15835.684, 15352.914, 14823.681, 14329.313, 13816.897, 13342.874, 12880.882, 12491.648, 12021.254, 11625.392, 11293.7610000001, 10813.697, 10456.209, 10099.074, 9755.39000000001, 9393.18500000006, 9047.57900000003, 8657.98499999999, 8395.85900000005, 8033, 7736.95900000003, 7430.59699999995, 7258.47699999996, 6924.58200000005, 6691.29399999999, 6357.92500000005, 6202.05700000003, 5921.19700000004, 5628.28399999999, 5404.96799999999, 5226.71100000001, 4990.75600000005, 4799.77399999998, 4622.93099999998, 4472.478, 4171.78700000001, 3957.46299999999, 3868.95200000005, 3691.14300000004, 3474.63100000005, 3341.67200000002, 3109.14000000001, 3071.97400000005, 2796.40399999998, 2756.17799999996, 2611.46999999997, 2471.93000000005, 2382.26399999997, 2209.22400000005, 2142.28399999999, 2013.96100000001,
+ 1911.18999999994, 1818.27099999995, 1668.47900000005, 1519.65800000005, 1469.67599999998, 1367.13800000004, 1248.52899999998, 1181.23600000003, 1022.71900000004, 1088.20700000005, 959.03600000008, 876.095999999903, 791.183999999892, 703.337000000058, 731.949999999953, 586.86400000006, 526.024999999907, 323.004999999888, 320.448000000091, 340.672999999952, 309.638999999966, 216.601999999955, 102.922999999952, 19.2399999999907, -0.114000000059605, -32.6240000000689, -89.3179999999702, -153.497999999905, -64.2970000000205, -143.695999999996, -259.497999999905, -253.017999999924, -213.948000000091, -397.590000000084, -434.006000000052, -403.475000000093, -297.958000000101, -404.317000000039, -528.898999999976, -506.621000000043, -513.205000000075, -479.351000000024, -596.139999999898, -527.016999999993, -664.681000000099, -680.306000000099, -704.050000000047, -850.486000000034, -757.43200000003, -713.308999999892, } };
+
+ public static boolean isLinearCounting(int precision, double estimate) {
+ return estimate <= thresholdData[precision - 4];
+ }
+
+ public static boolean isBiasCorrection(int numberOfBuckets, double estimate) {
+ return estimate <= 5 * numberOfBuckets;
+ }
+
+ public static double biasCorrection(int precision, double estimate) {
+ double[] estimateVector = rawEstimateData[precision - 4];
+ SortedMap<Double, Integer> estimateDistances = calcDistances(estimate, estimateVector);
+ int[] nearestNeighbors = getNearestNeighbors(estimateDistances);
+ double bias = getBias(precision, nearestNeighbors);
+ return estimate - bias;
+ }
+
+ public static SortedMap<Double, Integer> calcDistances(double estimate, double[] estimateVector) {
+ SortedMap<Double, Integer> distances = new TreeMap<Double, Integer>();
+ int index = 0;
+ for (double anEstimateVector : estimateVector) {
+ distances.put(Math.pow(estimate - anEstimateVector, 2), index++);
+ }
+ return distances;
+ }
+
+ public static double getBias(int precision, int[] nearestNeighbors) {
+ double[] biasVector = biasData[precision - 4];
+ double biasTotal = 0.0d;
+ for (int nearestNeighbor : nearestNeighbors) {
+ biasTotal += biasVector[nearestNeighbor];
+ }
+ return biasTotal / (nearestNeighbors.length);
+ }
+
+ public static int[] getNearestNeighbors(SortedMap<Double, Integer> distanceMap) {
+ int[] nearest = new int[6];
+ int i = 0;
+ for (Integer index : distanceMap.values()) {
+ nearest[i++] = index;
+ if (i >= 6) {
+ break;
+ }
+ }
+ return nearest;
+ }
+}
diff --git a/common/src/main/java/org/apache/kylin/common/lock/JobLock.java b/common/src/main/java/org/apache/kylin/common/lock/JobLock.java
new file mode 100644
index 0000000000..650a3f8169
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/lock/JobLock.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.lock;
+
+/**
+ */
+public interface JobLock {
+ boolean lock();
+
+ void unlock();
+}
diff --git a/common/src/main/java/org/apache/kylin/common/lock/MockJobLock.java b/common/src/main/java/org/apache/kylin/common/lock/MockJobLock.java
new file mode 100644
index 0000000000..52b34a22c0
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/lock/MockJobLock.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.lock;
+
+/**
+ */
+public class MockJobLock implements JobLock {
+ @Override
+ public boolean lock() {
+ return true;
+ }
+
+ @Override
+ public void unlock() {
+ return;
+ }
+}
diff --git a/common/src/main/java/org/apache/kylin/common/lock/ZookeeperJobLock.java b/common/src/main/java/org/apache/kylin/common/lock/ZookeeperJobLock.java
new file mode 100644
index 0000000000..2bed5c6ae1
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/lock/ZookeeperJobLock.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.lock;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.framework.imps.CuratorFrameworkState;
+import org.apache.curator.framework.recipes.locks.InterProcessMutex;
+import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.HadoopUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ */
+public class ZookeeperJobLock implements JobLock {
+ private Logger logger = LoggerFactory.getLogger(ZookeeperJobLock.class);
+
+ private static final String ZOOKEEPER_LOCK_PATH = "/kylin/job_engine/lock";
+
+ private String scheduleID;
+ private InterProcessMutex sharedLock;
+ private CuratorFramework zkClient;
+
+ @Override
+ public boolean lock() {
+ this.scheduleID = schedulerId();
+ String ZKConnectString = getZKConnectString();
+ if (StringUtils.isEmpty(ZKConnectString)) {
+ throw new IllegalArgumentException("ZOOKEEPER_QUORUM is empty!");
+ }
+
+ RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
+ this.zkClient = CuratorFrameworkFactory.newClient(ZKConnectString, retryPolicy);
+ this.zkClient.start();
+ this.sharedLock = new InterProcessMutex(zkClient, this.scheduleID);
+ boolean hasLock = false;
+ try {
+ hasLock = sharedLock.acquire(3, TimeUnit.SECONDS);
+ } catch (Exception e) {
+ logger.warn("error acquire lock", e);
+ }
+ if (!hasLock) {
+ logger.warn("fail to acquire lock, scheduler has not been started");
+ zkClient.close();
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public void unlock() {
+ releaseLock();
+ }
+
+ private String getZKConnectString() {
+ Configuration conf = HadoopUtil.getCurrentHBaseConfiguration();
+ return conf.get(HConstants.ZOOKEEPER_QUORUM) + ":" + conf.get(HConstants.ZOOKEEPER_CLIENT_PORT);
+ }
+
+ private void releaseLock() {
+ try {
+ if (zkClient.getState().equals(CuratorFrameworkState.STARTED)) {
+ // client.setData().forPath(ZOOKEEPER_LOCK_PATH, null);
+ if (zkClient.checkExists().forPath(scheduleID) != null) {
+ zkClient.delete().guaranteed().deletingChildrenIfNeeded().forPath(scheduleID);
+ }
+ }
+ } catch (Exception e) {
+ logger.error("error release lock:" + scheduleID);
+ throw new RuntimeException(e);
+ }
+ }
+
+ private String schedulerId() {
+ return ZOOKEEPER_LOCK_PATH + "/" + KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
+ }
+}
diff --git a/common/src/main/java/org/apache/kylin/common/mr/KylinMapper.java b/common/src/main/java/org/apache/kylin/common/mr/KylinMapper.java
new file mode 100644
index 0000000000..5508583014
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/mr/KylinMapper.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.mr;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.kylin.common.util.HadoopUtil;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 1/19/15.
+ */
+public class KylinMapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT> extends Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT> {
+ protected void publishConfiguration(Configuration conf) {
+ HadoopUtil.setCurrentConfiguration(conf);
+ }
+}
diff --git a/common/src/main/java/org/apache/kylin/common/mr/KylinReducer.java b/common/src/main/java/org/apache/kylin/common/mr/KylinReducer.java
new file mode 100644
index 0000000000..df28807091
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/mr/KylinReducer.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.mr;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.kylin.common.util.HadoopUtil;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 1/19/15.
+ */
+public class KylinReducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT> extends Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT> {
+ protected void publishConfiguration(Configuration conf) {
+ HadoopUtil.setCurrentConfiguration(conf);
+ }
+}
diff --git a/common/src/main/java/org/apache/kylin/common/persistence/AclEntity.java b/common/src/main/java/org/apache/kylin/common/persistence/AclEntity.java
new file mode 100644
index 0000000000..c443b71787
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/persistence/AclEntity.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.persistence;
+
+/**
+ * @author xduo
+ *
+ */
+public interface AclEntity {
+
+ public String getId();
+
+}
diff --git a/common/src/main/java/org/apache/kylin/common/persistence/ElasticSearchClient.java b/common/src/main/java/org/apache/kylin/common/persistence/ElasticSearchClient.java
new file mode 100644
index 0000000000..6ec504bd21
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/persistence/ElasticSearchClient.java
@@ -0,0 +1,71 @@
+package org.apache.kylin.common.persistence;
+
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Created by liuze on 2016/1/8 0008.
+ */
+public class ElasticSearchClient {
+
+
+
+ private static final Logger logger = LoggerFactory.getLogger(ElasticSearchClient.class);
+
+ private static final Map<String, Client> ConnPoolES = new ConcurrentHashMap<String, Client>();
+
+ static {
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ for (Client client : ConnPoolES.values()) {
+ client.close();
+
+ }
+ }
+ });
+ }
+
+ public static Client get(String esClusterUrl) {
+
+ Client client = ConnPoolES.get(esClusterUrl);
+ try {
+ if (client == null) {
+ Settings settings = ImmutableSettings.settingsBuilder().put("client.transport.sniff", true).put("cluster.name", getClusterName(esClusterUrl)).build();
+ client = new TransportClient(settings).addTransportAddress(new InetSocketTransportAddress(getClusterHost(esClusterUrl), getClusterPort(esClusterUrl)));
+ ConnPoolES.put(esClusterUrl, client);
+ }
+ } catch (Throwable t) {
+ throw new StorageException("Error when open client " + esClusterUrl, t);
+ }
+
+ return client;
+ }
+
+
+ private static String getClusterName(String esClusterUrl){
+ return esClusterUrl.split("@")[1];
+
+ }
+
+ private static String getClusterHost(String esClusterUrl){
+ return esClusterUrl.split("@")[0].split(":")[0];
+
+ }
+
+ private static int getClusterPort(String esClusterUrl){
+ return Integer.parseInt(esClusterUrl.split("@")[0].split(":")[1]);
+
+ }
+
+
+}
diff --git a/common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java b/common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
new file mode 100644
index 0000000000..646cd801d1
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.persistence;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.common.KylinConfig;
+
+import com.google.common.collect.Lists;
+
+public class FileResourceStore extends ResourceStore {
+
+ File root;
+
+ public FileResourceStore(KylinConfig kylinConfig) {
+ super(kylinConfig);
+ root = new File(kylinConfig.getMetadataUrl()).getAbsoluteFile();
+ if (root.exists() == false)
+ throw new IllegalArgumentException("File not exist by '" + kylinConfig.getMetadataUrl() + "': " + root.getAbsolutePath());
+ }
+
+ @Override
+ protected ArrayList<String> listResourcesImpl(String resPath) throws IOException {
+ String[] names = file(resPath).list();
+ if (names == null) // not a directory
+ return null;
+
+ ArrayList<String> r = new ArrayList<String>(names.length);
+ String prefix = resPath.endsWith("/") ? resPath : resPath + "/";
+ for (String n : names) {
+ r.add(prefix + n);
+ }
+ return r;
+ }
+
+ @Override
+ protected boolean existsImpl(String resPath) throws IOException {
+ File f = file(resPath);
+ return f.exists() && f.isFile(); // directory is not considered a
+ // resource
+ }
+
+ @Override
+ protected List<RawResource> getAllResources(String rangeStart, String rangeEnd) throws IOException {
+ List<RawResource> result = Lists.newArrayList();
+ try {
+ String commonPrefix = StringUtils.getCommonPrefix(rangeEnd, rangeStart);
+ commonPrefix = commonPrefix.substring(0, commonPrefix.lastIndexOf("/") + 1);
+ final ArrayList<String> resources = listResourcesImpl(commonPrefix);
+ for (String resource : resources) {
+ if (resource.compareTo(rangeStart) >= 0 && resource.compareTo(rangeEnd) <= 0) {
+ if (existsImpl(resource)) {
+ result.add(getResourceImpl(resource));
+ }
+ }
+ }
+ return result;
+ } catch (IOException ex) {
+ for (RawResource rawResource : result) {
+ IOUtils.closeQuietly(rawResource.inputStream);
+ }
+ throw ex;
+ } catch (Exception ex) {
+ throw new UnsupportedOperationException(ex);
+ }
+ }
+
+ @Override
+ protected RawResource getResourceImpl(String resPath) throws IOException {
+ File f = file(resPath);
+ if (f.exists() && f.isFile())
+ return new RawResource(new FileInputStream(f), f.lastModified());
+ else
+ return null;
+ }
+
+ @Override
+ protected long getResourceTimestampImpl(String resPath) throws IOException {
+ File f = file(resPath);
+ if (f.exists() && f.isFile())
+ return f.lastModified();
+ else
+ return 0;
+ }
+
+ @Override
+ protected void putResourceImpl(String resPath, InputStream content, long ts) throws IOException {
+ File f = file(resPath);
+ f.getParentFile().mkdirs();
+ FileOutputStream out = new FileOutputStream(f);
+ try {
+ IOUtils.copy(content, out);
+ } finally {
+ IOUtils.closeQuietly(out);
+ }
+
+ f.setLastModified(ts);
+ }
+
+ @Override
+ protected long checkAndPutResourceImpl(String resPath, byte[] content, long oldTS, long newTS) throws IOException, IllegalStateException {
+ File f = file(resPath);
+ if ((f.exists() && f.lastModified() != oldTS) || (f.exists() == false && oldTS != 0))
+ throw new IllegalStateException("Overwriting conflict " + resPath + ", expect old TS " + oldTS + ", but found " + f.lastModified());
+
+ putResourceImpl(resPath, new ByteArrayInputStream(content), newTS);
+
+ // some FS lose precision on given time stamp
+ return f.lastModified();
+ }
+
+ @Override
+ protected void deleteResourceImpl(String resPath) throws IOException {
+ File f = file(resPath);
+ f.delete();
+ }
+
+ @Override
+ protected String getReadableResourcePathImpl(String resPath) {
+ return file(resPath).toString();
+ }
+
+ private File file(String resPath) {
+ if (resPath.equals("/"))
+ return root;
+ else
+ return new File(root, resPath);
+ }
+
+}
diff --git a/common/src/main/java/org/apache/kylin/common/persistence/HBaseConnection.java b/common/src/main/java/org/apache/kylin/common/persistence/HBaseConnection.java
new file mode 100644
index 0000000000..9c86376fe4
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/persistence/HBaseConnection.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.persistence;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.kylin.common.util.HadoopUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author yangli9
+ *
+ */
+public class HBaseConnection {
+
+ private static final Logger logger = LoggerFactory.getLogger(HBaseConnection.class);
+
+ private static final Map<String, HConnection> ConnPool = new ConcurrentHashMap<String, HConnection>();
+
+ static {
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ for (HConnection conn : ConnPool.values()) {
+ try {
+ conn.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ });
+ }
+
+ public static void clearCache() {
+ ConnPool.clear();
+ }
+
+ public static HConnection get(String url) {
+ // find configuration
+ Configuration conf = HadoopUtil.getCurrentHBaseConfiguration();
+
+ HConnection connection = ConnPool.get(url);
+ try {
+ // I don't use DCL since recreate a connection is not a big issue.
+ if (connection == null) {
+ connection = HConnectionManager.createConnection(conf);
+ ConnPool.put(url, connection);
+ }
+ } catch (Throwable t) {
+ throw new StorageException("Error when open connection " + url, t);
+ }
+
+ return connection;
+ }
+
+ public static void createHTableIfNeeded(String hbaseUrl, String tableName, String... families) throws IOException {
+ createHTableIfNeeded(HBaseConnection.get(hbaseUrl), tableName, families);
+ }
+
+ public static void createHTableIfNeeded(HConnection conn, String tableName, String... families) throws IOException {
+ HBaseAdmin hbase = new HBaseAdmin(conn);
+
+ try {
+ boolean tableExist = false;
+ try {
+ hbase.getTableDescriptor(TableName.valueOf(tableName));
+ tableExist = true;
+ } catch (TableNotFoundException e) {
+ }
+
+ if (tableExist) {
+ logger.debug("HTable '" + tableName + "' already exists");
+ return;
+ }
+
+ logger.debug("Creating HTable '" + tableName + "'");
+
+ HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
+
+ if (null != families && families.length > 0) {
+ for (String family : families) {
+ HColumnDescriptor fd = new HColumnDescriptor(family);
+ fd.setInMemory(true); // metadata tables are best in memory
+ desc.addFamily(fd);
+ }
+ }
+ hbase.createTable(desc);
+
+ logger.debug("HTable '" + tableName + "' created");
+ } finally {
+ hbase.close();
+ }
+ }
+}
diff --git a/common/src/main/java/org/apache/kylin/common/persistence/HBaseResourceStore.java b/common/src/main/java/org/apache/kylin/common/persistence/HBaseResourceStore.java
new file mode 100644
index 0000000000..ac14e7b55e
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/persistence/HBaseResourceStore.java
@@ -0,0 +1,337 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.persistence;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.common.util.HadoopUtil;
+
+import com.google.common.collect.Lists;
+
+public class HBaseResourceStore extends ResourceStore {
+
+ private static final String DEFAULT_TABLE_NAME = "kylin_metadata";
+ private static final String FAMILY = "f";
+ private static final byte[] B_FAMILY = Bytes.toBytes(FAMILY);
+ private static final String COLUMN = "c";
+ private static final byte[] B_COLUMN = Bytes.toBytes(COLUMN);
+ private static final String COLUMN_TS = "t";
+ private static final byte[] B_COLUMN_TS = Bytes.toBytes(COLUMN_TS);
+
+ private static final Map<String, String> TABLE_SUFFIX_MAP = new LinkedHashMap<String, String>();
+
+ static {
+ TABLE_SUFFIX_MAP.put(CUBE_RESOURCE_ROOT + "/", "_cube");
+ TABLE_SUFFIX_MAP.put(DICT_RESOURCE_ROOT + "/", "_dict");
+ TABLE_SUFFIX_MAP.put("/invertedindex/", "_invertedindex");
+ TABLE_SUFFIX_MAP.put(JOB_PATH_ROOT + "/", "_job");
+ TABLE_SUFFIX_MAP.put(JOB_OUTPUT_PATH_ROOT + "/", "_job_output");
+ TABLE_SUFFIX_MAP.put(PROJECT_RESOURCE_ROOT + "/", "_proj");
+ TABLE_SUFFIX_MAP.put(SNAPSHOT_RESOURCE_ROOT + "/", "_table_snapshot");
+ TABLE_SUFFIX_MAP.put("", ""); // DEFAULT CASE
+ }
+
+ final String tableNameBase;
+ final String hbaseUrl;
+
+ // final Map<String, String> tableNameMap; // path prefix ==> HBase table name
+
+ private HConnection getConnection() throws IOException {
+ return HBaseConnection.get(hbaseUrl);
+ }
+
+ public HBaseResourceStore(KylinConfig kylinConfig) throws IOException {
+ super(kylinConfig);
+
+ String metadataUrl = kylinConfig.getMetadataUrl();
+ // split TABLE@HBASE_URL
+ int cut = metadataUrl.indexOf('@');
+ tableNameBase = cut < 0 ? DEFAULT_TABLE_NAME : metadataUrl.substring(0, cut);
+ hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1);
+ if (!(StringUtils.isEmpty(hbaseUrl) || "hbase".equals(hbaseUrl)))
+ throw new IllegalArgumentException("to use hbase storage, pls set 'kylin.metadata.url=kylin_metadata@hbase' in kylin.properties");
+
+ createHTableIfNeeded(getAllInOneTableName());
+
+ }
+
+ private void createHTableIfNeeded(String tableName) throws IOException {
+ HBaseConnection.createHTableIfNeeded(getConnection(), tableName, FAMILY);
+ }
+
+ private String getAllInOneTableName() {
+ return tableNameBase;
+ }
+
+ @Override
+ protected ArrayList<String> listResourcesImpl(String resPath) throws IOException {
+ assert resPath.startsWith("/");
+ String lookForPrefix = resPath.endsWith("/") ? resPath : resPath + "/";
+ byte[] startRow = Bytes.toBytes(lookForPrefix);
+ byte[] endRow = Bytes.toBytes(lookForPrefix);
+ endRow[endRow.length - 1]++;
+
+ ArrayList<String> result = new ArrayList<String>();
+
+ HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Scan scan = new Scan(startRow, endRow);
+ scan.setFilter(new KeyOnlyFilter());
+ try {
+ ResultScanner scanner = table.getScanner(scan);
+ for (Result r : scanner) {
+ String path = Bytes.toString(r.getRow());
+ assert path.startsWith(lookForPrefix);
+ int cut = path.indexOf('/', lookForPrefix.length());
+ String child = cut < 0 ? path : path.substring(0, cut);
+ if (result.contains(child) == false)
+ result.add(child);
+ }
+ } finally {
+ IOUtils.closeQuietly(table);
+ }
+ // return null to indicate not a folder
+ return result.isEmpty() ? null : result;
+ }
+
+ @Override
+ protected boolean existsImpl(String resPath) throws IOException {
+ Result r = getByScan(resPath, false, false);
+ return r != null;
+ }
+
+ @Override
+ protected List<RawResource> getAllResources(String rangeStart, String rangeEnd) throws IOException {
+ byte[] startRow = Bytes.toBytes(rangeStart);
+ byte[] endRow = plusZero(Bytes.toBytes(rangeEnd));
+
+ Scan scan = new Scan(startRow, endRow);
+ scan.addColumn(B_FAMILY, B_COLUMN_TS);
+ scan.addColumn(B_FAMILY, B_COLUMN);
+
+ HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ List<RawResource> result = Lists.newArrayList();
+ try {
+ ResultScanner scanner = table.getScanner(scan);
+ for (Result r : scanner) {
+ result.add(new RawResource(getInputStream(Bytes.toString(r.getRow()), r), getTimestamp(r)));
+ }
+ } catch (IOException e) {
+ for (RawResource rawResource : result) {
+ IOUtils.closeQuietly(rawResource.inputStream);
+ }
+ throw e;
+ } finally {
+ IOUtils.closeQuietly(table);
+ }
+ return result;
+ }
+
+ private InputStream getInputStream(String resPath, Result r) throws IOException {
+ if (r == null) {
+ return null;
+ }
+ byte[] value = r.getValue(B_FAMILY, B_COLUMN);
+ if (value.length == 0) {
+ Path redirectPath = bigCellHDFSPath(resPath);
+ Configuration hconf = HadoopUtil.getCurrentHBaseConfiguration();
+ FileSystem fileSystem = FileSystem.get(hconf);
+
+ return fileSystem.open(redirectPath);
+ } else {
+ return new ByteArrayInputStream(value);
+ }
+ }
+
+ private long getTimestamp(Result r) {
+ if (r == null || r.getValue(B_FAMILY, B_COLUMN_TS) == null) {
+ return 0;
+ } else {
+ return Bytes.toLong(r.getValue(B_FAMILY, B_COLUMN_TS));
+ }
+ }
+
+ @Override
+ protected RawResource getResourceImpl(String resPath) throws IOException {
+ Result r = getByScan(resPath, true, true);
+ if (r == null)
+ return null;
+ else
+ return new RawResource(getInputStream(resPath, r), getTimestamp(r));
+ }
+
+ @Override
+ protected long getResourceTimestampImpl(String resPath) throws IOException {
+ return getTimestamp(getByScan(resPath, false, true));
+ }
+
+ @Override
+ protected void putResourceImpl(String resPath, InputStream content, long ts) throws IOException {
+ ByteArrayOutputStream bout = new ByteArrayOutputStream();
+ IOUtils.copy(content, bout);
+ bout.close();
+
+ HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ try {
+ byte[] row = Bytes.toBytes(resPath);
+ Put put = buildPut(resPath, ts, row, bout.toByteArray(), table);
+
+ table.put(put);
+ table.flushCommits();
+ } finally {
+ IOUtils.closeQuietly(table);
+ }
+ }
+
+ @Override
+ protected long checkAndPutResourceImpl(String resPath, byte[] content, long oldTS, long newTS) throws IOException, IllegalStateException {
+ HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ try {
+ byte[] row = Bytes.toBytes(resPath);
+ byte[] bOldTS = oldTS == 0 ? null : Bytes.toBytes(oldTS);
+ Put put = buildPut(resPath, newTS, row, content, table);
+
+ boolean ok = table.checkAndPut(row, B_FAMILY, B_COLUMN_TS, bOldTS, put);
+ if (!ok) {
+ long real = getResourceTimestampImpl(resPath);
+ throw new IllegalStateException("Overwriting conflict " + resPath + ", expect old TS " + real + ", but it is " + oldTS);
+ }
+
+ table.flushCommits();
+
+ return newTS;
+ } finally {
+ IOUtils.closeQuietly(table);
+ }
+ }
+
+ @Override
+ protected void deleteResourceImpl(String resPath) throws IOException {
+ HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ try {
+ Delete del = new Delete(Bytes.toBytes(resPath));
+ table.delete(del);
+ table.flushCommits();
+ } finally {
+ IOUtils.closeQuietly(table);
+ }
+ }
+
+ @Override
+ protected String getReadableResourcePathImpl(String resPath) {
+ return getAllInOneTableName() + "(key='" + resPath + "')@" + kylinConfig.getMetadataUrl();
+ }
+
+ private Result getByScan(String path, boolean fetchContent, boolean fetchTimestamp) throws IOException {
+ byte[] startRow = Bytes.toBytes(path);
+ byte[] endRow = plusZero(startRow);
+
+ Scan scan = new Scan(startRow, endRow);
+ if (!fetchContent && !fetchTimestamp) {
+ scan.setFilter(new KeyOnlyFilter());
+ } else {
+ if (fetchContent)
+ scan.addColumn(B_FAMILY, B_COLUMN);
+ if (fetchTimestamp)
+ scan.addColumn(B_FAMILY, B_COLUMN_TS);
+ }
+
+ HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ try {
+ ResultScanner scanner = table.getScanner(scan);
+ Result result = null;
+ for (Result r : scanner) {
+ result = r;
+ }
+ return result == null || result.isEmpty() ? null : result;
+ } finally {
+ IOUtils.closeQuietly(table);
+ }
+ }
+
+ private byte[] plusZero(byte[] startRow) {
+ byte[] endRow = Arrays.copyOf(startRow, startRow.length + 1);
+ endRow[endRow.length - 1] = 0;
+ return endRow;
+ }
+
+ private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, HTableInterface table) throws IOException {
+ Path redirectPath = bigCellHDFSPath(resPath);
+ Configuration hconf = HadoopUtil.getCurrentHBaseConfiguration();
+ FileSystem fileSystem = FileSystem.get(hconf);
+
+ if (fileSystem.exists(redirectPath)) {
+ fileSystem.delete(redirectPath, true);
+ }
+
+ FSDataOutputStream out = fileSystem.create(redirectPath);
+
+ try {
+ out.write(largeColumn);
+ } finally {
+ IOUtils.closeQuietly(out);
+ }
+
+ return redirectPath;
+ }
+
+ public Path bigCellHDFSPath(String resPath) {
+ String hdfsWorkingDirectory = this.kylinConfig.getHdfsWorkingDirectory();
+ Path redirectPath = new Path(hdfsWorkingDirectory, "resources" + resPath);
+ return redirectPath;
+ }
+
+ private Put buildPut(String resPath, long ts, byte[] row, byte[] content, HTableInterface table) throws IOException {
+ int kvSizeLimit = this.kylinConfig.getHBaseKeyValueSize();
+ if (content.length > kvSizeLimit) {
+ writeLargeCellToHdfs(resPath, content, table);
+ content = BytesUtil.EMPTY_BYTE_ARRAY;
+ }
+
+ Put put = new Put(row);
+ put.add(B_FAMILY, B_COLUMN, content);
+ put.add(B_FAMILY, B_COLUMN_TS, Bytes.toBytes(ts));
+
+ return put;
+ }
+}
diff --git a/common/src/main/java/org/apache/kylin/common/persistence/JsonSerializer.java b/common/src/main/java/org/apache/kylin/common/persistence/JsonSerializer.java
new file mode 100644
index 0000000000..a23b933d3d
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/persistence/JsonSerializer.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.persistence;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.apache.kylin.common.util.JsonUtil;
+
+/**
+ * @author yangli9
+ */
+public class JsonSerializer<T extends RootPersistentEntity> implements Serializer<T> {
+
+ Class<T> clz;
+
+ public JsonSerializer(Class<T> clz) {
+ this.clz = clz;
+ }
+
+ @Override
+ public T deserialize(DataInputStream in) throws IOException {
+ return JsonUtil.readValue(in, clz);
+ }
+
+ @Override
+ public void serialize(T obj, DataOutputStream out) throws IOException {
+ JsonUtil.writeValueIndent(out, obj);
+ }
+}
diff --git a/common/src/main/java/org/apache/kylin/common/persistence/RawResource.java b/common/src/main/java/org/apache/kylin/common/persistence/RawResource.java
new file mode 100644
index 0000000000..4f52553a92
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/persistence/RawResource.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.persistence;
+
+import java.io.InputStream;
+
+/**
+ */
+public class RawResource {
+
+ public final InputStream inputStream;
+ public final long timestamp;
+
+ public RawResource(InputStream inputStream, long timestamp) {
+ this.inputStream = inputStream;
+ this.timestamp = timestamp;
+ }
+}
diff --git a/common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java b/common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
new file mode 100644
index 0000000000..505c72a7cd
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -0,0 +1,290 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.persistence;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+
+abstract public class ResourceStore {
+
+ private static final Logger logger = LoggerFactory.getLogger(ResourceStore.class);
+
+ public static final String CUBE_RESOURCE_ROOT = "/cube";
+ public static final String II_RESOURCE_ROOT = "/invertedindex";
+ public static final String CUBE_DESC_RESOURCE_ROOT = "/cube_desc";
+ public static final String II_DESC_RESOURCE_ROOT = "/invertedindex_desc";
+ public static final String DATA_MODEL_DESC_RESOURCE_ROOT = "/model_desc";
+ public static final String DICT_RESOURCE_ROOT = "/dict";
+ public static final String JOB_PATH_ROOT = "/job";
+ public static final String JOB_OUTPUT_PATH_ROOT = "/job_output";
+ public static final String PROJECT_RESOURCE_ROOT = "/project";
+ public static final String SNAPSHOT_RESOURCE_ROOT = "/table_snapshot";
+ public static final String TABLE_EXD_RESOURCE_ROOT = "/table_exd";
+ public static final String TABLE_RESOURCE_ROOT = "/table";
+ public static final String HYBRID_RESOURCE_ROOT = "/hybrid";
+ public static final String EXECUTE_PATH_ROOT = "/execute";
+ public static final String EXECUTE_OUTPUT_ROOT = "/execute_output";
+
+
+ private static ConcurrentHashMap<KylinConfig, ResourceStore> CACHE = new ConcurrentHashMap<KylinConfig, ResourceStore>();
+
+ public static final ArrayList<Class<? extends ResourceStore>> knownImpl = new ArrayList<Class<? extends ResourceStore>>();
+
+ static {
+ knownImpl.add(FileResourceStore.class);
+ knownImpl.add(HBaseResourceStore.class);
+ }
+
+ public static ResourceStore getStore(KylinConfig kylinConfig) {
+ ResourceStore r = CACHE.get(kylinConfig);
+ List<Throwable> es = new ArrayList<Throwable>();
+ if (r == null) {
+ logger.info("Using metadata url " + kylinConfig.getMetadataUrl() + " for resource store");
+ for (Class<? extends ResourceStore> cls : knownImpl) {
+
+ try {
+ r = cls.getConstructor(KylinConfig.class).newInstance(kylinConfig);
+ } catch (Exception e) {
+ es.add(e);
+ } catch (NoClassDefFoundError er) {
+ // may throw NoClassDefFoundError
+ es.add(er);
+ }
+ if (r != null) {
+ break;
+ }
+ }
+ if (r == null) {
+ for (Throwable exceptionOrError : es) {
+ logger.error("Create new store instance failed ", exceptionOrError);
+ }
+ throw new IllegalArgumentException("Failed to find metadata store by url: " + kylinConfig.getMetadataUrl());
+ }
+
+ CACHE.put(kylinConfig, r);
+ }
+ return r;
+ }
+
+ // ============================================================================
+
+ KylinConfig kylinConfig;
+
+ ResourceStore(KylinConfig kylinConfig) {
+ this.kylinConfig = kylinConfig;
+ }
+
+ /**
+ * return a list of child resources & folders under given path, return null
+ * if given path is not a folder
+ */
+ final public ArrayList<String> listResources(String resPath) throws IOException {
+ resPath = norm(resPath);
+ return listResourcesImpl(resPath);
+ }
+
+ abstract protected ArrayList<String> listResourcesImpl(String resPath) throws IOException;
+
+ /**
+ * return true if a resource exists, return false in case of folder or
+ * non-exist
+ */
+ final public boolean exists(String resPath) throws IOException {
+ return existsImpl(norm(resPath));
+ }
+
+ abstract protected boolean existsImpl(String resPath) throws IOException;
+
+ /**
+ * read a resource, return null in case of not found
+ */
+ final public <T extends RootPersistentEntity> T getResource(String resPath, Class<T> clz, Serializer<T> serializer) throws IOException {
+ resPath = norm(resPath);
+ RawResource res = getResourceImpl(resPath);
+ if (res == null)
+ return null;
+
+ DataInputStream din = new DataInputStream(res.inputStream);
+ try {
+ T r = serializer.deserialize(din);
+ r.setLastModified(res.timestamp);
+ return r;
+ } finally {
+ IOUtils.closeQuietly(din);
+ IOUtils.closeQuietly(res.inputStream);
+ }
+ }
+
+ final public RawResource getResource(String resPath) throws IOException {
+ return getResourceImpl(norm(resPath));
+ }
+
+ final public long getResourceTimestamp(String resPath) throws IOException {
+ return getResourceTimestampImpl(norm(resPath));
+ }
+
+ final public <T extends RootPersistentEntity> List<T> getAllResources(String rangeStart, String rangeEnd, Class<T> clazz, Serializer<T> serializer) throws IOException {
+ final List<RawResource> allResources = getAllResources(rangeStart, rangeEnd);
+ if (allResources.isEmpty()) {
+ return Collections.emptyList();
+ }
+ List<T> result = Lists.newArrayList();
+ try {
+ for (RawResource rawResource : allResources) {
+ final T element = serializer.deserialize(new DataInputStream(rawResource.inputStream));
+ element.setLastModified(rawResource.timestamp);
+ result.add(element);
+ }
+ return result;
+ } finally {
+ for (RawResource rawResource : allResources) {
+ IOUtils.closeQuietly(rawResource.inputStream);
+ }
+ }
+ }
+
+ abstract protected List<RawResource> getAllResources(String rangeStart, String rangeEnd) throws IOException;
+
+ /** returns null if not exists */
+ abstract protected RawResource getResourceImpl(String resPath) throws IOException;
+
+ /** returns 0 if not exists */
+ abstract protected long getResourceTimestampImpl(String resPath) throws IOException;
+
+ /**
+ * overwrite a resource without write conflict check
+ */
+ final public void putResource(String resPath, InputStream content, long ts) throws IOException {
+ resPath = norm(resPath);
+ logger.debug("Saving resource " + resPath + " (Store " + kylinConfig.getMetadataUrl() + ")");
+ putResourceImpl(resPath, content, ts);
+ }
+
+ abstract protected void putResourceImpl(String resPath, InputStream content, long ts) throws IOException;
+
+ /**
+ * check & set, overwrite a resource
+ */
+ final public <T extends RootPersistentEntity> long putResource(String resPath, T obj, Serializer<T> serializer) throws IOException {
+ resPath = norm(resPath);
+ logger.debug("Saving resource " + resPath + " (Store " + kylinConfig.getMetadataUrl() + ")");
+
+ long oldTS = obj.getLastModified();
+ long newTS = System.currentTimeMillis();
+ obj.setLastModified(newTS);
+
+ try {
+ ByteArrayOutputStream buf = new ByteArrayOutputStream();
+ DataOutputStream dout = new DataOutputStream(buf);
+ serializer.serialize(obj, dout);
+ dout.close();
+ buf.close();
+
+ newTS = checkAndPutResourceImpl(resPath, buf.toByteArray(), oldTS, newTS);
+ obj.setLastModified(newTS); // update again the confirmed TS
+ return newTS;
+ } catch (IOException e) {
+ obj.setLastModified(oldTS); // roll back TS when write fail
+ throw e;
+ } catch (RuntimeException e) {
+ obj.setLastModified(oldTS); // roll back TS when write fail
+ throw e;
+ }
+ }
+
+ /**
+ * checks old timestamp when overwriting existing
+ */
+ abstract protected long checkAndPutResourceImpl(String resPath, byte[] content, long oldTS, long newTS) throws IOException, IllegalStateException;
+
+ /**
+ * delete a resource, does nothing on a folder
+ */
+ final public void deleteResource(String resPath) throws IOException {
+ logger.debug("Deleting resource " + resPath + " (Store " + kylinConfig.getMetadataUrl() + ")");
+ deleteResourceImpl(norm(resPath));
+ }
+
+ abstract protected void deleteResourceImpl(String resPath) throws IOException;
+
+ /**
+ * get a readable string of a resource path
+ */
+ final public String getReadableResourcePath(String resPath) {
+ return getReadableResourcePathImpl(norm(resPath));
+ }
+
+ abstract protected String getReadableResourcePathImpl(String resPath);
+
+ private String norm(String resPath) {
+ resPath = resPath.trim();
+ while (resPath.startsWith("//"))
+ resPath = resPath.substring(1);
+ while (resPath.endsWith("/"))
+ resPath = resPath.substring(0, resPath.length() - 1);
+ if (resPath.startsWith("/") == false)
+ resPath = "/" + resPath;
+ return resPath;
+ }
+
+ // ============================================================================
+
+ public static interface Visitor {
+ void visit(String path) throws IOException;
+ }
+
+ public void scanRecursively(String path, Visitor visitor) throws IOException {
+ ArrayList<String> children = listResources(path);
+ if (children != null) {
+ for (String child : children)
+ scanRecursively(child, visitor);
+ return;
+ }
+
+ if (exists(path))
+ visitor.visit(path);
+ }
+
+ public List<String> collectResourceRecursively(String root, final String suffix) throws IOException {
+ final ArrayList<String> collector = Lists.newArrayList();
+ scanRecursively(root, new Visitor() {
+ @Override
+ public void visit(String path) {
+ if (path.endsWith(suffix))
+ collector.add(path);
+ }
+ });
+ return collector;
+ }
+
+}
diff --git a/common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java b/common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
new file mode 100644
index 0000000000..0ebed3d76e
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.persistence;
+
+import java.io.IOException;
+import java.util.ArrayList;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.StringUtil;
+
+public class ResourceTool {
+
+ private static String[] excludes = null;
+
+ public static void main(String[] args) throws IOException {
+ args = StringUtil.filterSystemArgs(args);
+
+ if (args.length == 0) {
+ System.out.println("Usage: MetadataTool reset");
+ System.out.println("Usage: MetadataTool list RESOURCE_PATH");
+ System.out.println("Usage: MetadataTool download LOCAL_DIR");
+ System.out.println("Usage: MetadataTool upload LOCAL_DIR");
+ return;
+ }
+
+ String exclude = System.getProperty("exclude");
+ if (exclude != null) {
+ excludes = exclude.split("\\s*,\\s*");
+ }
+
+ String cmd = args[0];
+ switch (cmd) {
+ case "reset":
+ reset(args.length == 1 ? KylinConfig.getInstanceFromEnv() : KylinConfig.createInstanceFromUri(args[1]));
+ break;
+ case "list":
+ list(KylinConfig.getInstanceFromEnv(), args[1]);
+ break;
+ case "download":
+ copy(KylinConfig.getInstanceFromEnv(), KylinConfig.createInstanceFromUri(args[1]));
+ break;
+ case "upload":
+ copy(KylinConfig.createInstanceFromUri(args[1]), KylinConfig.getInstanceFromEnv());
+ break;
+ case "remove":
+ remove(KylinConfig.getInstanceFromEnv(), args[1]);
+ break;
+ default:
+ System.out.println("Unknown cmd: " + cmd);
+ }
+ }
+
+ public static void list(KylinConfig config, String path) throws IOException {
+ ResourceStore store = ResourceStore.getStore(config);
+ ArrayList<String> result = store.listResources(path);
+ System.out.println("" + result);
+ }
+
+ public static void copy(KylinConfig srcConfig, KylinConfig dstConfig) throws IOException {
+
+ ResourceStore src = ResourceStore.getStore(srcConfig);
+ ResourceStore dst = ResourceStore.getStore(dstConfig);
+ copyR(src, dst, "/");
+ }
+
+ private static void copyR(ResourceStore src, ResourceStore dst, String path) throws IOException {
+ ArrayList<String> children = src.listResources(path);
+
+ // case of resource (not a folder)
+ if (children == null) {
+ if (matchExclude(path) == false) {
+ RawResource res = src.getResource(path);
+ if (res != null) {
+ dst.putResource(path, res.inputStream, res.timestamp);
+ res.inputStream.close();
+ } else {
+ System.out.println("Resource not exist for " + path);
+ }
+ }
+ }
+ // case of folder
+ else {
+ for (String child : children)
+ copyR(src, dst, child);
+ }
+ }
+
+ private static boolean matchExclude(String path) {
+ if (excludes == null)
+ return false;
+ for (String exclude : excludes) {
+ if (path.startsWith(exclude))
+ return true;
+ }
+ return false;
+ }
+
+ public static void reset(KylinConfig config) throws IOException {
+ ResourceStore store = ResourceStore.getStore(config);
+ resetR(store, "/");
+ }
+
+ private static void resetR(ResourceStore store, String path) throws IOException {
+ ArrayList<String> children = store.listResources(path);
+ if (children == null) { // path is a resource (not a folder)
+ if (matchExclude(path) == false) {
+ store.deleteResource(path);
+ }
+ } else {
+ for (String child : children)
+ resetR(store, child);
+ }
+ }
+
+ private static void remove(KylinConfig config, String path) throws IOException {
+ ResourceStore store = ResourceStore.getStore(config);
+ resetR(store, path);
+ }
+}
diff --git a/common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java b/common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
new file mode 100644
index 0000000000..c6f3f55fe7
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
@@ -0,0 +1,124 @@
(This diff was longer than 20,000 lines, and has been truncated...)
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services