You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jd...@apache.org on 2017/05/12 13:45:08 UTC
lucene-solr:master: SOLR-10617: JDBCStream to support additional
types, minor refactoring to separate out CalciteJDBCStream,
upgrade hsqldb for JDBCStream & DIH tests.
Repository: lucene-solr
Updated Branches:
refs/heads/master 904df0eb6 -> e61b5b34b
SOLR-10617: JDBCStream to support additional types, minor refactoring to separate out CalciteJDBCStream, upgrade hsqldb for JDBCStream & DIH tests.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e61b5b34
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e61b5b34
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e61b5b34
Branch: refs/heads/master
Commit: e61b5b34bf14b9addd98eeafdad43b92e6208d5f
Parents: 904df0e
Author: jdyer1 <jd...@apache.org>
Authored: Fri May 12 08:44:16 2017 -0500
Committer: jdyer1 <jd...@apache.org>
Committed: Fri May 12 08:44:16 2017 -0500
----------------------------------------------------------------------
dev-tools/idea/.idea/libraries/HSQLDB.xml | 2 +-
lucene/ivy-versions.properties | 3 +-
solr/CHANGES.txt | 7 +-
solr/contrib/dataimporthandler/ivy.xml | 2 +-
.../TestVariableResolverEndToEnd.java | 8 +-
.../apache/solr/handler/CalciteJDBCStream.java | 75 ++++
.../org/apache/solr/handler/SQLHandler.java | 3 +-
solr/example/example-DIH/ivy.xml | 2 +-
solr/licenses/hsqldb-1.8.0.10.jar.sha1 | 1 -
solr/licenses/hsqldb-2.4.0.jar.sha1 | 1 +
solr/licenses/hsqldb-LICENSE-BSD_LIKE.txt | 9 +-
solr/licenses/hsqldb-NOTICE.txt | 135 ++++----
.../solr/client/solrj/io/stream/JDBCStream.java | 342 ++++++++++++++-----
.../client/solrj/io/stream/JDBCStreamTest.java | 34 +-
14 files changed, 450 insertions(+), 174 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e61b5b34/dev-tools/idea/.idea/libraries/HSQLDB.xml
----------------------------------------------------------------------
diff --git a/dev-tools/idea/.idea/libraries/HSQLDB.xml b/dev-tools/idea/.idea/libraries/HSQLDB.xml
index 4eb35d5..39efcbf 100644
--- a/dev-tools/idea/.idea/libraries/HSQLDB.xml
+++ b/dev-tools/idea/.idea/libraries/HSQLDB.xml
@@ -1,7 +1,7 @@
<component name="libraryTable">
<library name="HSQLDB">
<CLASSES>
- <root url="jar://$PROJECT_DIR$/solr/example/example-DIH/solr/db/lib/hsqldb-1.8.0.10.jar!/" />
+ <root url="jar://$PROJECT_DIR$/solr/example/example-DIH/solr/db/lib/hsqldb-2.4.0.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES />
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e61b5b34/lucene/ivy-versions.properties
----------------------------------------------------------------------
diff --git a/lucene/ivy-versions.properties b/lucene/ivy-versions.properties
index 3a1efbb..7e0e7c7 100644
--- a/lucene/ivy-versions.properties
+++ b/lucene/ivy-versions.properties
@@ -53,7 +53,6 @@ com.sun.jersey.version = 1.9
/commons-logging/commons-logging = 1.1.3
/de.l3s.boilerpipe/boilerpipe = 1.1.0
/dom4j/dom4j = 1.6.1
-/hsqldb/hsqldb = 1.8.0.10
/info.ganglia.gmetric4j/gmetric4j = 1.0.7
io.dropwizard.metrics.version = 3.2.2
@@ -242,6 +241,8 @@ org.gagravarr.vorbis.java.version = 0.8
/org.gagravarr/vorbis-java-core = ${org.gagravarr.vorbis.java.version}
/org.gagravarr/vorbis-java-tika = ${org.gagravarr.vorbis.java.version}
+/org.hsqldb/hsqldb = 2.4.0
+
/org.jsoup/jsoup = 1.8.2
/org.locationtech.spatial4j/spatial4j = 0.6
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e61b5b34/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 5f6f482..10edc52 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -158,9 +158,14 @@ Apache UIMA 2.3.1
Apache ZooKeeper 3.4.10
Jetty 9.3.14.v20161028
+Detailed Change List
+----------------------
-(No Changes)
+Other Changes
+----------------------
+* SOLR-10617: JDBCStream accepts columns of type TIME, DATE & TIMESTAMP as well as CLOBs and decimal
+ numeric types (James Dyer)
================== 6.6.0 ==================
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e61b5b34/solr/contrib/dataimporthandler/ivy.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/ivy.xml b/solr/contrib/dataimporthandler/ivy.xml
index ea138dd..67af77b 100644
--- a/solr/contrib/dataimporthandler/ivy.xml
+++ b/solr/contrib/dataimporthandler/ivy.xml
@@ -23,7 +23,7 @@
<conf name="test" transitive="false"/>
</configurations>
<dependencies>
- <dependency org="hsqldb" name="hsqldb" rev="${/hsqldb/hsqldb}" conf="test"/>
+ <dependency org="org.hsqldb" name="hsqldb" rev="${/org.hsqldb/hsqldb}" conf="test"/>
<dependency org="org.apache.derby" name="derby" rev="${/org.apache.derby/derby}" conf="test"/>
<dependency org="org.mockito" name="mockito-core" rev="${/org.mockito/mockito-core}" conf="test"/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e61b5b34/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java
index 0f53b93..8ee6878 100644
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java
+++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java
@@ -73,14 +73,14 @@ public class TestVariableResolverEndToEnd extends AbstractDIHJdbcTestCase {
"select " +
" 1 as id, " +
" 'SELECT' as SELECT_KEYWORD, " +
- " CURRENT_TIMESTAMP as FIRST_TS " +
+ " {ts '2017-02-18 12:34:56'} as FIRST_TS " +
"from DUAL \" >\n");
sb.append(" <field column=\"SELECT_KEYWORD\" name=\"select_keyword_s\" /> \n");
sb.append(" <entity name=\"SECOND\" processor=\"SqlEntityProcessor\" dataSource=\"hsqldb\" transformer=\"TemplateTransformer\" ");
sb.append(" query=\"" +
"${dataimporter.functions.encodeUrl(FIRST.SELECT_KEYWORD)} " +
" 1 as SORT, " +
- " CURRENT_TIMESTAMP as SECOND_TS, " +
+ " {ts '2017-02-18 12:34:56'} as SECOND_TS, " +
" '${dataimporter.functions.formatDate(FIRST.FIRST_TS, 'yyyy'" + thirdLocaleParam + ")}' as SECOND1_S, " +
" 'PORK' AS MEAT, " +
" 'GRILL' AS METHOD, " +
@@ -91,7 +91,7 @@ public class TestVariableResolverEndToEnd extends AbstractDIHJdbcTestCase {
"UNION " +
"${dataimporter.functions.encodeUrl(FIRST.SELECT_KEYWORD)} " +
" 2 as SORT, " +
- " CURRENT_TIMESTAMP as SECOND_TS, " +
+ " {ts '2017-02-18 12:34:56'} as SECOND_TS, " +
" '${dataimporter.functions.formatDate(FIRST.FIRST_TS, 'yyyy'" + thirdLocaleParam + ")}' as SECOND1_S, " +
" 'FISH' AS MEAT, " +
" 'FRY' AS METHOD, " +
@@ -112,7 +112,7 @@ public class TestVariableResolverEndToEnd extends AbstractDIHJdbcTestCase {
sb.append("</document> \n");
sb.append("</dataConfig> \n");
String config = sb.toString();
- log.debug(config);
+ log.info(config);
return config;
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e61b5b34/solr/core/src/java/org/apache/solr/handler/CalciteJDBCStream.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/CalciteJDBCStream.java b/solr/core/src/java/org/apache/solr/handler/CalciteJDBCStream.java
new file mode 100644
index 0000000..3738f62
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/CalciteJDBCStream.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler;
+
+import java.io.IOException;
+import java.sql.Array;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.util.Properties;
+
+import org.apache.solr.client.solrj.io.comp.StreamComparator;
+import org.apache.solr.client.solrj.io.stream.JDBCStream;
+
+/**
+ * Used with o.a.s.Handler.SQLHandler.
+ *
+ * @lucene.internal
+ */
+public class CalciteJDBCStream extends JDBCStream {
+ private static final long serialVersionUID = 1L;
+
+ public CalciteJDBCStream(String connectionUrl, String sqlQuery, StreamComparator definedSort,
+ Properties connectionProperties, String driverClassName) throws IOException {
+ super(connectionUrl, sqlQuery, definedSort, connectionProperties, driverClassName);
+ }
+
+ @Override
+ protected ResultSetValueSelector determineValueSelector(int columnIdx, ResultSetMetaData metadata)
+ throws SQLException {
+ ResultSetValueSelector valueSelector = super.determineValueSelector(columnIdx, metadata);
+ if (valueSelector == null) {
+ final int columnNumber = columnIdx + 1;
+ final String columnName = metadata.getColumnLabel(columnNumber);
+ final String className = metadata.getColumnClassName(columnNumber);
+ if (Array.class.getName().equals(className)) {
+ valueSelector = new ResultSetValueSelector() {
+ @Override
+ public Object selectValue(ResultSet resultSet) throws SQLException {
+ Object o = resultSet.getObject(columnNumber);
+ if (resultSet.wasNull()) {
+ return null;
+ }
+ if (o instanceof Array) {
+ Array array = (Array) o;
+ return array.getArray();
+ } else {
+ return o;
+ }
+ }
+
+ @Override
+ public String getColumnName() {
+ return columnName;
+ }
+ };
+ }
+ }
+ return valueSelector;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e61b5b34/solr/core/src/java/org/apache/solr/handler/SQLHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/SQLHandler.java b/solr/core/src/java/org/apache/solr/handler/SQLHandler.java
index c80d0d7..67ea1f6 100644
--- a/solr/core/src/java/org/apache/solr/handler/SQLHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/SQLHandler.java
@@ -31,7 +31,6 @@ import org.apache.calcite.config.Lex;
import org.apache.solr.client.solrj.io.Tuple;
import org.apache.solr.client.solrj.io.comp.StreamComparator;
import org.apache.solr.client.solrj.io.stream.ExceptionStream;
-import org.apache.solr.client.solrj.io.stream.JDBCStream;
import org.apache.solr.client.solrj.io.stream.TupleStream;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
@@ -141,7 +140,7 @@ public class SQLHandler extends RequestHandlerBase implements SolrCoreAware, Per
/*
* Only necessary for SolrJ JDBC driver since metadata has to be passed back
*/
- private static class SqlHandlerStream extends JDBCStream {
+ private static class SqlHandlerStream extends CalciteJDBCStream {
private final boolean includeMetadata;
private boolean firstTuple = true;
List<String> metadataFields = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e61b5b34/solr/example/example-DIH/ivy.xml
----------------------------------------------------------------------
diff --git a/solr/example/example-DIH/ivy.xml b/solr/example/example-DIH/ivy.xml
index 4060e37..3f67a79 100644
--- a/solr/example/example-DIH/ivy.xml
+++ b/solr/example/example-DIH/ivy.xml
@@ -22,7 +22,7 @@
<conf name="compile" transitive="false"/>
</configurations>
<dependencies>
- <dependency org="hsqldb" name="hsqldb" rev="${/hsqldb/hsqldb}" conf="compile"/>
+ <dependency org="org.hsqldb" name="hsqldb" rev="${/org.hsqldb/hsqldb}" conf="compile"/>
<dependency org="org.apache.derby" name="derby" rev="${/org.apache.derby/derby}" conf="compile"/>
<exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/>
</dependencies>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e61b5b34/solr/licenses/hsqldb-1.8.0.10.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/hsqldb-1.8.0.10.jar.sha1 b/solr/licenses/hsqldb-1.8.0.10.jar.sha1
deleted file mode 100644
index 8661e69..0000000
--- a/solr/licenses/hsqldb-1.8.0.10.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-7e9978fdb754bce5fcd5161133e7734ecb683036
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e61b5b34/solr/licenses/hsqldb-2.4.0.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/hsqldb-2.4.0.jar.sha1 b/solr/licenses/hsqldb-2.4.0.jar.sha1
new file mode 100644
index 0000000..72520ee
--- /dev/null
+++ b/solr/licenses/hsqldb-2.4.0.jar.sha1
@@ -0,0 +1 @@
+195957160ed990dbc798207c0d577280d9919208
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e61b5b34/solr/licenses/hsqldb-LICENSE-BSD_LIKE.txt
----------------------------------------------------------------------
diff --git a/solr/licenses/hsqldb-LICENSE-BSD_LIKE.txt b/solr/licenses/hsqldb-LICENSE-BSD_LIKE.txt
index 953bfa0..b8ac4f5 100644
--- a/solr/licenses/hsqldb-LICENSE-BSD_LIKE.txt
+++ b/solr/licenses/hsqldb-LICENSE-BSD_LIKE.txt
@@ -1,4 +1,4 @@
-/* Copyright (c) 2001-2005, The HSQL Development Group
+/* Copyright (c) 2001-2017, The HSQL Development Group
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -18,9 +18,9 @@
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL HSQL DEVELOPMENT GROUP, HSQLDB.ORG,
- * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
- * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * ARE DISCLAIMED. IN NO EVENT SHALL HSQL DEVELOPMENT GROUP, HSQLDB.ORG,
+ * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
@@ -28,4 +28,3 @@
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
-
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e61b5b34/solr/licenses/hsqldb-NOTICE.txt
----------------------------------------------------------------------
diff --git a/solr/licenses/hsqldb-NOTICE.txt b/solr/licenses/hsqldb-NOTICE.txt
index 0929e9b..829f8a2 100644
--- a/solr/licenses/hsqldb-NOTICE.txt
+++ b/solr/licenses/hsqldb-NOTICE.txt
@@ -1,68 +1,69 @@
-=========================================================================
-== HSQLDB Notice ==
-=========================================================================
+/*
+ * For work developed by the HSQL Development Group:
+ *
+ * Copyright (c) 2001-2017, The HSQL Development Group
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * Neither the name of the HSQL Development Group nor the names of its
+ * contributors may be used to endorse or promote products derived from this
+ * software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL HSQL DEVELOPMENT GROUP, HSQLDB.ORG,
+ * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *
+ *
+ * For work originally developed by the Hypersonic SQL Group:
+ *
+ * Copyright (c) 1995-2000, The Hypersonic SQL Group.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * Neither the name of the Hypersonic SQL Group nor the names of its
+ * contributors may be used to endorse or promote products derived from this
+ * software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE HYPERSONIC SQL GROUP,
+ * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * This software consists of voluntary contributions made by many individuals
+ * on behalf of the Hypersonic SQL Group.
+ */
-For content, code, and products originally developed by Thomas Mueller and the Hypersonic SQL Group:
-
-Copyright (c) 1995-2000 by the Hypersonic SQL Group.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-Redistributions of source code must retain the above copyright notice, this
-list of conditions and the following disclaimer.
-
-Redistributions in binary form must reproduce the above copyright notice,
-this list of conditions and the following disclaimer in the documentation
-and/or other materials provided with the distribution.
-
-Neither the name of the Hypersonic SQL Group nor the names of its
-contributors may be used to endorse or promote products derived from this
-software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE HYPERSONIC SQL GROUP,
-OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-This software consists of voluntary contributions made by many individuals on behalf of the
-Hypersonic SQL Group.
-
-For work added by the HSQL Development Group (a.k.a. hsqldb_lic.txt):
-
-Copyright (c) 2001-2005, The HSQL Development Group
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-Redistributions of source code must retain the above copyright notice, this
-list of conditions and the following disclaimer.
-
-Redistributions in binary form must reproduce the above copyright notice,
-this list of conditions and the following disclaimer in the documentation
-and/or other materials provided with the distribution.
-
-Neither the name of the HSQL Development Group nor the names of its
-contributors may be used to endorse or promote products derived from this
-software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL HSQL DEVELOPMENT GROUP, HSQLDB.ORG,
-OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e61b5b34/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/JDBCStream.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/JDBCStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/JDBCStream.java
index 172b9ef..99b090f 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/JDBCStream.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/JDBCStream.java
@@ -17,13 +17,19 @@
package org.apache.solr.client.solrj.io.stream;
import java.io.IOException;
-import java.sql.Array;
+import java.math.BigDecimal;
+import java.sql.Clob;
import java.sql.Connection;
+import java.sql.Date;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
+import java.sql.Time;
+import java.sql.Timestamp;
+import java.sql.Types;
+import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -51,16 +57,64 @@ import static org.apache.solr.common.params.CommonParams.SORT;
* Connects to a datasource using a registered JDBC driver and execute a query. The results of
* that query will be returned as tuples. An EOF tuple will indicate that all have been read.
*
- * Supported Datatypes
- * JDBC Type | Tuple Type
- * --------------|---------------
- * String | String
- * Short | Long
- * Integer | Long
- * Long | Long
- * Float | Double
- * Double | Double
- * Boolean | Boolean
+ * Supported Datatypes for most types vary by JDBC driver based on the specific
+ * java type as reported by {@link java.sql.ResultSetMetaData#getColumnClassName(int)}.
+ * The exception are {@link Types#DATE}, {@link Types#TIME} or {@link Types#TIMESTAMP}
+ * which are determined by the JDBC type.
+ *
+ * <table rules="all" frame="box" cellpadding="3" summary="Supported Java Types">
+ * <tr>
+ * <th>Java or JDBC Type</th>
+ * <th>Tuple Type</th>
+ * <th>Notes</th>
+ * </tr>
+ * <tr>
+ * <td>Boolean</td>
+ * <td>Boolean</td>
+ * <td></td>
+ * </tr>
+ * <tr>
+ * <td>String</td>
+ * <td>String</td>
+ * <td></td>
+ * </tr>
+ * <tr>
+ * <td>Short, Integer, Long</td>
+ * <td>Long</td>
+ * <td></td>
+ * </tr>
+ * <tr>
+ * <td>Float, Double</td>
+ * <td>Double</td>
+ * <td></td>
+ * </tr>
+ * <tr>
+ * <td>{@link Clob} and subclasses</td>
+ * <td>String</td>
+ * <td>Clobs up to length 2<sup>31</sup>-1 are supported.</td>
+ * </tr>
+ * <tr>
+ * <td>Other subclasses of {@link Number}</td>
+ * <td>Long, Double</td>
+ * <td>Tuple Type based on {@link BigDecimal#scale()}.</td>
+ * </tr>
+ * <tr>
+ * <td>JDBC {@link Types#DATE}</td>
+ * <td>String</td>
+ * <td>yyyy-MM-dd, calls {@link Date#toString}</td>
+ * </tr>
+ * <tr>
+ * <td>JDBC {@link Types#TIME}</td>
+ * <td>String</td>
+ * <td>hh:mm:ss, calls {@link Time#toString}</td>
+ * </tr>
+ * <tr>
+ * <td>JDBC {@link Types#TIMESTAMP}</td>
+ * <td>String</td>
+ * <td>See {@link DateTimeFormatter#ISO_INSTANT}</td>
+ * </tr>
+ * </table>
+ *
**/
public class JDBCStream extends TupleStream implements Expressible {
@@ -227,91 +281,200 @@ public class JDBCStream extends TupleStream implements Expressible {
}
private ResultSetValueSelector[] constructValueSelectors(ResultSetMetaData metadata) throws SQLException{
- ResultSetValueSelector[] valueSelectors = new ResultSetValueSelector[metadata.getColumnCount()];
+ ResultSetValueSelector[] valueSelectors = new ResultSetValueSelector[metadata.getColumnCount()];
+ for (int columnIdx = 0; columnIdx < metadata.getColumnCount(); ++columnIdx) {
+ ResultSetValueSelector valueSelector = determineValueSelector(columnIdx, metadata);
+ if(valueSelector==null) {
+ int columnNumber = columnIdx + 1;
+ String columnName = metadata.getColumnLabel(columnNumber);
+ String className = metadata.getColumnClassName(columnNumber);
+ String typeName = metadata.getColumnTypeName(columnNumber);
+ throw new SQLException(String.format(Locale.ROOT,
+ "Unable to determine the valueSelector for column '%s' (col #%d) of java class '%s' and type '%s'",
+ columnName, columnNumber, className, typeName));
+ }
+ valueSelectors[columnIdx] = valueSelector;
+ }
+ return valueSelectors;
+ }
+
+ protected ResultSetValueSelector determineValueSelector(int columnIdx, ResultSetMetaData metadata) throws SQLException {
+ final int columnNumber = columnIdx + 1; // cause it starts at 1
+ // Use getColumnLabel instead of getColumnName to make sure fields renamed with AS as picked up properly
+ final String columnName = metadata.getColumnLabel(columnNumber);
+ final int jdbcType = metadata.getColumnType(columnNumber);
+ final String className = metadata.getColumnClassName(columnNumber);
+ ResultSetValueSelector valueSelector = null;
- for(int columnIdx = 0; columnIdx < metadata.getColumnCount(); ++columnIdx){
- final int columnNumber = columnIdx + 1; // cause it starts at 1
- // Use getColumnLabel instead of getColumnName to make sure fields renamed with AS as picked up properly
- final String columnName = metadata.getColumnLabel(columnNumber);
- String className = metadata.getColumnClassName(columnNumber);
- String typeName = metadata.getColumnTypeName(columnNumber);
-
- if(directSupportedTypes.contains(className)){
- valueSelectors[columnIdx] = new ResultSetValueSelector() {
- public Object selectValue(ResultSet resultSet) throws SQLException {
- Object obj = resultSet.getObject(columnNumber);
- if(resultSet.wasNull()){ return null; }
- if(obj instanceof String) {
- String s = (String)obj;
- if(s.indexOf(sep) > -1) {
- s = s.substring(1);
- return s.split(sep);
- }
+ // Directly supported types can be just directly returned - no conversion really necessary
+ if(directSupportedTypes.contains(className)){
+ valueSelector = new ResultSetValueSelector() {
+ @Override
+ public Object selectValue(ResultSet resultSet) throws SQLException {
+ Object obj = resultSet.getObject(columnNumber);
+ if(resultSet.wasNull()){ return null; }
+ if(obj instanceof String) {
+ String s = (String)obj;
+ if(s.indexOf(sep) > -1) {
+ s = s.substring(1);
+ return s.split(sep);
}
-
- return obj;
- }
- public String getColumnName() {
- return columnName;
- }
- };
- } else if(Short.class.getName().equals(className)) {
- valueSelectors[columnIdx] = new ResultSetValueSelector() {
- public Object selectValue(ResultSet resultSet) throws SQLException {
- Short obj = resultSet.getShort(columnNumber);
- if(resultSet.wasNull()){ return null; }
- return obj.longValue();
- }
- public String getColumnName() {
- return columnName;
- }
- };
- } else if(Integer.class.getName().equals(className)) {
- valueSelectors[columnIdx] = new ResultSetValueSelector() {
- public Object selectValue(ResultSet resultSet) throws SQLException {
- Integer obj = resultSet.getInt(columnNumber);
- if(resultSet.wasNull()){ return null; }
- return obj.longValue();
}
- public String getColumnName() {
- return columnName;
- }
- };
- } else if(Float.class.getName().equals(className)) {
- valueSelectors[columnIdx] = new ResultSetValueSelector() {
- public Object selectValue(ResultSet resultSet) throws SQLException {
- Float obj = resultSet.getFloat(columnNumber);
- if(resultSet.wasNull()){ return null; }
- return obj.doubleValue();
- }
- public String getColumnName() {
- return columnName;
- }
- };
- } else if(Array.class.getName().equals(className)) {
- valueSelectors[columnIdx] = new ResultSetValueSelector() {
+
+ return obj;
+ }
+ @Override
+ public String getColumnName() {
+ return columnName;
+ }
+ };
+ }
+ // We're checking the Java class names because there are lots of SQL types across
+ // lots of database drivers that can be mapped to standard Java types. Basically,
+ // this makes it easier and we don't have to worry about esoteric type names in the
+ // JDBC family of types
+ else if(Short.class.getName().equals(className)) {
+ valueSelector = new ResultSetValueSelector() {
+ @Override
+ public Object selectValue(ResultSet resultSet) throws SQLException {
+ Short obj = resultSet.getShort(columnNumber);
+ if(resultSet.wasNull()){ return null; }
+ return obj.longValue();
+ }
+ @Override
+ public String getColumnName() {
+ return columnName;
+ }
+ };
+ } else if(Integer.class.getName().equals(className)) {
+ valueSelector = new ResultSetValueSelector() {
+ @Override
+ public Object selectValue(ResultSet resultSet) throws SQLException {
+ Integer obj = resultSet.getInt(columnNumber);
+ if(resultSet.wasNull()){ return null; }
+ return obj.longValue();
+ }
+ @Override
+ public String getColumnName() {
+ return columnName;
+ }
+ };
+ } else if(Float.class.getName().equals(className)) {
+ valueSelector = new ResultSetValueSelector() {
+ @Override
+ public Object selectValue(ResultSet resultSet) throws SQLException {
+ Float obj = resultSet.getFloat(columnNumber);
+ if(resultSet.wasNull()){ return null; }
+ return obj.doubleValue();
+ }
+ @Override
+ public String getColumnName() {
+ return columnName;
+ }
+ };
+ }
+ // Here we are switching to check against the SQL type because date/times are
+ // notorious for not being consistent. We don't know if the driver is mapping
+ // to a java.time.* type or some old-school type.
+ else if (jdbcType == Types.DATE) {
+ valueSelector = new ResultSetValueSelector() {
+ @Override
+ public Object selectValue(ResultSet resultSet) throws SQLException {
+ Date sqlDate = resultSet.getDate(columnNumber);
+ return resultSet.wasNull() ? null : sqlDate.toString();
+ }
+ @Override
+ public String getColumnName() {
+ return columnName;
+ }
+ };
+ } else if (jdbcType == Types.TIME ) {
+ valueSelector = new ResultSetValueSelector() {
+ @Override
+ public Object selectValue(ResultSet resultSet) throws SQLException {
+ Time sqlTime = resultSet.getTime(columnNumber);
+ return resultSet.wasNull() ? null : sqlTime.toString();
+ }
+ @Override
+ public String getColumnName() {
+ return columnName;
+ }
+ };
+ } else if (jdbcType == Types.TIMESTAMP) {
+ valueSelector = new ResultSetValueSelector() {
+ @Override
+ public Object selectValue(ResultSet resultSet) throws SQLException {
+ Timestamp sqlTimestamp = resultSet.getTimestamp(columnNumber);
+ return resultSet.wasNull() ? null : sqlTimestamp.toInstant().toString();
+ }
+ @Override
+ public String getColumnName() {
+ return columnName;
+ }
+ };
+ }
+ // Now we're going to start seeing if things are assignable from the returned type
+ // to a more general type - this allows us to cover cases where something we weren't
+ // explicitly expecting, but can handle, is being returned.
+ else {
+ Class<?> clazz;
+ try {
+ clazz = Class.forName(className, false, getClass().getClassLoader());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ final int scale = metadata.getScale(columnNumber);
+ if (Number.class.isAssignableFrom(clazz)) {
+ if (scale > 0) {
+ valueSelector = new ResultSetValueSelector() {
+ @Override
+ public Object selectValue(ResultSet resultSet) throws SQLException {
+ BigDecimal bd = resultSet.getBigDecimal(columnNumber);
+ return resultSet.wasNull() ? null : bd.doubleValue();
+ }
+ @Override
+ public String getColumnName() {
+ return columnName;
+ }
+ };
+ } else {
+ valueSelector = new ResultSetValueSelector() {
+ @Override
+ public Object selectValue(ResultSet resultSet) throws SQLException {
+ BigDecimal bd = resultSet.getBigDecimal(columnNumber);
+ return resultSet.wasNull() ? null : bd.longValue();
+ }
+ @Override
+ public String getColumnName() {
+ return columnName;
+ }
+ };
+ }
+ } else if (Clob.class.isAssignableFrom(clazz)) {
+ valueSelector = new ResultSetValueSelector() {
+ @Override
public Object selectValue(ResultSet resultSet) throws SQLException {
- Object o = resultSet.getObject(columnNumber);
- if(resultSet.wasNull()){ return null; }
- if(o instanceof Array) {
- Array array = (Array)o;
- return array.getArray();
- } else {
- return o;
+ Clob c = resultSet.getClob(columnNumber);
+ if (resultSet.wasNull()) {
+ return null;
}
+ long length = c.length();
+ int lengthInt = (int) length;
+ if (length != lengthInt) {
+ throw new SQLException(String.format(Locale.ROOT,
+ "Encountered a clob of length #%l in column '%s' (col #%d). Max supported length is #%i.",
+ length, columnName, columnNumber, Integer.MAX_VALUE));
+ }
+ return c.getSubString(1, lengthInt);
}
+ @Override
public String getColumnName() {
return columnName;
}
};
- } else {
- throw new SQLException(String.format(Locale.ROOT,
- "Unable to determine the valueSelector for column '%s' (col #%d) of java class '%s' and type '%s'",
- columnName, columnNumber, className, typeName));
- }
+ }
}
-
- return valueSelectors;
+ return valueSelector;
}
/**
@@ -432,9 +595,10 @@ public class JDBCStream extends TupleStream implements Expressible {
// it's already in the sqlQuery but there's no way we can reliably determine the sort from the query.
return definedSort;
}
+
+ public interface ResultSetValueSelector {
+ String getColumnName();
+ Object selectValue(ResultSet resultSet) throws SQLException;
+ }
}
-interface ResultSetValueSelector {
- String getColumnName();
- Object selectValue(ResultSet resultSet) throws SQLException;
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e61b5b34/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/JDBCStreamTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/JDBCStreamTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/JDBCStreamTest.java
index 9fff33a..8813f24 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/JDBCStreamTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/JDBCStreamTest.java
@@ -19,6 +19,7 @@ package org.apache.solr.client.solrj.io.stream;
import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
+import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
@@ -96,6 +97,8 @@ public class JDBCStreamTest extends SolrCloudTestCase {
statement.executeUpdate("create table PEOPLE(ID int not null primary key, NAME varchar(50), COUNTRY_CODE char(2), DELETED char(1) default 'N')");
statement.executeUpdate("create table PEOPLE_SPORTS(ID int not null primary key, PERSON_ID int, SPORT_NAME varchar(50), DELETED char(1) default 'N')");
statement.executeUpdate("create table UNSUPPORTED_COLUMNS(ID int not null primary key, UNSP binary)");
+ statement.executeUpdate("create table DUAL(ID int not null primary key)");
+ statement.executeUpdate("insert into DUAL values(1)");
}
@@ -158,8 +161,29 @@ public class JDBCStreamTest extends SolrCloudTestCase {
assertOrderOf(tuples, "CODE", "NP", "NL", "NO", "US");
assertOrderOf(tuples, "COUNTRY_NAME", "Nepal", "Netherlands", "Norway", "United States");
+ // Additional Types
+ String query = "select 1 as ID1, {ts '2017-02-18 12:34:56.789'} as TS1, {t '01:02:03'} as T1, "
+ + "{d '1593-03-14'} as D1, cast(12.34 AS DECIMAL(4,2)) as DEC4_2, "
+ + "cast(1234 AS DECIMAL(4,0)) as DEC4_0, cast('big stuff' as CLOB(100)) as CLOB1 "
+ + "from DUAL order by ID1";
+ stream = new JDBCStream("jdbc:hsqldb:mem:.", query, new FieldComparator("ID1", ComparatorOrder.ASCENDING));
+ tuples = getTuples(stream);
+ assertEquals(1, tuples.size());
+ Tuple t;
+ try (Connection connection = DriverManager.getConnection("jdbc:hsqldb:mem:.");
+ Statement statement = connection.createStatement()) {
+ ResultSet rs = statement.executeQuery(query);
+ rs.next();
+ t = tuples.iterator().next();
+ assertString(t, "CLOB1", rs.getString("CLOB1"));
+ assertString(t, "TS1", rs.getTimestamp("TS1").toInstant().toString());
+ assertString(t, "T1", rs.getTime("T1").toString());
+ assertString(t, "D1", rs.getDate("D1").toString());
+ assertDouble(t, "DEC4_2", rs.getDouble("DEC4_2"));
+ assertLong(t, "DEC4_0", rs.getLong("DEC4_0"));
+ }
}
-
+
@Test
public void testJDBCJoin() throws Exception {
@@ -643,6 +667,14 @@ public class JDBCStreamTest extends SolrCloudTestCase {
return true;
}
+ public boolean assertDouble(Tuple tuple, String fieldName, double d) throws Exception {
+ double dv = (double)tuple.get(fieldName);
+ if(dv != d) {
+ throw new Exception("Doubles not equal:"+d+" : "+dv);
+ }
+ return true;
+ }
+
public boolean assertString(Tuple tuple, String fieldName, String expected) throws Exception {
String actual = (String)tuple.get(fieldName);