You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jb...@apache.org on 2017/03/15 23:54:41 UTC
[1/3] lucene-solr:branch_6x: SOLR-10270: Stop exporting _version_
during GROUP BY aggregations in map_reduce mode
Repository: lucene-solr
Updated Branches:
refs/heads/branch_6x c6a1aa20d -> 1bf015777
SOLR-10270: Stop exporting _version_ during GROUP BY aggregations in map_reduce mode
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e4d71a02
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e4d71a02
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e4d71a02
Branch: refs/heads/branch_6x
Commit: e4d71a0200ec43eeb8b1d1d013a6e8de438bee14
Parents: c6a1aa2
Author: Joel Bernstein <jb...@apache.org>
Authored: Mon Mar 13 12:50:17 2017 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Wed Mar 15 19:39:30 2017 -0400
----------------------------------------------------------------------
.../core/src/java/org/apache/solr/handler/sql/SolrTable.java | 8 --------
1 file changed, 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e4d71a02/solr/core/src/java/org/apache/solr/handler/sql/SolrTable.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/sql/SolrTable.java b/solr/core/src/java/org/apache/solr/handler/sql/SolrTable.java
index 9375bc0..37ed7d9 100644
--- a/solr/core/src/java/org/apache/solr/handler/sql/SolrTable.java
+++ b/solr/core/src/java/org/apache/solr/handler/sql/SolrTable.java
@@ -330,24 +330,16 @@ class SolrTable extends AbstractQueryableTable implements TranslatableTable {
private String getFields(Set<String> fieldSet) {
StringBuilder buf = new StringBuilder();
- boolean appendVersion = true;
for(String field : fieldSet) {
if(buf.length() > 0) {
buf.append(",");
}
- if(field.equals("_version_")) {
- appendVersion = false;
- }
buf.append(field);
}
- if(appendVersion){
- buf.append(",_version_");
- }
-
return buf.toString();
}
[3/3] lucene-solr:branch_6x: SOLR-10085: SQL result set fields should
be ordered by the field list
Posted by jb...@apache.org.
SOLR-10085: SQL result set fields should be ordered by the field list
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/1bf01577
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/1bf01577
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/1bf01577
Branch: refs/heads/branch_6x
Commit: 1bf01577748da82a394f3c64a2af68d5d8ae4ac2
Parents: 802dd64
Author: Joel Bernstein <jb...@apache.org>
Authored: Wed Mar 15 18:31:14 2017 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Wed Mar 15 19:40:26 2017 -0400
----------------------------------------------------------------------
.../org/apache/solr/handler/SQLHandler.java | 37 +++++++++++++++-----
.../org/apache/solr/handler/TestSQLHandler.java | 31 ++++++++++++++++
.../org/apache/solr/client/solrj/io/Tuple.java | 21 +++++++----
3 files changed, 74 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1bf01577/solr/core/src/java/org/apache/solr/handler/SQLHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/SQLHandler.java b/solr/core/src/java/org/apache/solr/handler/SQLHandler.java
index d65ea56..7563fe8 100644
--- a/solr/core/src/java/org/apache/solr/handler/SQLHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/SQLHandler.java
@@ -34,7 +34,9 @@ import org.apache.solr.client.solrj.io.stream.ExceptionStream;
import org.apache.solr.client.solrj.io.stream.JDBCStream;
import org.apache.solr.client.solrj.io.stream.TupleStream;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.sql.CalciteSolrDriver;
@@ -74,6 +76,9 @@ public class SQLHandler extends RequestHandlerBase implements SolrCoreAware, Per
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
ModifiableSolrParams params = new ModifiableSolrParams(req.getParams());
+ params = adjustParams(params);
+ req.setParams(params);
+
String sql = params.get("stmt");
// Set defaults for parameters
params.set("numWorkers", params.getInt("numWorkers", 1));
@@ -139,6 +144,8 @@ public class SQLHandler extends RequestHandlerBase implements SolrCoreAware, Per
private class SqlHandlerStream extends JDBCStream {
private final boolean includeMetadata;
private boolean firstTuple = true;
+ List<String> metadataFields = new ArrayList<>();
+ Map<String, String> metadataAliases = new HashMap<>();
SqlHandlerStream(String connectionUrl, String sqlQuery, StreamComparator definedSort,
Properties connectionProperties, String driverClassName, boolean includeMetadata)
@@ -151,7 +158,7 @@ public class SQLHandler extends RequestHandlerBase implements SolrCoreAware, Per
@Override
public Tuple read() throws IOException {
// Return a metadata tuple as the first tuple and then pass through to the JDBCStream.
- if(includeMetadata && firstTuple) {
+ if(firstTuple) {
try {
Map<String, Object> fields = new HashMap<>();
@@ -159,8 +166,6 @@ public class SQLHandler extends RequestHandlerBase implements SolrCoreAware, Per
ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
- List<String> metadataFields = new ArrayList<>();
- Map<String, String> metadataAliases = new HashMap<>();
for(int i = 1; i <= resultSetMetaData.getColumnCount(); i++) {
String columnName = resultSetMetaData.getColumnName(i);
String columnLabel = resultSetMetaData.getColumnLabel(i);
@@ -168,16 +173,30 @@ public class SQLHandler extends RequestHandlerBase implements SolrCoreAware, Per
metadataAliases.put(columnName, columnLabel);
}
- fields.put("isMetadata", true);
- fields.put("fields", metadataFields);
- fields.put("aliases", metadataAliases);
- return new Tuple(fields);
+ if(includeMetadata) {
+ fields.put("isMetadata", true);
+ fields.put("fields", metadataFields);
+ fields.put("aliases", metadataAliases);
+ return new Tuple(fields);
+ }
} catch (SQLException e) {
throw new IOException(e);
}
- } else {
- return super.read();
}
+
+ Tuple tuple = super.read();
+ if(!tuple.EOF) {
+ tuple.fieldNames = metadataFields;
+ tuple.fieldLabels = metadataAliases;
+ }
+ return tuple;
}
}
+
+ private ModifiableSolrParams adjustParams(SolrParams params) {
+ ModifiableSolrParams adjustedParams = new ModifiableSolrParams();
+ adjustedParams.add(params);
+ adjustedParams.add(CommonParams.OMIT_HEADER, "true");
+ return adjustedParams;
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1bf01577/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java b/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java
index f222cee..cb16f03 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java
@@ -16,21 +16,30 @@
*/
package org.apache.solr.handler;
+import java.io.BufferedReader;
import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.InputStreamResponseParser;
import org.apache.solr.client.solrj.io.Tuple;
import org.apache.solr.client.solrj.io.stream.ExceptionStream;
import org.apache.solr.client.solrj.io.stream.SolrStream;
import org.apache.solr.client.solrj.io.stream.TupleStream;
+import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -161,6 +170,9 @@ public class TestSQLHandler extends AbstractFullDistribZkTestBase {
assert(tuple.getLong("field_i") == 7);
assert(tuple.get("str_s").equals("a"));
+ //Assert field order
+ assertResponseContains(clients.get(0), sParams, "{\"docs\":[{\"id\":8,\"field_i\":60,\"str_s\":\"c\"}");
+
//Test unlimited unsorted result. Should sort on _version_ desc
sParams = mapParams(CommonParams.QT, "/sql", "stmt", "select id, field_i, str_s from collection1 where text='XXXX'");
@@ -2362,4 +2374,23 @@ public class TestSQLHandler extends AbstractFullDistribZkTestBase {
return params;
}
+ public void assertResponseContains(SolrClient server, SolrParams requestParams, String json) throws IOException, SolrServerException {
+ String p = requestParams.get("qt");
+ if(p != null) {
+ ModifiableSolrParams modifiableSolrParams = (ModifiableSolrParams) requestParams;
+ modifiableSolrParams.remove("qt");
+ }
+
+ QueryRequest query = new QueryRequest( requestParams );
+ query.setPath(p);
+ query.setResponseParser(new InputStreamResponseParser("json"));
+ query.setMethod(SolrRequest.METHOD.POST);
+ NamedList<Object> genericResponse = server.request(query);
+ InputStream stream = (InputStream)genericResponse.get("stream");
+ InputStreamReader reader = new InputStreamReader(stream, "UTF-8");
+ BufferedReader bufferedReader = new BufferedReader(reader);
+ String response = bufferedReader.readLine();
+ assertTrue(response.contains(json));
+ }
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1bf01577/solr/solrj/src/java/org/apache/solr/client/solrj/io/Tuple.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/Tuple.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/Tuple.java
index 58d948d..fdf44c9 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/Tuple.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/Tuple.java
@@ -45,6 +45,8 @@ public class Tuple implements Cloneable, MapWriter {
public boolean EXCEPTION;
public Map fields = new HashMap();
+ public List<String> fieldNames;
+ public Map<String, String> fieldLabels;
public Tuple(Map fields) {
if(fields.containsKey("EOF")) {
@@ -198,12 +200,19 @@ public class Tuple implements Cloneable, MapWriter {
@Override
public void writeMap(EntryWriter ew) throws IOException {
- fields.forEach((k, v) -> {
- try {
- ew.put((String)k,v);
- } catch (IOException e) {
- throw new RuntimeException(e);
+ if(fieldNames == null) {
+ fields.forEach((k, v) -> {
+ try {
+ ew.put((String) k, v);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ } else {
+ for(String fieldName : fieldNames) {
+ String label = fieldLabels.get(fieldName);
+ ew.put(label, fields.get(label));
}
- });
+ }
}
}
[2/3] lucene-solr:branch_6x: SOLR-10271: SQL aggregations in
map_reduce mode should use javabin transport
Posted by jb...@apache.org.
SOLR-10271: SQL aggregations in map_reduce mode should use javabin transport
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/802dd648
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/802dd648
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/802dd648
Branch: refs/heads/branch_6x
Commit: 802dd6486bcdbb6ee929d01b6c25698e716a1c44
Parents: e4d71a0
Author: Joel Bernstein <jb...@apache.org>
Authored: Mon Mar 13 17:10:21 2017 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Wed Mar 15 19:39:56 2017 -0400
----------------------------------------------------------------------
solr/core/src/java/org/apache/solr/handler/sql/SolrTable.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/802dd648/solr/core/src/java/org/apache/solr/handler/sql/SolrTable.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/sql/SolrTable.java b/solr/core/src/java/org/apache/solr/handler/sql/SolrTable.java
index 37ed7d9..644ed97 100644
--- a/solr/core/src/java/org/apache/solr/handler/sql/SolrTable.java
+++ b/solr/core/src/java/org/apache/solr/handler/sql/SolrTable.java
@@ -453,6 +453,7 @@ class SolrTable extends AbstractQueryableTable implements TranslatableTable {
params.set(CommonParams.FL, fl);
params.set(CommonParams.Q, query);
+ params.set(CommonParams.WT, CommonParams.JAVABIN);
//Always use the /export handler for Group By Queries because it requires exporting full result sets.
params.set(CommonParams.QT, "/export");
@@ -691,6 +692,7 @@ class SolrTable extends AbstractQueryableTable implements TranslatableTable {
params.set(CommonParams.FL, fl);
params.set(CommonParams.Q, query);
+ params.set(CommonParams.WT, CommonParams.JAVABIN);
//Always use the /export handler for Distinct Queries because it requires exporting full result sets.
params.set(CommonParams.QT, "/export");