You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2018/08/09 19:27:15 UTC
[13/17] lucene-solr:jira/solr-12470: SOLR-12616: Optimize Export
writer upto 4 sort fields to get better performance. This was removed in
SOLR-11598 but brought back in the same version
SOLR-12616: Optimize Export writer upto 4 sort fields to get better performance. This was removed in SOLR-11598 but brought back in the same version
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e9f3a3ce
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e9f3a3ce
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e9f3a3ce
Branch: refs/heads/jira/solr-12470
Commit: e9f3a3ce1d482bd90ba8aca6e8cb7fe6c86756eb
Parents: 7c4584b
Author: Varun Thacker <va...@apache.org>
Authored: Wed Aug 8 11:10:21 2018 -0700
Committer: Varun Thacker <va...@apache.org>
Committed: Wed Aug 8 13:28:00 2018 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +
.../solr/handler/export/DoubleValueSortDoc.java | 93 ++++++++++++++
.../solr/handler/export/ExportWriter.java | 13 +-
.../solr/handler/export/QuadValueSortDoc.java | 126 +++++++++++++++++++
.../solr/handler/export/SingleValueSortDoc.java | 82 ++++++++++++
.../org/apache/solr/handler/export/SortDoc.java | 7 +-
.../solr/handler/export/TripleValueSortDoc.java | 110 ++++++++++++++++
7 files changed, 431 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e9f3a3ce/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 9c80773..9e6ece5 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -253,6 +253,9 @@ Optimizations
* SOLR-11881: Retry update requests sent by leaders to it's followers (Varun Thacker, Mark Miller, Tomás Fernández Löbbe)
+* SOLR-12616: Optimize Export writer upto 4 sort fields to get better performance.
+ This was removed in SOLR-11598 but brought back in the same version (Amrit Sarkar, Varun Thacker)
+
Other Changes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e9f3a3ce/solr/core/src/java/org/apache/solr/handler/export/DoubleValueSortDoc.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/DoubleValueSortDoc.java b/solr/core/src/java/org/apache/solr/handler/export/DoubleValueSortDoc.java
new file mode 100644
index 0000000..30fff33
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/DoubleValueSortDoc.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.LeafReaderContext;
+
+class DoubleValueSortDoc extends SingleValueSortDoc {
+
+ protected SortValue value2;
+
+ public void setNextReader(LeafReaderContext context) throws IOException {
+ this.ord = context.ord;
+ this.docBase = context.docBase;
+ value1.setNextReader(context);
+ value2.setNextReader(context);
+ }
+
+ public void reset() {
+ this.docId = -1;
+ this.docBase = -1;
+ value1.reset();
+ value2.reset();
+ }
+
+ public void setValues(int docId) throws IOException {
+ this.docId = docId;
+ value1.setCurrentValue(docId);
+ value2.setCurrentValue(docId);
+ }
+
+ public void setValues(SortDoc sortDoc) {
+ this.docId = sortDoc.docId;
+ this.ord = sortDoc.ord;
+ this.docBase = sortDoc.docBase;
+ value1.setCurrentValue(((DoubleValueSortDoc)sortDoc).value1);
+ value2.setCurrentValue(((DoubleValueSortDoc)sortDoc).value2);
+ }
+
+ public DoubleValueSortDoc(SortValue value1, SortValue value2) {
+ super(value1);
+ this.value2 = value2;
+ }
+
+ public SortDoc copy() {
+ return new DoubleValueSortDoc(value1.copy(), value2.copy());
+ }
+
+ public boolean lessThan(Object o) {
+ DoubleValueSortDoc sd = (DoubleValueSortDoc)o;
+ int comp = value1.compareTo(sd.value1);
+ if(comp == -1) {
+ return true;
+ } else if (comp == 1) {
+ return false;
+ } else {
+ comp = value2.compareTo(sd.value2);
+ if(comp == -1) {
+ return true;
+ } else if (comp == 1) {
+ return false;
+ } else {
+ return docId+docBase > sd.docId+sd.docBase;
+ }
+ }
+ }
+
+ public int compareTo(Object o) {
+ DoubleValueSortDoc sd = (DoubleValueSortDoc)o;
+ int comp = value1.compareTo(sd.value1);
+ if (comp == 0) {
+ return value2.compareTo(sd.value2);
+ } else {
+ return comp;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e9f3a3ce/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java b/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java
index 57ce600..9cacfcc 100644
--- a/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java
+++ b/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java
@@ -431,7 +431,18 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {
throw new IOException("Sort fields must be one of the following types: int,float,long,double,string,date,boolean");
}
}
-
+ //SingleValueSortDoc etc are specialized classes which don't have array lookups. On benchmarking large datasets
+ //This is faster than the using an array in SortDoc . So upto 4 sort fields we still want to keep specialized classes.
+ //SOLR-12616 has more details
+ if (sortValues.length == 1) {
+ return new SingleValueSortDoc(sortValues[0]);
+ } else if (sortValues.length == 2) {
+ return new DoubleValueSortDoc(sortValues[0], sortValues[1]);
+ } else if (sortValues.length == 3) {
+ return new TripleValueSortDoc(sortValues[0], sortValues[1], sortValues[2]);
+ } else if (sortValues.length == 4) {
+ return new QuadValueSortDoc(sortValues[0], sortValues[1], sortValues[2], sortValues[3]);
+ }
return new SortDoc(sortValues);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e9f3a3ce/solr/core/src/java/org/apache/solr/handler/export/QuadValueSortDoc.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/QuadValueSortDoc.java b/solr/core/src/java/org/apache/solr/handler/export/QuadValueSortDoc.java
new file mode 100644
index 0000000..c2dfcba
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/QuadValueSortDoc.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.LeafReaderContext;
+
+class QuadValueSortDoc extends TripleValueSortDoc {
+
+ protected SortValue value4;
+
+ public void setNextReader(LeafReaderContext context) throws IOException {
+ this.ord = context.ord;
+ this.docBase = context.docBase;
+ value1.setNextReader(context);
+ value2.setNextReader(context);
+ value3.setNextReader(context);
+ value4.setNextReader(context);
+ }
+
+ public void reset() {
+ this.docId = -1;
+ this.docBase = -1;
+ value1.reset();
+ value2.reset();
+ value3.reset();
+ value4.reset();
+ }
+
+ public void setValues(int docId) throws IOException {
+ this.docId = docId;
+ value1.setCurrentValue(docId);
+ value2.setCurrentValue(docId);
+ value3.setCurrentValue(docId);
+ value4.setCurrentValue(docId);
+ }
+
+ public void setValues(SortDoc sortDoc) {
+ this.docId = sortDoc.docId;
+ this.ord = sortDoc.ord;
+ this.docBase = sortDoc.docBase;
+ value1.setCurrentValue(((QuadValueSortDoc)sortDoc).value1);
+ value2.setCurrentValue(((QuadValueSortDoc)sortDoc).value2);
+ value3.setCurrentValue(((QuadValueSortDoc)sortDoc).value3);
+ value4.setCurrentValue(((QuadValueSortDoc)sortDoc).value4);
+ }
+
+ public QuadValueSortDoc(SortValue value1, SortValue value2, SortValue value3, SortValue value4) {
+ super(value1, value2, value3);
+ this.value4 = value4;
+ }
+
+ public SortDoc copy() {
+ return new QuadValueSortDoc(value1.copy(), value2.copy(), value3.copy(), value4.copy());
+ }
+
+ public boolean lessThan(Object o) {
+
+ QuadValueSortDoc sd = (QuadValueSortDoc)o;
+ int comp = value1.compareTo(sd.value1);
+ if(comp == -1) {
+ return true;
+ } else if (comp == 1) {
+ return false;
+ } else {
+ comp = value2.compareTo(sd.value2);
+ if(comp == -1) {
+ return true;
+ } else if (comp == 1) {
+ return false;
+ } else {
+ comp = value3.compareTo(sd.value3);
+ if(comp == -1) {
+ return true;
+ } else if (comp == 1) {
+ return false;
+ } else {
+ comp = value4.compareTo(sd.value4);
+ if(comp == -1) {
+ return true;
+ } else if (comp == 1) {
+ return false;
+ } else {
+ return docId+docBase > sd.docId+sd.docBase;
+ }
+ }
+ }
+ }
+ }
+
+ public int compareTo(Object o) {
+ QuadValueSortDoc sd = (QuadValueSortDoc)o;
+ int comp = value1.compareTo(sd.value1);
+ if(comp == 0) {
+ comp = value2.compareTo(sd.value2);
+ if(comp == 0) {
+ comp = value3.compareTo(sd.value3);
+ if(comp == 0) {
+ return value4.compareTo(sd.value4);
+ } else {
+ return comp;
+ }
+ } else {
+ return comp;
+ }
+ } else {
+ return comp;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e9f3a3ce/solr/core/src/java/org/apache/solr/handler/export/SingleValueSortDoc.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/SingleValueSortDoc.java b/solr/core/src/java/org/apache/solr/handler/export/SingleValueSortDoc.java
new file mode 100644
index 0000000..88fc799
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/SingleValueSortDoc.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.LeafReaderContext;
+
+class SingleValueSortDoc extends SortDoc {
+
+ protected SortValue value1;
+
+ public void setNextReader(LeafReaderContext context) throws IOException {
+ this.ord = context.ord;
+ this.docBase = context.docBase;
+ value1.setNextReader(context);
+ }
+
+ public void reset() {
+ this.docId = -1;
+ this.docBase = -1;
+ this.value1.reset();
+ }
+
+ public void setValues(int docId) throws IOException {
+ this.docId = docId;
+ value1.setCurrentValue(docId);
+ }
+
+ public void setValues(SortDoc sortDoc) {
+ this.docId = sortDoc.docId;
+ this.ord = sortDoc.ord;
+ this.docBase = sortDoc.docBase;
+ value1.setCurrentValue(((SingleValueSortDoc)sortDoc).value1);
+ }
+
+ public SingleValueSortDoc(SortValue value1) {
+ super();
+ this.value1 = value1;
+ }
+
+ public SortDoc copy() {
+ return new SingleValueSortDoc(value1.copy());
+ }
+
+ public boolean lessThan(Object o) {
+ SingleValueSortDoc sd = (SingleValueSortDoc)o;
+ int comp = value1.compareTo(sd.value1);
+ if(comp == -1) {
+ return true;
+ } else if (comp == 1) {
+ return false;
+ } else {
+ return docId+docBase > sd.docId+sd.docBase;
+ }
+ }
+
+ public int compareTo(Object o) {
+ SingleValueSortDoc sd = (SingleValueSortDoc)o;
+ return value1.compareTo(sd.value1);
+ }
+
+ public String toString() {
+ return docId+":"+value1.toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e9f3a3ce/solr/core/src/java/org/apache/solr/handler/export/SortDoc.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/SortDoc.java b/solr/core/src/java/org/apache/solr/handler/export/SortDoc.java
index 3cf478b..1302332 100644
--- a/solr/core/src/java/org/apache/solr/handler/export/SortDoc.java
+++ b/solr/core/src/java/org/apache/solr/handler/export/SortDoc.java
@@ -32,6 +32,8 @@ class SortDoc {
public SortDoc(SortValue[] sortValues) {
this.sortValues = sortValues;
}
+ public SortDoc() {
+ }
public void setNextReader(LeafReaderContext context) throws IOException {
this.ord = context.ord;
@@ -43,6 +45,7 @@ class SortDoc {
public void reset() {
this.docId = -1;
+ this.docBase = -1;
for (SortValue value : sortValues) {
value.reset();
}
@@ -82,9 +85,9 @@ class SortDoc {
SortValue[] sortValues1 = sd.sortValues;
for(int i=0; i<sortValues.length; i++) {
int comp = sortValues[i].compareTo(sortValues1[i]);
- if(comp < 0) {
+ if (comp < 0) {
return true;
- } if(comp > 0) {
+ } else if (comp > 0) {
return false;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e9f3a3ce/solr/core/src/java/org/apache/solr/handler/export/TripleValueSortDoc.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/TripleValueSortDoc.java b/solr/core/src/java/org/apache/solr/handler/export/TripleValueSortDoc.java
new file mode 100644
index 0000000..0241ebc
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/TripleValueSortDoc.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.LeafReaderContext;
+
+class TripleValueSortDoc extends DoubleValueSortDoc {
+
+ protected SortValue value3;
+
+ public void setNextReader(LeafReaderContext context) throws IOException {
+ this.ord = context.ord;
+ this.docBase = context.docBase;
+ value1.setNextReader(context);
+ value2.setNextReader(context);
+ value3.setNextReader(context);
+ }
+
+ public void reset() {
+ this.docId = -1;
+ this.docBase = -1;
+ value1.reset();
+ value2.reset();
+ value3.reset();
+ }
+
+ public void setValues(int docId) throws IOException {
+ this.docId = docId;
+ value1.setCurrentValue(docId);
+ value2.setCurrentValue(docId);
+ value3.setCurrentValue(docId);
+ }
+
+ public void setValues(SortDoc sortDoc) {
+ this.docId = sortDoc.docId;
+ this.ord = sortDoc.ord;
+ this.docBase = sortDoc.docBase;
+ value1.setCurrentValue(((TripleValueSortDoc)sortDoc).value1);
+ value2.setCurrentValue(((TripleValueSortDoc)sortDoc).value2);
+ value3.setCurrentValue(((TripleValueSortDoc)sortDoc).value3);
+ }
+
+ public TripleValueSortDoc(SortValue value1, SortValue value2, SortValue value3) {
+ super(value1, value2);
+ this.value3 = value3;
+ }
+
+ public SortDoc copy() {
+ return new TripleValueSortDoc(value1.copy(), value2.copy(), value3.copy());
+ }
+
+ public boolean lessThan(Object o) {
+
+ TripleValueSortDoc sd = (TripleValueSortDoc)o;
+ int comp = value1.compareTo(sd.value1);
+ if(comp == -1) {
+ return true;
+ } else if (comp == 1) {
+ return false;
+ } else {
+ comp = value2.compareTo(sd.value2);
+ if(comp == -1) {
+ return true;
+ } else if (comp == 1) {
+ return false;
+ } else {
+ comp = value3.compareTo(sd.value3);
+ if(comp == -1) {
+ return true;
+ } else if (comp == 1) {
+ return false;
+ } else {
+ return docId+docBase > sd.docId+sd.docBase;
+ }
+ }
+ }
+ }
+
+ public int compareTo(Object o) {
+ TripleValueSortDoc sd = (TripleValueSortDoc)o;
+ int comp = value1.compareTo(sd.value1);
+ if (comp == 0) {
+ comp = value2.compareTo(sd.value2);
+ if (comp == 0) {
+ return value3.compareTo(sd.value3);
+ } else {
+ return comp;
+ }
+ } else {
+ return comp;
+ }
+ }
+}