You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by gu...@apache.org on 2025/03/19 15:43:11 UTC

(lucene) branch dynamic_pruning_data created (now 7975606642e)

This is an automated email from the ASF dual-hosted git repository.

guofeng pushed a change to branch dynamic_pruning_data
in repository https://gitbox.apache.org/repos/asf/lucene.git


      at 7975606642e iter

This branch includes the following new commits:

     new e81c0a6f6c7 data
     new a74168f3ad6 iter
     new e08c12062bd iter
     new 7975606642e iter

The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



(lucene) 04/04: iter

Posted by gu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

guofeng pushed a commit to branch dynamic_pruning_data
in repository https://gitbox.apache.org/repos/asf/lucene.git

commit 7975606642e79075ca5b06ee05bbdeb8ffa6ea45
Author: guofeng.my <gu...@bytedance.com>
AuthorDate: Wed Mar 19 23:42:48 2025 +0800

    iter
---
 .../org/apache/lucene/search/comparators/NumericComparator.java   | 8 ++++++++
 lucene/core/src/java/org/apache/lucene/util/DocIdSetBuilder.java  | 4 ++--
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.java b/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.java
index 7a9d97c652b..c87b0bad6b3 100644
--- a/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.java
+++ b/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.java
@@ -223,6 +223,9 @@ public abstract class NumericComparator<T extends Number> extends FieldComparato
         encodeBottom();
       }
 
+      int[] filtered = {0};
+      int[] collected = {0};
+
       DocIdSetBuilder result = new DocIdSetBuilder(maxDoc);
       PointValues.IntersectVisitor visitor =
           new PointValues.IntersectVisitor() {
@@ -236,16 +239,20 @@ public abstract class NumericComparator<T extends Number> extends FieldComparato
             @Override
             public void visit(int docID) {
               if (docID <= maxDocVisited) {
+                filtered[0]++;
                 return; // Already visited or skipped
               }
+              collected[0]++;
               adder.add(docID);
             }
 
             @Override
             public void visit(int docID, byte[] packedValue) {
               if (docID <= maxDocVisited) {
+                filtered[0]++;
                 return; // already visited or skipped
               }
+              collected[0]++;
               long l = sortableBytesToLong(packedValue);
               if (l >= minValueAsLong && l <= maxValueAsLong) {
                 adder.add(docID); // doc is competitive
@@ -287,6 +294,7 @@ public abstract class NumericComparator<T extends Number> extends FieldComparato
         return;
       }
       pointValues.intersect(visitor);
+      System.out.println("filtered: " + filtered[0] + ", collected: " + collected);
       competitiveIterator = result.build().iterator();
       iteratorCost = competitiveIterator.cost();
       updateSkipInterval(true);
diff --git a/lucene/core/src/java/org/apache/lucene/util/DocIdSetBuilder.java b/lucene/core/src/java/org/apache/lucene/util/DocIdSetBuilder.java
index 0d0476d3d9e..95956f44dc1 100644
--- a/lucene/core/src/java/org/apache/lucene/util/DocIdSetBuilder.java
+++ b/lucene/core/src/java/org/apache/lucene/util/DocIdSetBuilder.java
@@ -291,9 +291,9 @@ public final class DocIdSetBuilder {
         int base = min & 0xFFFFFFC0;
         int distance = max - base + 1;
         if ((distance >> 7) < concatenated.length) {
-          System.out.println("true " + (distance / concatenated.length));
+          System.out.println("true, distance:" + distance + ", concatenated" + concatenated.length + ", factor: " + (distance / concatenated.length));
         } else {
-          System.out.println("false " + (distance / concatenated.length));
+          System.out.println("false, distance:" + distance + ", concatenated" + concatenated.length + ", factor: " + (distance / concatenated.length));
         }
         LSBRadixSorter sorter = new LSBRadixSorter();
         sorter.sort(PackedInts.bitsRequired(max), concatenated.array, concatenated.length);


(lucene) 01/04: data

Posted by gu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

guofeng pushed a commit to branch dynamic_pruning_data
in repository https://gitbox.apache.org/repos/asf/lucene.git

commit e81c0a6f6c726b3d250aefb56af61341373e5ca7
Author: guofeng.my <gu...@bytedance.com>
AuthorDate: Wed Mar 19 23:11:11 2025 +0800

    data
---
 .../src/java/org/apache/lucene/util/DocIdSetBuilder.java | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/lucene/core/src/java/org/apache/lucene/util/DocIdSetBuilder.java b/lucene/core/src/java/org/apache/lucene/util/DocIdSetBuilder.java
index 6d0b9f3b860..553dd3a26f2 100644
--- a/lucene/core/src/java/org/apache/lucene/util/DocIdSetBuilder.java
+++ b/lucene/core/src/java/org/apache/lucene/util/DocIdSetBuilder.java
@@ -279,8 +279,21 @@ public final class DocIdSetBuilder {
         return new BitDocIdSet(bitSet, cost);
       } else {
         Buffer concatenated = concat(buffers);
+        int[] array = concatenated.array;
+        int min = concatenated.array[0], max = concatenated.array[0];
+        for (int i = 0, to = concatenated.length; i < to; i++) {
+          min = Math.min(array[i], min);
+          max = Math.max(array[i], max);
+        }
+        int base = min & 0xFFFFFFC0;
+        int distance = max - base + 1;
+        if ((distance >> 7) < concatenated.length) {
+          System.out.println(true + " " + (distance / concatenated.length));
+        } else {
+          System.out.println(false + " " + (distance / concatenated.length));
+        }
         LSBRadixSorter sorter = new LSBRadixSorter();
-        sorter.sort(PackedInts.bitsRequired(maxDoc - 1), concatenated.array, concatenated.length);
+        sorter.sort(PackedInts.bitsRequired(max), concatenated.array, concatenated.length);
         final int l;
         if (multivalued) {
           l = dedup(concatenated.array, concatenated.length);
@@ -291,6 +304,7 @@ public final class DocIdSetBuilder {
         assert l <= concatenated.length;
         concatenated.array[l] = DocIdSetIterator.NO_MORE_DOCS;
         return new IntArrayDocIdSet(concatenated.array, l);
+
       }
     } finally {
       this.buffers = null;


(lucene) 02/04: iter

Posted by gu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

guofeng pushed a commit to branch dynamic_pruning_data
in repository https://gitbox.apache.org/repos/asf/lucene.git

commit a74168f3ad652fdbbfcd78753e9dff0b6bbedb23
Author: guofeng.my <gu...@bytedance.com>
AuthorDate: Wed Mar 19 23:18:16 2025 +0800

    iter
---
 .../apache/lucene/search/comparators/NumericComparator.java    | 10 ----------
 .../core/src/java/org/apache/lucene/util/DocIdSetBuilder.java  |  4 ++--
 2 files changed, 2 insertions(+), 12 deletions(-)

diff --git a/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.java b/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.java
index df7f57933a1..7a9d97c652b 100644
--- a/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.java
+++ b/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.java
@@ -252,16 +252,6 @@ public abstract class NumericComparator<T extends Number> extends FieldComparato
               }
             }
 
-            @Override
-            public void visit(IntsRef ref) {
-              adder.add(ref);
-            }
-
-            @Override
-            public void visit(DocIdSetIterator iterator) throws IOException {
-              adder.add(iterator);
-            }
-
             @Override
             public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
               long min = sortableBytesToLong(minPackedValue);
diff --git a/lucene/core/src/java/org/apache/lucene/util/DocIdSetBuilder.java b/lucene/core/src/java/org/apache/lucene/util/DocIdSetBuilder.java
index 553dd3a26f2..d8824f8c0e3 100644
--- a/lucene/core/src/java/org/apache/lucene/util/DocIdSetBuilder.java
+++ b/lucene/core/src/java/org/apache/lucene/util/DocIdSetBuilder.java
@@ -288,9 +288,9 @@ public final class DocIdSetBuilder {
         int base = min & 0xFFFFFFC0;
         int distance = max - base + 1;
         if ((distance >> 7) < concatenated.length) {
-          System.out.println(true + " " + (distance / concatenated.length));
+          System.out.println("true " + (distance / concatenated.length));
         } else {
-          System.out.println(false + " " + (distance / concatenated.length));
+          System.out.println("false " + (distance / concatenated.length));
         }
         LSBRadixSorter sorter = new LSBRadixSorter();
         sorter.sort(PackedInts.bitsRequired(max), concatenated.array, concatenated.length);


(lucene) 03/04: iter

Posted by gu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

guofeng pushed a commit to branch dynamic_pruning_data
in repository https://gitbox.apache.org/repos/asf/lucene.git

commit e08c12062bdc159ffe03a6795e90a0ebe68bc622
Author: guofeng.my <gu...@bytedance.com>
AuthorDate: Wed Mar 19 23:24:20 2025 +0800

    iter
---
 lucene/core/src/java/org/apache/lucene/util/DocIdSetBuilder.java | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lucene/core/src/java/org/apache/lucene/util/DocIdSetBuilder.java b/lucene/core/src/java/org/apache/lucene/util/DocIdSetBuilder.java
index d8824f8c0e3..0d0476d3d9e 100644
--- a/lucene/core/src/java/org/apache/lucene/util/DocIdSetBuilder.java
+++ b/lucene/core/src/java/org/apache/lucene/util/DocIdSetBuilder.java
@@ -279,6 +279,9 @@ public final class DocIdSetBuilder {
         return new BitDocIdSet(bitSet, cost);
       } else {
         Buffer concatenated = concat(buffers);
+        if (concatenated.length == 0) {
+          return DocIdSet.EMPTY;
+        }
         int[] array = concatenated.array;
         int min = concatenated.array[0], max = concatenated.array[0];
         for (int i = 0, to = concatenated.length; i < to; i++) {