[tika] branch main updated: TIKA-3513 -- add quintile reports for dice and overlap

[ta...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

tallison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git


The following commit(s) were added to refs/heads/main by this push:
     new 1a28bd6  TIKA-3513 -- add quintile reports for dice and overlap
1a28bd6 is described below

commit 1a28bd6905a742a28105bfbddf9b056e14402446
Author: tallison <ta...@apache.org>
AuthorDate: Thu Aug 5 11:26:44 2021 -0400

    TIKA-3513 -- add quintile reports for dice and overlap
---
 .../src/main/resources/comparison-reports.xml      | 45 ++++++++++++++++++++++
 1 file changed, 45 insertions(+)

diff --git a/tika-eval/tika-eval-app/src/main/resources/comparison-reports.xml b/tika-eval/tika-eval-app/src/main/resources/comparison-reports.xml
index 9f9090b..9a8e29b 100644
--- a/tika-eval/tika-eval-app/src/main/resources/comparison-reports.xml
+++ b/tika-eval/tika-eval-app/src/main/resources/comparison-reports.xml
@@ -808,6 +808,51 @@
         </sql>
     </report>
 
+    <report reportName="DiceQuintiles"
+            reportFilename="content/dice_quintiles.xlsx"
+            format="xlsx"
+            includeSql="true">
+        <sql>
+            SELECT
+            case
+            when dice_coefficient &lt; 0.20 then '0&lt;0.20%'
+            when dice_coefficient &lt; 0.40 then '20%&lt;40%'
+            when dice_coefficient &lt; 0.60 then '40%&lt;60%'
+            when dice_coefficient &lt; 0.80 then '60%&lt;80%'
+            when dice_coefficient &lt; 1.01 then '80%-100%'
+            else 'other'
+            end as  range,
+            COUNT(*) AS COUNT,
+            FROM   content_comparisons cc
+            join contents_a ca on cc.id = ca.id
+            join contents_b cb on cc.id = cb.id
+            where ca.num_tokens > 10 or cb.num_tokens > 10
+            GROUP  BY  range
+        </sql>
+    </report>
+
+    <report reportName="OverlapQuintiles"
+            reportFilename="content/overlap_quintiles.xlsx"
+            format="xlsx"
+            includeSql="true">
+        <sql>
+            SELECT
+            case
+            when overlap &lt; 0.20 then '0&lt;0.20%'
+            when overlap &lt; 0.40 then '20%&lt;40%'
+            when overlap &lt; 0.60 then '40%&lt;60%'
+            when overlap &lt; 0.80 then '60%&lt;80%'
+            when overlap &lt; 1.01 then '80%-100%'
+            else 'other'
+            end as  range,
+            COUNT(*) AS COUNT,
+            FROM   content_comparisons cc
+            join contents_a ca on cc.id = ca.id
+            join contents_b cb on cc.id = cb.id
+            where ca.num_tokens > 10 or cb.num_tokens > 10
+            GROUP  BY  range
+        </sql>
+    </report>
     <report reportName="contentDiffsWExceptions"
             reportFilename="content/content_diffs_with_exceptions.xlsx"
             format="xlsx"