You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2023/07/20 15:40:41 UTC
[tika] branch TIKA-4102 created (now ac62c965a)
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a change to branch TIKA-4102
in repository https://gitbox.apache.org/repos/asf/tika.git
at ac62c965a TIKA-4102 -- add file_name to reports that may include embedded files
This branch includes the following new commits:
new 7625478fd TIKA-4102 -- reformat comparison-reports.xml file
new ac62c965a TIKA-4102 -- add file_name to reports that may include embedded files
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[tika] 02/02: TIKA-4102 -- add file_name to reports that may include embedded files
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch TIKA-4102
in repository https://gitbox.apache.org/repos/asf/tika.git
commit ac62c965af1afbfaddc47d6ae2a52fc9ee9da3ab
Author: tallison <ta...@apache.org>
AuthorDate: Thu Jul 20 11:40:23 2023 -0400
TIKA-4102 -- add file_name to reports that may include embedded files
---
.../src/main/resources/comparison-reports.xml | 18 ++++++++++++++++--
1 file changed, 16 insertions(+), 2 deletions(-)
diff --git a/tika-eval/tika-eval-app/src/main/resources/comparison-reports.xml b/tika-eval/tika-eval-app/src/main/resources/comparison-reports.xml
index 2c96bda58..2bd83431e 100644
--- a/tika-eval/tika-eval-app/src/main/resources/comparison-reports.xml
+++ b/tika-eval/tika-eval-app/src/main/resources/comparison-reports.xml
@@ -454,7 +454,7 @@
<sql>
select concat(ma.mime_string, ' -> ', mb.mime_string) as
MIME_A_TO_MIME_B,
- file_path,
+ file_path, a.file_name,
c.length as CONTAINER_LENGTH,
a.file_name
from profiles_a a
@@ -576,6 +576,7 @@
<sql>
select file_path as FILE_PATH,
+ pa.file_name as FILE_NAME_A,
c.length as CONTAINER_LENGTH,
ca.NUM_TOKENS as NUM_TOKENS_A,
cb.NUM_TOKENS as NUM_TOKENS_B,
@@ -629,6 +630,7 @@
<sql>
select
file_path,
+ pa.file_name as FILE_NAME_A,
c.length as CONTAINER_LENGTH,
ma.mime_string as MIME_TYPE_A,
mb.mime_string as MIME_TYPE_B,
@@ -652,6 +654,7 @@
<sql>
select file_path,
+ pa.file_name as FILE_NAME_A,
c.length as CONTAINER_LENGTH,
ma.mime_string as MIME_TYPE_A,
mb.mime_string as MIME_TYPE_B,
@@ -723,6 +726,7 @@
<sql>
select file_path,
+ pa.file_name as FILE_NAME_A,
c.length as CONTAINER_LENGTH,
ma.mime_string as MIME_TYPE_A,
mb.mime_string as MIME_TYPE_B,
@@ -764,6 +768,7 @@
<sql>
select file_path,
+ p.file_name as FILE_NAME_A,
c.length as CONTAINER_LENGTH,
mime_string as MIME_TYPE,
orig_stack_trace, sort_stack_trace
@@ -783,6 +788,7 @@
<sql>
select file_path,
+ p.file_name as FILE_NAME_B,
c.length as CONTAINER_LENGTH,
mime_string as MIME_TYPE,
orig_stack_trace, sort_stack_trace
@@ -940,6 +946,7 @@
<sql>
select file_path,
+ pa.file_name as FILE_NAME_A,
c.length as CONTAINER_LENGTH,
ma.mime_string as MIME_STRING_A,
mb.mime_string as MIME_STRING_B,
@@ -990,6 +997,7 @@
<sql>
select file_path,
+ pa.file_name as FILE_NAME_A,
c.length as CONTAINER_LENGTH,
ma.mime_string as MIME_STRING_A,
mb.mime_string as MIME_STRING_B,
@@ -1052,6 +1060,7 @@
<sql>
select file_path,
+ pa.file_name as FILE_NAME_A,
c.length as CONTAINER_LENGTH,
ma.mime_string as MIME_STRING_A,
mb.mime_string as MIME_STRING_B,
@@ -1155,6 +1164,7 @@
<sql>
select file_path,
+ pa.file_name as FILE_NAME_A,
c.length as CONTAINER_LENGTH,
ma.mime_string as MIME_STRING_A,
mb.mime_string as MIME_STRING_B,
@@ -1184,6 +1194,7 @@
<sql>
select file_path,
+ pa.file_name as FILE_NAME_A,
c.length as CONTAINER_LENGTH,
ma.mime_string as MIME_STRING_A,
mb.mime_string as MIME_STRING_B,
@@ -1306,6 +1317,7 @@
<sql>
select file_path,
+ pa.file_name as FILE_NAME_A,
ma.mime_string as mime_string_a,
mb.mime_string as mime_string_b,
pa.num_metadata_values as num_metadata_values_a,
@@ -1349,7 +1361,9 @@
format="xlsx"
includeSql="true">
<sql>
- select file_path, c.length as CONTAINTER_LENGTH,
+ select file_path,
+ pa.file_name as FILE_NAME_A,
+ c.length as CONTAINTER_LENGTH,
ma.mime_string as MIME_STRING_A,
mb.mime_string as MIME_STRING_B,
pa.elapsed_time_millis as TOTAL_MILLIS_A,
[tika] 01/02: TIKA-4102 -- reformat comparison-reports.xml file
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch TIKA-4102
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 7625478fd04299afd3bc555a70799a3a0ca8d515
Author: tallison <ta...@apache.org>
AuthorDate: Thu Jul 20 11:15:14 2023 -0400
TIKA-4102 -- reformat comparison-reports.xml file
---
.../src/main/resources/comparison-reports.xml | 104 ++++++++++++---------
1 file changed, 62 insertions(+), 42 deletions(-)
diff --git a/tika-eval/tika-eval-app/src/main/resources/comparison-reports.xml b/tika-eval/tika-eval-app/src/main/resources/comparison-reports.xml
index 69bda9cc6..2c96bda58 100644
--- a/tika-eval/tika-eval-app/src/main/resources/comparison-reports.xml
+++ b/tika-eval/tika-eval-app/src/main/resources/comparison-reports.xml
@@ -92,22 +92,24 @@
on exceptions_compared(mime_id_b);
</sql>
- <!-- we need to create literal tmp tables in h2 TIKA-4100 -->
- <sql>drop table if exists exc_count_a;</sql>
+ <!-- we need to create literal tmp tables in h2 TIKA-4100 -->
+ <sql>drop table if exists exc_count_a;</sql>
<sql>create table exc_count_a (mime_id_a integer, mime_id_b integer, cnt integer);</sql>
<sql>insert into exc_count_a (
- select ma.mime_id mime_id_a, mb.mime_id mime_id_b, count(1) as cnt
- from exceptions_a ea
- join profiles_a pa on ea.id=pa.id
- join profiles_b pb on pb.id=pa.id
- join mimes ma on pa.mime_id=ma.mime_id
- join mimes mb on pb.mime_id=mb.mime_id
- group by ma.mime_id, mb.mime_id
- );</sql>
+ select ma.mime_id mime_id_a, mb.mime_id mime_id_b, count(1) as cnt
+ from exceptions_a ea
+ join profiles_a pa on ea.id=pa.id
+ join profiles_b pb on pb.id=pa.id
+ join mimes ma on pa.mime_id=ma.mime_id
+ join mimes mb on pb.mime_id=mb.mime_id
+ group by ma.mime_id, mb.mime_id
+ );
+ </sql>
<sql>update exceptions_compared ec set
- exc_cnt_a = (
- select cnt from exc_count_a ea
- where ec.mime_id_a=ea.mime_id_a and ec.mime_id_b=ea.mime_id_b);</sql>
+ exc_cnt_a = (
+ select cnt from exc_count_a ea
+ where ec.mime_id_a=ea.mime_id_a and ec.mime_id_b=ea.mime_id_b);
+ </sql>
<sql>drop table exc_count_a</sql>
<sql>drop table if exists exc_count_b;</sql>
@@ -120,11 +122,13 @@
join mimes ma on pa.mime_id=ma.mime_id
join mimes mb on pb.mime_id=mb.mime_id
group by ma.mime_id, mb.mime_id
- );</sql>
+ );
+ </sql>
<sql>update exceptions_compared ec set
exc_cnt_b = (
select cnt from exc_count_b eb
- where ec.mime_id_a=eb.mime_id_a and ec.mime_id_b=eb.mime_id_b);</sql>
+ where ec.mime_id_a=eb.mime_id_a and ec.mime_id_b=eb.mime_id_b);
+ </sql>
<sql>drop table exc_count_b</sql>
<sql>
@@ -186,15 +190,17 @@
<sql>create table tcc_tmp (mime_id_a integer, mime_id_b integer, cnt long default 0);</sql>
<!-- num_tokens_a -->
<sql>insert into tcc_tmp (
- select pa.mime_id, pb.mime_id, sum(num_tokens) as cnt from profiles_a pa
- join profiles_b pb on pa.id=pb.id
- join contents_a c on c.id = pa.id
- group by pa.mime_id, pb.mime_id
- );</sql>
+ select pa.mime_id, pb.mime_id, sum(num_tokens) as cnt from profiles_a pa
+ join profiles_b pb on pa.id=pb.id
+ join contents_a c on c.id = pa.id
+ group by pa.mime_id, pb.mime_id
+ );
+ </sql>
<sql>update token_counts_compared tcc set
- num_tokens_a = (
- select cnt from tcc_tmp tmp
- where tcc.mime_id_a=tmp.mime_id_a and tcc.mime_id_b=tmp.mime_id_b);</sql>
+ num_tokens_a = (
+ select cnt from tcc_tmp tmp
+ where tcc.mime_id_a=tmp.mime_id_a and tcc.mime_id_b=tmp.mime_id_b);
+ </sql>
<sql>truncate table tcc_tmp;</sql>
<!-- num_tokens_b -->
@@ -203,11 +209,13 @@
join profiles_a pa on pa.id=pb.id
join contents_b c on c.id = pb.id
group by pa.mime_id, pb.mime_id
- );</sql>
+ );
+ </sql>
<sql>update token_counts_compared tcc set
num_tokens_b = (
select cnt from tcc_tmp tmp
- where tcc.mime_id_a=tmp.mime_id_a and tcc.mime_id_b=tmp.mime_id_b);</sql>
+ where tcc.mime_id_a=tmp.mime_id_a and tcc.mime_id_b=tmp.mime_id_b);
+ </sql>
<sql>truncate table tcc_tmp;</sql>
<!-- num_tokens_alphabetic_tokens_a -->
@@ -216,11 +224,13 @@
join profiles_b pb on pa.id=pb.id
join contents_a c on c.id = pa.id
group by pa.mime_id, pb.mime_id
- );</sql>
+ );
+ </sql>
<sql>update token_counts_compared tcc set
num_alphabetic_tokens_a = (
select cnt from tcc_tmp tmp
- where tcc.mime_id_a=tmp.mime_id_a and tcc.mime_id_b=tmp.mime_id_b);</sql>
+ where tcc.mime_id_a=tmp.mime_id_a and tcc.mime_id_b=tmp.mime_id_b);
+ </sql>
<sql>truncate table tcc_tmp;</sql>
<!-- num_alphabetic_tokens_b -->
@@ -229,11 +239,13 @@
join profiles_a pa on pa.id=pb.id
join contents_b c on c.id = pb.id
group by pa.mime_id, pb.mime_id
- );</sql>
+ );
+ </sql>
<sql>update token_counts_compared tcc set
num_alphabetic_tokens_b = (
select cnt from tcc_tmp tmp
- where tcc.mime_id_a=tmp.mime_id_a and tcc.mime_id_b=tmp.mime_id_b);</sql>
+ where tcc.mime_id_a=tmp.mime_id_a and tcc.mime_id_b=tmp.mime_id_b);
+ </sql>
<sql>truncate table tcc_tmp;</sql>
<!-- num_common_tokens_a -->
@@ -242,11 +254,13 @@
join profiles_b pb on pa.id=pb.id
join contents_a c on c.id = pa.id
group by pa.mime_id, pb.mime_id
- );</sql>
+ );
+ </sql>
<sql>update token_counts_compared tcc set
num_common_tokens_a = (
select cnt from tcc_tmp tmp
- where tcc.mime_id_a=tmp.mime_id_a and tcc.mime_id_b=tmp.mime_id_b);</sql>
+ where tcc.mime_id_a=tmp.mime_id_a and tcc.mime_id_b=tmp.mime_id_b);
+ </sql>
<sql>truncate table tcc_tmp;</sql>
<!-- num_common_tokens_b -->
@@ -255,11 +269,13 @@
join profiles_a pa on pa.id=pb.id
join contents_b c on c.id = pb.id
group by pa.mime_id, pb.mime_id
- );</sql>
+ );
+ </sql>
<sql>update token_counts_compared tcc set
num_common_tokens_b = (
select cnt from tcc_tmp tmp
- where tcc.mime_id_a=tmp.mime_id_a and tcc.mime_id_b=tmp.mime_id_b);</sql>
+ where tcc.mime_id_a=tmp.mime_id_a and tcc.mime_id_b=tmp.mime_id_b);
+ </sql>
<sql>drop table tcc_tmp;</sql>
<sql>
drop table if exists parse_time_compared;
@@ -294,14 +310,16 @@
<!-- total parse time a -->
<sql>create table ptc_tmp (mime_id_a integer, mime_id_b integer, total long default 0);</sql>
<sql>insert into ptc_tmp (
- select pa.mime_id, pb.mime_id, sum(pa.elapsed_time_millis) as total from profiles_a pa
- join profiles_b pb on pa.id=pb.id
- group by pa. mime_id, pb.mime_id
- );</sql>
+ select pa.mime_id, pb.mime_id, sum(pa.elapsed_time_millis) as total from profiles_a pa
+ join profiles_b pb on pa.id=pb.id
+ group by pa. mime_id, pb.mime_id
+ );
+ </sql>
<sql>update parse_time_compared ptc set
- total_a = (
- select total from ptc_tmp tmp
- where ptc.mime_id_a=tmp.mime_id_a and ptc.mime_id_b=tmp.mime_id_b);</sql>
+ total_a = (
+ select total from ptc_tmp tmp
+ where ptc.mime_id_a=tmp.mime_id_a and ptc.mime_id_b=tmp.mime_id_b);
+ </sql>
<sql>truncate table ptc_tmp;</sql>
<!-- total parse time b -->
@@ -309,11 +327,13 @@
select pa.mime_id, pb.mime_id, sum(pb.elapsed_time_millis) as total from profiles_b pb
join profiles_a pa on pb.id=pa.id
group by pa.mime_id, pb.mime_id
- );</sql>
+ );
+ </sql>
<sql>update parse_time_compared ptc set
total_b = (
select total from ptc_tmp tmp
- where ptc.mime_id_a=tmp.mime_id_a and ptc.mime_id_b=tmp.mime_id_b);</sql>
+ where ptc.mime_id_a=tmp.mime_id_a and ptc.mime_id_b=tmp.mime_id_b);
+ </sql>
<sql>drop table ptc_tmp;</sql>
<sql>
update parse_time_compared ptc set prcnt_increase=(100.0 *