You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-user@lucene.apache.org by Ma...@rzf.fin-nrw.de on 2010/05/12 15:00:26 UTC

strange behaviour when sorting, fields are missing in result

when i do a search, eg. 

http://xxx:8983/solr/select?q=steuer&fl=score,id,__intern,title,__source,_dienststelle,_erstelldatum,__cyear,_stelle

i get a normal result, like

<result name="response" numFound="21502" start="0" maxScore="1.3633566">
<doc>
<float name="score">1.3633566</float>
<int name="__cyear">2009</int>
<str name="__intern">0</str>
<str name="__source">zzz</str>
<str name="_dienststelle">xyz</str>
<long name="_erstelldatum">20090202000000</long>
<str name="_stelle">Presse- u. Informationsreferat</str>
<str name="id">34931684</str>
<str name="title">Merkblatt Vereine und Steuern</str>
</doc>

when i do a search with the sort param, my result is suddenly truncated:

http://xxx:8983/solr/select?q=steuer&fl=score,id,__intern,title,__source,_dienststelle,_erstelldatum,__cyear,_stelle&sort=_erstelldatum+asc

<result name="response" numFound="21502" start="0" maxScore="1.3633566">
<doc>
<float name="score">0.14290115</float>
<str name="__intern">0</str>
<str name="__source">isys</str>
<str name="id">18205520</str>
<str name="title">Amtsübersicht </str>
</doc>

so, not all of the fields from fl-param are displayed. this is what admin schema browser says about _erstelldatum:

Field Type: long
Properties: Indexed, Tokenized, Stored, Omit Norms, undefined
Schema: Indexed, Tokenized, Stored, Omit Norms, undefined
Index: Stored, Omit Norms, Binary
Index Analyzer: org.apache.solr.analysis.TokenizerChain Details
Tokenizer Class: org.apache.solr.analysis.TrieTokenizerFactory
Query Analyzer: org.apache.solr.analysis.TokenizerChain Details
Tokenizer Class: org.apache.solr.analysis.TrieTokenizerFactory
Docs: 118496
Distinct: 11477
Top Terms
term	frequency
20090721000000	655
20031110000000	500
20061108000000	428
20100129000000	412
20060320000000	356
20030620000000	354
20100415000000	313
20100430000000	310
20100301000000	296
20081101120000	260

and this for validFrom

Field Type: long
Properties: Indexed, Tokenized, Stored, Omit Norms, undefined
Schema: Indexed, Tokenized, Stored, Omit Norms, undefined
Index: Stored, Omit Norms, Binary
Index Analyzer: org.apache.solr.analysis.TokenizerChain Details
Tokenizer Class: org.apache.solr.analysis.TrieTokenizerFactory
Query Analyzer: org.apache.solr.analysis.TokenizerChain Details
Tokenizer Class: org.apache.solr.analysis.TrieTokenizerFactory
Docs: 111762
Distinct: 66649
Top Terms
term	frequency
20021017000000	315
20031110000000	309
20021021000000	293
20090423120000	258
20060320152000	229
20050607000000	227
20100415000000	215
20070101000000	207
20050610000000	205
20100129000000	200



i have checked all of our fields, with some of them sort works and with some of them it doesn't work. this is our finding:

(+ means it works, - it doesn't work)

_aktenzeichen	+
_autor			-
_dienststelle	+
_dokumententyp	+
_erstelldatum	-
_hauptthema		-
_kurzbeschreibung -
kurzinfoGruppe	-
lastChanged		+
objClass		-
objType			-
publicationHinweis	-
publicationNavigationstitel 	-
publicationStichwort	-
_stelle			+
_stichwort		-
_unterthema		-
title			+
validFrom		+
validUntil		-
_verteiler		- 
_vertraulich	-
_zielgruppen	+
__dst			+ (all fields but not _stelle)
__intern		-
__lokal			+ (all fields but not _stelle)
__cdate			-
__cyear			-
__source		-
__doctype		+
__mikronav		-

what can lead to this problem?


we have the following fields defined in our schema.xml

<!-- RZF isys -->
   <field name="_aktenzeichen" type="string" indexed="true" stored="true" />
   <field name="_anlagedoc" type="string" indexed="false" stored="false" />
   <field name="_autor" type="textgen" indexed="true" stored="true" />
   <field name="_dienststelle" type="string" indexed="true" stored="true" />
   <field name="_dokumententyp" type="string" indexed="true" stored="true" />
   <field name="_erstelldatum" type="long" indexed="true" stored="true" />
   <field name="_hauptthema" type="text_de" indexed="true" stored="true" />
   <field name="_kurzbeschreibung" type="text_de" indexed="true" stored="true" />
   <field name="kurzinfoGruppe" type="long" indexed="true" stored="true" mulitValued="true"/>
   <field name="lastChanged" type="long" indexed="true" stored="true" />
   <field name="objClass" type="string" indexed="true" stored="true" />
   <field name="objType" type="string" indexed="true" stored="true" />
   <field name="parent" type="long" indexed="false" stored="false" />
   <field name="path" type="string" indexed="false" stored="true" />
   <field name="publicationHinweis" type="text_de" indexed="true" stored="true" />
   <field name="publicationNavigationstitel" type="string" indexed="true" stored="true" />
   <field name="publicationStichwort" type="text_de" indexed="true" stored="true" />
   <field name="_stelle" type="string" indexed="true" stored="true" />
   <field name="_stichwort" type="text_de" indexed="true" stored="true" termVectors="true"/>
   <field name="_unterthema" type="text_de" indexed="true" stored="true" />
   <field name="title" type="text_de" indexed="true" stored="true" />
   <field name="validFrom" type="long" indexed="true" stored="true" />
   <field name="validUntil" type="long" indexed="true" stored="true" />
   <field name="_verteiler" type="text_de" indexed="true" stored="true" />
   <field name="_vertraulich" type="text_de" indexed="true" stored="true" />
   <field name="visiblePath" type="string" indexed="false" stored="true" />
   <field name="jurislinkUrl" type="string" indexed="false" stored="true" />
   <field name="destinationUrl" type="string" indexed="false" stored="true" />
   <field name="_zielgruppen" type="string" indexed="true" stored="true" multiValued="true"/>
   <field name="body" type="text_de" indexed="true" stored="true" />
<!-- berechnete werte -->
   <field name="__dst" type="string" indexed="true" stored="true" multiValued="true"/>
   <field name="__intern" type="string" indexed="true" stored="true"/>
   <field name="__lokal" type="string" indexed="true" stored="true"/>
        <field name="__cdate" type="date" indexed="true" stored="true" />
   <field name="__cyear" type="int" indexed="true" stored="true" />
   <field name="__source" type="string" indexed="true" stored="true" />
   <field name="__doctype" type="string" indexed="true" stored="true" />
   <field name="__mikronav" type="string" indexed="true" stored="true" multiValued="true"/>
<!-- /RZF isys -->

RE: strange behaviour when sorting, fields are missing in result

Posted by "Nagelberg, Kallin" <KN...@globeandmail.com>.
I'm not sure I understand how your results are truncated. They both find 21502 documents. The fact that you are sorting on '_erstelldatum' ascending and not seeing any results for that field on the first page leads me to think that you have 'sortMissingLast="false"' on that field's fieldType. In that case it would put all the documents missing the '_erstelldatum' first. 

-Kallin Nagelberg



-----Original Message-----
From: Markus.Rietzler@rzf.fin-nrw.de [mailto:Markus.Rietzler@rzf.fin-nrw.de] 
Sent: Wednesday, May 12, 2010 9:00 AM
To: solr-user@lucene.apache.org
Subject: strange behaviour when sorting, fields are missing in result

when i do a search, eg. 

http://xxx:8983/solr/select?q=steuer&fl=score,id,__intern,title,__source,_dienststelle,_erstelldatum,__cyear,_stelle

i get a normal result, like

<result name="response" numFound="21502" start="0" maxScore="1.3633566">
<doc>
<float name="score">1.3633566</float>
<int name="__cyear">2009</int>
<str name="__intern">0</str>
<str name="__source">zzz</str>
<str name="_dienststelle">xyz</str>
<long name="_erstelldatum">20090202000000</long>
<str name="_stelle">Presse- u. Informationsreferat</str>
<str name="id">34931684</str>
<str name="title">Merkblatt Vereine und Steuern</str>
</doc>

when i do a search with the sort param, my result is suddenly truncated:

http://xxx:8983/solr/select?q=steuer&fl=score,id,__intern,title,__source,_dienststelle,_erstelldatum,__cyear,_stelle&sort=_erstelldatum+asc

<result name="response" numFound="21502" start="0" maxScore="1.3633566">
<doc>
<float name="score">0.14290115</float>
<str name="__intern">0</str>
<str name="__source">isys</str>
<str name="id">18205520</str>
<str name="title">Amtsübersicht </str>
</doc>

so, not all of the fields from fl-param are displayed. this is what admin schema browser says about _erstelldatum:

Field Type: long
Properties: Indexed, Tokenized, Stored, Omit Norms, undefined
Schema: Indexed, Tokenized, Stored, Omit Norms, undefined
Index: Stored, Omit Norms, Binary
Index Analyzer: org.apache.solr.analysis.TokenizerChain Details
Tokenizer Class: org.apache.solr.analysis.TrieTokenizerFactory
Query Analyzer: org.apache.solr.analysis.TokenizerChain Details
Tokenizer Class: org.apache.solr.analysis.TrieTokenizerFactory
Docs: 118496
Distinct: 11477
Top Terms
term	frequency
20090721000000	655
20031110000000	500
20061108000000	428
20100129000000	412
20060320000000	356
20030620000000	354
20100415000000	313
20100430000000	310
20100301000000	296
20081101120000	260

and this for validFrom

Field Type: long
Properties: Indexed, Tokenized, Stored, Omit Norms, undefined
Schema: Indexed, Tokenized, Stored, Omit Norms, undefined
Index: Stored, Omit Norms, Binary
Index Analyzer: org.apache.solr.analysis.TokenizerChain Details
Tokenizer Class: org.apache.solr.analysis.TrieTokenizerFactory
Query Analyzer: org.apache.solr.analysis.TokenizerChain Details
Tokenizer Class: org.apache.solr.analysis.TrieTokenizerFactory
Docs: 111762
Distinct: 66649
Top Terms
term	frequency
20021017000000	315
20031110000000	309
20021021000000	293
20090423120000	258
20060320152000	229
20050607000000	227
20100415000000	215
20070101000000	207
20050610000000	205
20100129000000	200



i have checked all of our fields, with some of them sort works and with some of them it doesn't work. this is our finding:

(+ means it works, - it doesn't work)

_aktenzeichen	+
_autor			-
_dienststelle	+
_dokumententyp	+
_erstelldatum	-
_hauptthema		-
_kurzbeschreibung -
kurzinfoGruppe	-
lastChanged		+
objClass		-
objType			-
publicationHinweis	-
publicationNavigationstitel 	-
publicationStichwort	-
_stelle			+
_stichwort		-
_unterthema		-
title			+
validFrom		+
validUntil		-
_verteiler		- 
_vertraulich	-
_zielgruppen	+
__dst			+ (all fields but not _stelle)
__intern		-
__lokal			+ (all fields but not _stelle)
__cdate			-
__cyear			-
__source		-
__doctype		+
__mikronav		-

what can lead to this problem?


we have the following fields defined in our schema.xml

<!-- RZF isys -->
   <field name="_aktenzeichen" type="string" indexed="true" stored="true" />
   <field name="_anlagedoc" type="string" indexed="false" stored="false" />
   <field name="_autor" type="textgen" indexed="true" stored="true" />
   <field name="_dienststelle" type="string" indexed="true" stored="true" />
   <field name="_dokumententyp" type="string" indexed="true" stored="true" />
   <field name="_erstelldatum" type="long" indexed="true" stored="true" />
   <field name="_hauptthema" type="text_de" indexed="true" stored="true" />
   <field name="_kurzbeschreibung" type="text_de" indexed="true" stored="true" />
   <field name="kurzinfoGruppe" type="long" indexed="true" stored="true" mulitValued="true"/>
   <field name="lastChanged" type="long" indexed="true" stored="true" />
   <field name="objClass" type="string" indexed="true" stored="true" />
   <field name="objType" type="string" indexed="true" stored="true" />
   <field name="parent" type="long" indexed="false" stored="false" />
   <field name="path" type="string" indexed="false" stored="true" />
   <field name="publicationHinweis" type="text_de" indexed="true" stored="true" />
   <field name="publicationNavigationstitel" type="string" indexed="true" stored="true" />
   <field name="publicationStichwort" type="text_de" indexed="true" stored="true" />
   <field name="_stelle" type="string" indexed="true" stored="true" />
   <field name="_stichwort" type="text_de" indexed="true" stored="true" termVectors="true"/>
   <field name="_unterthema" type="text_de" indexed="true" stored="true" />
   <field name="title" type="text_de" indexed="true" stored="true" />
   <field name="validFrom" type="long" indexed="true" stored="true" />
   <field name="validUntil" type="long" indexed="true" stored="true" />
   <field name="_verteiler" type="text_de" indexed="true" stored="true" />
   <field name="_vertraulich" type="text_de" indexed="true" stored="true" />
   <field name="visiblePath" type="string" indexed="false" stored="true" />
   <field name="jurislinkUrl" type="string" indexed="false" stored="true" />
   <field name="destinationUrl" type="string" indexed="false" stored="true" />
   <field name="_zielgruppen" type="string" indexed="true" stored="true" multiValued="true"/>
   <field name="body" type="text_de" indexed="true" stored="true" />
<!-- berechnete werte -->
   <field name="__dst" type="string" indexed="true" stored="true" multiValued="true"/>
   <field name="__intern" type="string" indexed="true" stored="true"/>
   <field name="__lokal" type="string" indexed="true" stored="true"/>
        <field name="__cdate" type="date" indexed="true" stored="true" />
   <field name="__cyear" type="int" indexed="true" stored="true" />
   <field name="__source" type="string" indexed="true" stored="true" />
   <field name="__doctype" type="string" indexed="true" stored="true" />
   <field name="__mikronav" type="string" indexed="true" stored="true" multiValued="true"/>
<!-- /RZF isys -->

AW: strange behaviour when sorting, fields are missing in result (SOLVED)

Posted by Ma...@rzf.fin-nrw.de.
sorry, the problem was sitting in front of the monitor!

it is not an error or something, i forgot that some documents didn't have all fields filled,
so its absolutly normal, that not all fields were in the result.


markus

 

> -----Ursprüngliche Nachricht-----
> Von: Markus.Rietzler@rzf.fin-nrw.de 
> [mailto:Markus.Rietzler@rzf.fin-nrw.de] 
> Gesendet: Mittwoch, 12. Mai 2010 15:00
> An: solr-user@lucene.apache.org
> Betreff: strange behaviour when sorting, fields are missing in result
> 
> when i do a search, eg. 
> 
> http://xxx:8983/solr/select?q=steuer&fl=score,id,__intern,titl
> e,__source,_dienststelle,_erstelldatum,__cyear,_stelle
> 
> i get a normal result, like
> 
> <result name="response" numFound="21502" start="0" 
> maxScore="1.3633566">
> <doc>
> <float name="score">1.3633566</float>
> <int name="__cyear">2009</int>
> <str name="__intern">0</str>
> <str name="__source">zzz</str>
> <str name="_dienststelle">xyz</str>
> <long name="_erstelldatum">20090202000000</long>
> <str name="_stelle">Presse- u. Informationsreferat</str>
> <str name="id">34931684</str>
> <str name="title">Merkblatt Vereine und Steuern</str>
> </doc>
> 
> when i do a search with the sort param, my result is suddenly 
> truncated:
> 
> http://xxx:8983/solr/select?q=steuer&fl=score,id,__intern,titl
> e,__source,_dienststelle,_erstelldatum,__cyear,_stelle&sort=_e
> rstelldatum+asc
> 
> <result name="response" numFound="21502" start="0" 
> maxScore="1.3633566">
> <doc>
> <float name="score">0.14290115</float>
> <str name="__intern">0</str>
> <str name="__source">isys</str>
> <str name="id">18205520</str>
> <str name="title">Amtsübersicht </str>
> </doc>
> 
> so, not all of the fields from fl-param are displayed. this 
> is what admin schema browser says about _erstelldatum:
> 
> Field Type: long
> Properties: Indexed, Tokenized, Stored, Omit Norms, undefined
> Schema: Indexed, Tokenized, Stored, Omit Norms, undefined
> Index: Stored, Omit Norms, Binary
> Index Analyzer: org.apache.solr.analysis.TokenizerChain Details
> Tokenizer Class: org.apache.solr.analysis.TrieTokenizerFactory
> Query Analyzer: org.apache.solr.analysis.TokenizerChain Details
> Tokenizer Class: org.apache.solr.analysis.TrieTokenizerFactory
> Docs: 118496
> Distinct: 11477
> Top Terms
> term	frequency
> 20090721000000	655
> 20031110000000	500
> 20061108000000	428
> 20100129000000	412
> 20060320000000	356
> 20030620000000	354
> 20100415000000	313
> 20100430000000	310
> 20100301000000	296
> 20081101120000	260
> 
> and this for validFrom
> 
> Field Type: long
> Properties: Indexed, Tokenized, Stored, Omit Norms, undefined
> Schema: Indexed, Tokenized, Stored, Omit Norms, undefined
> Index: Stored, Omit Norms, Binary
> Index Analyzer: org.apache.solr.analysis.TokenizerChain Details
> Tokenizer Class: org.apache.solr.analysis.TrieTokenizerFactory
> Query Analyzer: org.apache.solr.analysis.TokenizerChain Details
> Tokenizer Class: org.apache.solr.analysis.TrieTokenizerFactory
> Docs: 111762
> Distinct: 66649
> Top Terms
> term	frequency
> 20021017000000	315
> 20031110000000	309
> 20021021000000	293
> 20090423120000	258
> 20060320152000	229
> 20050607000000	227
> 20100415000000	215
> 20070101000000	207
> 20050610000000	205
> 20100129000000	200
> 
> 
> 
> i have checked all of our fields, with some of them sort 
> works and with some of them it doesn't work. this is our finding:
> 
> (+ means it works, - it doesn't work)
> 
> _aktenzeichen	+
> _autor			-
> _dienststelle	+
> _dokumententyp	+
> _erstelldatum	-
> _hauptthema		-
> _kurzbeschreibung -
> kurzinfoGruppe	-
> lastChanged		+
> objClass		-
> objType			-
> publicationHinweis	-
> publicationNavigationstitel 	-
> publicationStichwort	-
> _stelle			+
> _stichwort		-
> _unterthema		-
> title			+
> validFrom		+
> validUntil		-
> _verteiler		- 
> _vertraulich	-
> _zielgruppen	+
> __dst			+ (all fields but not _stelle)
> __intern		-
> __lokal			+ (all fields but not _stelle)
> __cdate			-
> __cyear			-
> __source		-
> __doctype		+
> __mikronav		-
> 
> what can lead to this problem?
> 
> 
> we have the following fields defined in our schema.xml
> 
> <!-- RZF isys -->
>    <field name="_aktenzeichen" type="string" indexed="true" 
> stored="true" />
>    <field name="_anlagedoc" type="string" indexed="false" 
> stored="false" />
>    <field name="_autor" type="textgen" indexed="true" stored="true" />
>    <field name="_dienststelle" type="string" indexed="true" 
> stored="true" />
>    <field name="_dokumententyp" type="string" indexed="true" 
> stored="true" />
>    <field name="_erstelldatum" type="long" indexed="true" 
> stored="true" />
>    <field name="_hauptthema" type="text_de" indexed="true" 
> stored="true" />
>    <field name="_kurzbeschreibung" type="text_de" 
> indexed="true" stored="true" />
>    <field name="kurzinfoGruppe" type="long" indexed="true" 
> stored="true" mulitValued="true"/>
>    <field name="lastChanged" type="long" indexed="true" 
> stored="true" />
>    <field name="objClass" type="string" indexed="true" 
> stored="true" />
>    <field name="objType" type="string" indexed="true" stored="true" />
>    <field name="parent" type="long" indexed="false" stored="false" />
>    <field name="path" type="string" indexed="false" stored="true" />
>    <field name="publicationHinweis" type="text_de" 
> indexed="true" stored="true" />
>    <field name="publicationNavigationstitel" type="string" 
> indexed="true" stored="true" />
>    <field name="publicationStichwort" type="text_de" 
> indexed="true" stored="true" />
>    <field name="_stelle" type="string" indexed="true" stored="true" />
>    <field name="_stichwort" type="text_de" indexed="true" 
> stored="true" termVectors="true"/>
>    <field name="_unterthema" type="text_de" indexed="true" 
> stored="true" />
>    <field name="title" type="text_de" indexed="true" stored="true" />
>    <field name="validFrom" type="long" indexed="true" stored="true" />
>    <field name="validUntil" type="long" indexed="true" 
> stored="true" />
>    <field name="_verteiler" type="text_de" indexed="true" 
> stored="true" />
>    <field name="_vertraulich" type="text_de" indexed="true" 
> stored="true" />
>    <field name="visiblePath" type="string" indexed="false" 
> stored="true" />
>    <field name="jurislinkUrl" type="string" indexed="false" 
> stored="true" />
>    <field name="destinationUrl" type="string" indexed="false" 
> stored="true" />
>    <field name="_zielgruppen" type="string" indexed="true" 
> stored="true" multiValued="true"/>
>    <field name="body" type="text_de" indexed="true" stored="true" />
> <!-- berechnete werte -->
>    <field name="__dst" type="string" indexed="true" 
> stored="true" multiValued="true"/>
>    <field name="__intern" type="string" indexed="true" stored="true"/>
>    <field name="__lokal" type="string" indexed="true" stored="true"/>
>         <field name="__cdate" type="date" indexed="true" 
> stored="true" />
>    <field name="__cyear" type="int" indexed="true" stored="true" />
>    <field name="__source" type="string" indexed="true" 
> stored="true" />
>    <field name="__doctype" type="string" indexed="true" 
> stored="true" />
>    <field name="__mikronav" type="string" indexed="true" 
> stored="true" multiValued="true"/>
> <!-- /RZF isys -->
>