You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-user@lucene.apache.org by Ma...@rzf.fin-nrw.de on 2010/05/12 15:00:26 UTC
strange behaviour when sorting, fields are missing in result
when i do a search, eg.
http://xxx:8983/solr/select?q=steuer&fl=score,id,__intern,title,__source,_dienststelle,_erstelldatum,__cyear,_stelle
i get a normal result, like
<result name="response" numFound="21502" start="0" maxScore="1.3633566">
<doc>
<float name="score">1.3633566</float>
<int name="__cyear">2009</int>
<str name="__intern">0</str>
<str name="__source">zzz</str>
<str name="_dienststelle">xyz</str>
<long name="_erstelldatum">20090202000000</long>
<str name="_stelle">Presse- u. Informationsreferat</str>
<str name="id">34931684</str>
<str name="title">Merkblatt Vereine und Steuern</str>
</doc>
when i do a search with the sort param, my result is suddenly truncated:
http://xxx:8983/solr/select?q=steuer&fl=score,id,__intern,title,__source,_dienststelle,_erstelldatum,__cyear,_stelle&sort=_erstelldatum+asc
<result name="response" numFound="21502" start="0" maxScore="1.3633566">
<doc>
<float name="score">0.14290115</float>
<str name="__intern">0</str>
<str name="__source">isys</str>
<str name="id">18205520</str>
<str name="title">Amtsübersicht </str>
</doc>
so, not all of the fields from fl-param are displayed. this is what admin schema browser says about _erstelldatum:
Field Type: long
Properties: Indexed, Tokenized, Stored, Omit Norms, undefined
Schema: Indexed, Tokenized, Stored, Omit Norms, undefined
Index: Stored, Omit Norms, Binary
Index Analyzer: org.apache.solr.analysis.TokenizerChain Details
Tokenizer Class: org.apache.solr.analysis.TrieTokenizerFactory
Query Analyzer: org.apache.solr.analysis.TokenizerChain Details
Tokenizer Class: org.apache.solr.analysis.TrieTokenizerFactory
Docs: 118496
Distinct: 11477
Top Terms
term frequency
20090721000000 655
20031110000000 500
20061108000000 428
20100129000000 412
20060320000000 356
20030620000000 354
20100415000000 313
20100430000000 310
20100301000000 296
20081101120000 260
and this for validFrom
Field Type: long
Properties: Indexed, Tokenized, Stored, Omit Norms, undefined
Schema: Indexed, Tokenized, Stored, Omit Norms, undefined
Index: Stored, Omit Norms, Binary
Index Analyzer: org.apache.solr.analysis.TokenizerChain Details
Tokenizer Class: org.apache.solr.analysis.TrieTokenizerFactory
Query Analyzer: org.apache.solr.analysis.TokenizerChain Details
Tokenizer Class: org.apache.solr.analysis.TrieTokenizerFactory
Docs: 111762
Distinct: 66649
Top Terms
term frequency
20021017000000 315
20031110000000 309
20021021000000 293
20090423120000 258
20060320152000 229
20050607000000 227
20100415000000 215
20070101000000 207
20050610000000 205
20100129000000 200
i have checked all of our fields, with some of them sort works and with some of them it doesn't work. this is our finding:
(+ means it works, - it doesn't work)
_aktenzeichen +
_autor -
_dienststelle +
_dokumententyp +
_erstelldatum -
_hauptthema -
_kurzbeschreibung -
kurzinfoGruppe -
lastChanged +
objClass -
objType -
publicationHinweis -
publicationNavigationstitel -
publicationStichwort -
_stelle +
_stichwort -
_unterthema -
title +
validFrom +
validUntil -
_verteiler -
_vertraulich -
_zielgruppen +
__dst + (all fields but not _stelle)
__intern -
__lokal + (all fields but not _stelle)
__cdate -
__cyear -
__source -
__doctype +
__mikronav -
what can lead to this problem?
we have the following fields defined in our schema.xml
<!-- RZF isys -->
<field name="_aktenzeichen" type="string" indexed="true" stored="true" />
<field name="_anlagedoc" type="string" indexed="false" stored="false" />
<field name="_autor" type="textgen" indexed="true" stored="true" />
<field name="_dienststelle" type="string" indexed="true" stored="true" />
<field name="_dokumententyp" type="string" indexed="true" stored="true" />
<field name="_erstelldatum" type="long" indexed="true" stored="true" />
<field name="_hauptthema" type="text_de" indexed="true" stored="true" />
<field name="_kurzbeschreibung" type="text_de" indexed="true" stored="true" />
<field name="kurzinfoGruppe" type="long" indexed="true" stored="true" mulitValued="true"/>
<field name="lastChanged" type="long" indexed="true" stored="true" />
<field name="objClass" type="string" indexed="true" stored="true" />
<field name="objType" type="string" indexed="true" stored="true" />
<field name="parent" type="long" indexed="false" stored="false" />
<field name="path" type="string" indexed="false" stored="true" />
<field name="publicationHinweis" type="text_de" indexed="true" stored="true" />
<field name="publicationNavigationstitel" type="string" indexed="true" stored="true" />
<field name="publicationStichwort" type="text_de" indexed="true" stored="true" />
<field name="_stelle" type="string" indexed="true" stored="true" />
<field name="_stichwort" type="text_de" indexed="true" stored="true" termVectors="true"/>
<field name="_unterthema" type="text_de" indexed="true" stored="true" />
<field name="title" type="text_de" indexed="true" stored="true" />
<field name="validFrom" type="long" indexed="true" stored="true" />
<field name="validUntil" type="long" indexed="true" stored="true" />
<field name="_verteiler" type="text_de" indexed="true" stored="true" />
<field name="_vertraulich" type="text_de" indexed="true" stored="true" />
<field name="visiblePath" type="string" indexed="false" stored="true" />
<field name="jurislinkUrl" type="string" indexed="false" stored="true" />
<field name="destinationUrl" type="string" indexed="false" stored="true" />
<field name="_zielgruppen" type="string" indexed="true" stored="true" multiValued="true"/>
<field name="body" type="text_de" indexed="true" stored="true" />
<!-- berechnete werte -->
<field name="__dst" type="string" indexed="true" stored="true" multiValued="true"/>
<field name="__intern" type="string" indexed="true" stored="true"/>
<field name="__lokal" type="string" indexed="true" stored="true"/>
<field name="__cdate" type="date" indexed="true" stored="true" />
<field name="__cyear" type="int" indexed="true" stored="true" />
<field name="__source" type="string" indexed="true" stored="true" />
<field name="__doctype" type="string" indexed="true" stored="true" />
<field name="__mikronav" type="string" indexed="true" stored="true" multiValued="true"/>
<!-- /RZF isys -->
RE: strange behaviour when sorting, fields are missing in result
Posted by "Nagelberg, Kallin" <KN...@globeandmail.com>.
I'm not sure I understand how your results are truncated. They both find 21502 documents. The fact that you are sorting on '_erstelldatum' ascending and not seeing any results for that field on the first page leads me to think that you have 'sortMissingLast="false"' on that field's fieldType. In that case it would put all the documents missing the '_erstelldatum' first.
-Kallin Nagelberg
-----Original Message-----
From: Markus.Rietzler@rzf.fin-nrw.de [mailto:Markus.Rietzler@rzf.fin-nrw.de]
Sent: Wednesday, May 12, 2010 9:00 AM
To: solr-user@lucene.apache.org
Subject: strange behaviour when sorting, fields are missing in result
when i do a search, eg.
http://xxx:8983/solr/select?q=steuer&fl=score,id,__intern,title,__source,_dienststelle,_erstelldatum,__cyear,_stelle
i get a normal result, like
<result name="response" numFound="21502" start="0" maxScore="1.3633566">
<doc>
<float name="score">1.3633566</float>
<int name="__cyear">2009</int>
<str name="__intern">0</str>
<str name="__source">zzz</str>
<str name="_dienststelle">xyz</str>
<long name="_erstelldatum">20090202000000</long>
<str name="_stelle">Presse- u. Informationsreferat</str>
<str name="id">34931684</str>
<str name="title">Merkblatt Vereine und Steuern</str>
</doc>
when i do a search with the sort param, my result is suddenly truncated:
http://xxx:8983/solr/select?q=steuer&fl=score,id,__intern,title,__source,_dienststelle,_erstelldatum,__cyear,_stelle&sort=_erstelldatum+asc
<result name="response" numFound="21502" start="0" maxScore="1.3633566">
<doc>
<float name="score">0.14290115</float>
<str name="__intern">0</str>
<str name="__source">isys</str>
<str name="id">18205520</str>
<str name="title">Amtsübersicht </str>
</doc>
so, not all of the fields from fl-param are displayed. this is what admin schema browser says about _erstelldatum:
Field Type: long
Properties: Indexed, Tokenized, Stored, Omit Norms, undefined
Schema: Indexed, Tokenized, Stored, Omit Norms, undefined
Index: Stored, Omit Norms, Binary
Index Analyzer: org.apache.solr.analysis.TokenizerChain Details
Tokenizer Class: org.apache.solr.analysis.TrieTokenizerFactory
Query Analyzer: org.apache.solr.analysis.TokenizerChain Details
Tokenizer Class: org.apache.solr.analysis.TrieTokenizerFactory
Docs: 118496
Distinct: 11477
Top Terms
term frequency
20090721000000 655
20031110000000 500
20061108000000 428
20100129000000 412
20060320000000 356
20030620000000 354
20100415000000 313
20100430000000 310
20100301000000 296
20081101120000 260
and this for validFrom
Field Type: long
Properties: Indexed, Tokenized, Stored, Omit Norms, undefined
Schema: Indexed, Tokenized, Stored, Omit Norms, undefined
Index: Stored, Omit Norms, Binary
Index Analyzer: org.apache.solr.analysis.TokenizerChain Details
Tokenizer Class: org.apache.solr.analysis.TrieTokenizerFactory
Query Analyzer: org.apache.solr.analysis.TokenizerChain Details
Tokenizer Class: org.apache.solr.analysis.TrieTokenizerFactory
Docs: 111762
Distinct: 66649
Top Terms
term frequency
20021017000000 315
20031110000000 309
20021021000000 293
20090423120000 258
20060320152000 229
20050607000000 227
20100415000000 215
20070101000000 207
20050610000000 205
20100129000000 200
i have checked all of our fields, with some of them sort works and with some of them it doesn't work. this is our finding:
(+ means it works, - it doesn't work)
_aktenzeichen +
_autor -
_dienststelle +
_dokumententyp +
_erstelldatum -
_hauptthema -
_kurzbeschreibung -
kurzinfoGruppe -
lastChanged +
objClass -
objType -
publicationHinweis -
publicationNavigationstitel -
publicationStichwort -
_stelle +
_stichwort -
_unterthema -
title +
validFrom +
validUntil -
_verteiler -
_vertraulich -
_zielgruppen +
__dst + (all fields but not _stelle)
__intern -
__lokal + (all fields but not _stelle)
__cdate -
__cyear -
__source -
__doctype +
__mikronav -
what can lead to this problem?
we have the following fields defined in our schema.xml
<!-- RZF isys -->
<field name="_aktenzeichen" type="string" indexed="true" stored="true" />
<field name="_anlagedoc" type="string" indexed="false" stored="false" />
<field name="_autor" type="textgen" indexed="true" stored="true" />
<field name="_dienststelle" type="string" indexed="true" stored="true" />
<field name="_dokumententyp" type="string" indexed="true" stored="true" />
<field name="_erstelldatum" type="long" indexed="true" stored="true" />
<field name="_hauptthema" type="text_de" indexed="true" stored="true" />
<field name="_kurzbeschreibung" type="text_de" indexed="true" stored="true" />
<field name="kurzinfoGruppe" type="long" indexed="true" stored="true" mulitValued="true"/>
<field name="lastChanged" type="long" indexed="true" stored="true" />
<field name="objClass" type="string" indexed="true" stored="true" />
<field name="objType" type="string" indexed="true" stored="true" />
<field name="parent" type="long" indexed="false" stored="false" />
<field name="path" type="string" indexed="false" stored="true" />
<field name="publicationHinweis" type="text_de" indexed="true" stored="true" />
<field name="publicationNavigationstitel" type="string" indexed="true" stored="true" />
<field name="publicationStichwort" type="text_de" indexed="true" stored="true" />
<field name="_stelle" type="string" indexed="true" stored="true" />
<field name="_stichwort" type="text_de" indexed="true" stored="true" termVectors="true"/>
<field name="_unterthema" type="text_de" indexed="true" stored="true" />
<field name="title" type="text_de" indexed="true" stored="true" />
<field name="validFrom" type="long" indexed="true" stored="true" />
<field name="validUntil" type="long" indexed="true" stored="true" />
<field name="_verteiler" type="text_de" indexed="true" stored="true" />
<field name="_vertraulich" type="text_de" indexed="true" stored="true" />
<field name="visiblePath" type="string" indexed="false" stored="true" />
<field name="jurislinkUrl" type="string" indexed="false" stored="true" />
<field name="destinationUrl" type="string" indexed="false" stored="true" />
<field name="_zielgruppen" type="string" indexed="true" stored="true" multiValued="true"/>
<field name="body" type="text_de" indexed="true" stored="true" />
<!-- berechnete werte -->
<field name="__dst" type="string" indexed="true" stored="true" multiValued="true"/>
<field name="__intern" type="string" indexed="true" stored="true"/>
<field name="__lokal" type="string" indexed="true" stored="true"/>
<field name="__cdate" type="date" indexed="true" stored="true" />
<field name="__cyear" type="int" indexed="true" stored="true" />
<field name="__source" type="string" indexed="true" stored="true" />
<field name="__doctype" type="string" indexed="true" stored="true" />
<field name="__mikronav" type="string" indexed="true" stored="true" multiValued="true"/>
<!-- /RZF isys -->
AW: strange behaviour when sorting, fields are missing in result (SOLVED)
Posted by Ma...@rzf.fin-nrw.de.
sorry, the problem was sitting in front of the monitor!
it is not an error or something, i forgot that some documents didn't have all fields filled,
so its absolutly normal, that not all fields were in the result.
markus
> -----Ursprüngliche Nachricht-----
> Von: Markus.Rietzler@rzf.fin-nrw.de
> [mailto:Markus.Rietzler@rzf.fin-nrw.de]
> Gesendet: Mittwoch, 12. Mai 2010 15:00
> An: solr-user@lucene.apache.org
> Betreff: strange behaviour when sorting, fields are missing in result
>
> when i do a search, eg.
>
> http://xxx:8983/solr/select?q=steuer&fl=score,id,__intern,titl
> e,__source,_dienststelle,_erstelldatum,__cyear,_stelle
>
> i get a normal result, like
>
> <result name="response" numFound="21502" start="0"
> maxScore="1.3633566">
> <doc>
> <float name="score">1.3633566</float>
> <int name="__cyear">2009</int>
> <str name="__intern">0</str>
> <str name="__source">zzz</str>
> <str name="_dienststelle">xyz</str>
> <long name="_erstelldatum">20090202000000</long>
> <str name="_stelle">Presse- u. Informationsreferat</str>
> <str name="id">34931684</str>
> <str name="title">Merkblatt Vereine und Steuern</str>
> </doc>
>
> when i do a search with the sort param, my result is suddenly
> truncated:
>
> http://xxx:8983/solr/select?q=steuer&fl=score,id,__intern,titl
> e,__source,_dienststelle,_erstelldatum,__cyear,_stelle&sort=_e
> rstelldatum+asc
>
> <result name="response" numFound="21502" start="0"
> maxScore="1.3633566">
> <doc>
> <float name="score">0.14290115</float>
> <str name="__intern">0</str>
> <str name="__source">isys</str>
> <str name="id">18205520</str>
> <str name="title">Amtsübersicht </str>
> </doc>
>
> so, not all of the fields from fl-param are displayed. this
> is what admin schema browser says about _erstelldatum:
>
> Field Type: long
> Properties: Indexed, Tokenized, Stored, Omit Norms, undefined
> Schema: Indexed, Tokenized, Stored, Omit Norms, undefined
> Index: Stored, Omit Norms, Binary
> Index Analyzer: org.apache.solr.analysis.TokenizerChain Details
> Tokenizer Class: org.apache.solr.analysis.TrieTokenizerFactory
> Query Analyzer: org.apache.solr.analysis.TokenizerChain Details
> Tokenizer Class: org.apache.solr.analysis.TrieTokenizerFactory
> Docs: 118496
> Distinct: 11477
> Top Terms
> term frequency
> 20090721000000 655
> 20031110000000 500
> 20061108000000 428
> 20100129000000 412
> 20060320000000 356
> 20030620000000 354
> 20100415000000 313
> 20100430000000 310
> 20100301000000 296
> 20081101120000 260
>
> and this for validFrom
>
> Field Type: long
> Properties: Indexed, Tokenized, Stored, Omit Norms, undefined
> Schema: Indexed, Tokenized, Stored, Omit Norms, undefined
> Index: Stored, Omit Norms, Binary
> Index Analyzer: org.apache.solr.analysis.TokenizerChain Details
> Tokenizer Class: org.apache.solr.analysis.TrieTokenizerFactory
> Query Analyzer: org.apache.solr.analysis.TokenizerChain Details
> Tokenizer Class: org.apache.solr.analysis.TrieTokenizerFactory
> Docs: 111762
> Distinct: 66649
> Top Terms
> term frequency
> 20021017000000 315
> 20031110000000 309
> 20021021000000 293
> 20090423120000 258
> 20060320152000 229
> 20050607000000 227
> 20100415000000 215
> 20070101000000 207
> 20050610000000 205
> 20100129000000 200
>
>
>
> i have checked all of our fields, with some of them sort
> works and with some of them it doesn't work. this is our finding:
>
> (+ means it works, - it doesn't work)
>
> _aktenzeichen +
> _autor -
> _dienststelle +
> _dokumententyp +
> _erstelldatum -
> _hauptthema -
> _kurzbeschreibung -
> kurzinfoGruppe -
> lastChanged +
> objClass -
> objType -
> publicationHinweis -
> publicationNavigationstitel -
> publicationStichwort -
> _stelle +
> _stichwort -
> _unterthema -
> title +
> validFrom +
> validUntil -
> _verteiler -
> _vertraulich -
> _zielgruppen +
> __dst + (all fields but not _stelle)
> __intern -
> __lokal + (all fields but not _stelle)
> __cdate -
> __cyear -
> __source -
> __doctype +
> __mikronav -
>
> what can lead to this problem?
>
>
> we have the following fields defined in our schema.xml
>
> <!-- RZF isys -->
> <field name="_aktenzeichen" type="string" indexed="true"
> stored="true" />
> <field name="_anlagedoc" type="string" indexed="false"
> stored="false" />
> <field name="_autor" type="textgen" indexed="true" stored="true" />
> <field name="_dienststelle" type="string" indexed="true"
> stored="true" />
> <field name="_dokumententyp" type="string" indexed="true"
> stored="true" />
> <field name="_erstelldatum" type="long" indexed="true"
> stored="true" />
> <field name="_hauptthema" type="text_de" indexed="true"
> stored="true" />
> <field name="_kurzbeschreibung" type="text_de"
> indexed="true" stored="true" />
> <field name="kurzinfoGruppe" type="long" indexed="true"
> stored="true" mulitValued="true"/>
> <field name="lastChanged" type="long" indexed="true"
> stored="true" />
> <field name="objClass" type="string" indexed="true"
> stored="true" />
> <field name="objType" type="string" indexed="true" stored="true" />
> <field name="parent" type="long" indexed="false" stored="false" />
> <field name="path" type="string" indexed="false" stored="true" />
> <field name="publicationHinweis" type="text_de"
> indexed="true" stored="true" />
> <field name="publicationNavigationstitel" type="string"
> indexed="true" stored="true" />
> <field name="publicationStichwort" type="text_de"
> indexed="true" stored="true" />
> <field name="_stelle" type="string" indexed="true" stored="true" />
> <field name="_stichwort" type="text_de" indexed="true"
> stored="true" termVectors="true"/>
> <field name="_unterthema" type="text_de" indexed="true"
> stored="true" />
> <field name="title" type="text_de" indexed="true" stored="true" />
> <field name="validFrom" type="long" indexed="true" stored="true" />
> <field name="validUntil" type="long" indexed="true"
> stored="true" />
> <field name="_verteiler" type="text_de" indexed="true"
> stored="true" />
> <field name="_vertraulich" type="text_de" indexed="true"
> stored="true" />
> <field name="visiblePath" type="string" indexed="false"
> stored="true" />
> <field name="jurislinkUrl" type="string" indexed="false"
> stored="true" />
> <field name="destinationUrl" type="string" indexed="false"
> stored="true" />
> <field name="_zielgruppen" type="string" indexed="true"
> stored="true" multiValued="true"/>
> <field name="body" type="text_de" indexed="true" stored="true" />
> <!-- berechnete werte -->
> <field name="__dst" type="string" indexed="true"
> stored="true" multiValued="true"/>
> <field name="__intern" type="string" indexed="true" stored="true"/>
> <field name="__lokal" type="string" indexed="true" stored="true"/>
> <field name="__cdate" type="date" indexed="true"
> stored="true" />
> <field name="__cyear" type="int" indexed="true" stored="true" />
> <field name="__source" type="string" indexed="true"
> stored="true" />
> <field name="__doctype" type="string" indexed="true"
> stored="true" />
> <field name="__mikronav" type="string" indexed="true"
> stored="true" multiValued="true"/>
> <!-- /RZF isys -->
>