You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-user@lucene.apache.org by mpcmarcos <mp...@gmail.com> on 2012/08/28 11:02:15 UTC
Query Time problem on Big Index Solr 3.5
Hello,
I have a problem, I'm working with Solr 3.5, with a index that has 8.000.000
of documents (13Gb), each document has a lot of fields, I include the schema
at bottom the message for more information.
The query time is very high, a simple query has a query time of 300-1.000
ms, and a complex query to 10.000 ms. I have a master, and 6 slaves, they
are been syncronized every 10 minutes. And the index always is optimized.
What can I do?
- I think that cache system is working ok, when I do the same query two
times, the query time decrease to 0 ms.
Here a example of query, there are any incorrect o anything that can I
change?
http://xxx:8893/solr/candidate/select/?q=+(IdCandidateStatus:2)+(IdCobranded:3)+(IdLocation1:12))+(LastLoginDate:[2011-08-26T00:00:00Z
TO 2012-08-28T00:00:00Z])
*Schema:*
<field name="IdCandidate" type="slong" indexed="true" stored="true"
required="true" />
<field name="IdUser" type="slong" indexed="true" stored="true"
required="true" />
<field name="Email" type="string" indexed="true" stored="true"
required="true" />
<field name="Name" type="string" indexed="true" stored="true"
required="true" />
<field name="NameFormated" type="alphaOnlySort" indexed="true"
stored="true"/>
<field name="Surname" type="string" indexed="true" stored="true"
required="true" />
<field name="SurnameFormated" type="alphaOnlySort" indexed="true"
stored="true"/>
<field name="IdSex" type="string" indexed="true" stored="true"
required="true" />
<field name="IdWorkingHours" type="sint" indexed="true" stored="true"
required="true" />
<field name="IdContractWorkType" type="sint" indexed="true"
stored="true" required="true" />
<field name="IdLocation1" type="sint" indexed="true" stored="true"
required="true" />
<field name="IdLocation2" type="sint" indexed="true" stored="true"
required="true" />
<field name="Location2" type="string" indexed="true" stored="true"
required="true" />
<field name="IdLocation3" type="slong" indexed="true" stored="true"
required="true" />
<field name="IdLocation4" type="slong" indexed="true" stored="true"
required="true" />
<field name="Location4" type="string" indexed="true" stored="true"
required="true" />
<field name="IdLocation5" type="slong" indexed="true" stored="true"
required="true" />
<field name="Location5" type="string" indexed="true" stored="true"
required="true" />
<field name="IdRegion1" type="slong" indexed="true" stored="true"
required="true" />
<field name="Region1" type="string" indexed="true" stored="true"
required="true" />
<field name="IdRegion2" type="slong" indexed="true" stored="true"
required="true" />
<field name="Region2" type="string" indexed="true" stored="true"
required="true" />
<field name="LastLoginDate" type="tdate" indexed="true" stored="true"
required="true" />
<field name="BirthDate" type="tdate" indexed="true" stored="true"
required="true" />
<field name="InsertDate" type="tdate" indexed="true" stored="true"
required="true" />
<field name="ModifyDate" type="tdate" indexed="true" stored="true"
required="true" />
<field name="IdModifyRangeDate" type="sint" indexed="true" stored="true"
required="true" />
<field name="Age" type="sint" indexed="true" stored="true"
required="true" />
<field name="IdAgeRange" type="sint" indexed="true" stored="true"
required="true" />
<field name="Travel" type="sint" indexed="true" stored="true"
required="true" />
<field name="ChangeResidence" type="sint" indexed="true" stored="true"
required="true" />
<field name="IdEmployed" type="sint" indexed="true" stored="true"
required="true" />
<field name="SalaryMax" type="sdouble" indexed="true" stored="true"
required="true" />
<field name="SalaryMin" type="sdouble" indexed="true" stored="true"
required="true" />
<field name="IdSalaryRange" type="sint" indexed="true" stored="true"
required="true" />
<field name="IdPreferenceManagerialLevelMin" type="sint" indexed="true"
stored="true" required="true" />
<field name="IdPreferenceManagerialLevelMax" type="sint" indexed="true"
stored="true" required="true" />
<field name="IdStudie1Max" type="sint" indexed="true" stored="true"
required="true" />
<field name="IdCategory2Last" type="sint" indexed="true" stored="true"
required="true" />
<field name="Category2Last" type="string" indexed="true" stored="true"
required="true" />
<field name="Category2LastFormated" type="alphaOnlySort" indexed="true"
stored="true" />
<field name="IdCategory1Last" type="sint" indexed="true" stored="true"
required="true" />
<field name="IdExperienceTime" type="sint" indexed="true" stored="true"
required="true" />
<field name="IdExperienceRange" type="sint" indexed="true" stored="true"
required="true" />
<field name="IsDeficiency" type="string" indexed="true" stored="true"
required="true" />
<field name="IdStudie1" type="sint" indexed="true" stored="true"
required="false" multiValued="true" />
<field name="IdStudie2" type="sint" indexed="true" stored="true"
required="false" multiValued="true" />
<field name="IdStudie2Status" type="sint" indexed="true" stored="true"
required="false" multiValued="true" />
<field name="IdLanguage" type="sint" indexed="true" stored="true"
required="false" multiValued="true" />
<field name="IdKnowledge1" type="sint" indexed="true" stored="true"
required="false" multiValued="true" />
<field name="IdKnowledge2" type="sint" indexed="true" stored="true"
required="false" multiValued="true" />
<field name="IdDeficiency1" type="sint" indexed="true" stored="true"
required="false" multiValued="true" />
<field name="IdDeficiency2" type="sint" indexed="true" stored="true"
required="false" multiValued="true" />
<field name="IdCategory1" type="sint" indexed="true" stored="true"
required="false" multiValued="true" />
<field name="IdCategory2" type="sint" indexed="true" stored="true"
required="false" multiValued="true" />
<field name="IdCategory2Latest" type="sint" indexed="true" stored="true"
required="false" multiValued="true" />
<field name="IdPreferenceCategory1" type="sint" indexed="true"
stored="true" required="false" multiValued="true" />
<field name="IdPreferenceCategory2" type="sint" indexed="true"
stored="true" required="false" multiValued="true" />
<field name="IdLicense" type="sint" indexed="true" stored="true"
required="false" multiValued="true" />
<field name="IdVehicle" type="sint" indexed="true" stored="true"
required="false" multiValued="true" />
<field name="Phone1" type="string" indexed="true" stored="true"
required="true" />
<field name="Phone2" type="string" indexed="true" stored="true"
required="true" />
<field name="HasPhoto" type="string" indexed="true" stored="true"
required="true" />
<field name="Photo" type="string" indexed="true" stored="true"
required="true" />
<field name="StudieCentreLast" type="string" indexed="true"
stored="true" required="true" />
<field name="IdManagerialLevelMax" type="sint" indexed="true"
stored="true" required="true" />
<field name="IdManagerialLevel" type="sint" indexed="true" stored="true"
required="false" multiValued="true" />
<field name="CEP" type="string" indexed="true" stored="true"
required="true" />
<field name="Complement" type="string" indexed="true" stored="true"
required="true" />
<field name="DeficiencyInformation" type="string" indexed="true"
stored="true" required="true" />
<field name="Employed" type="string" indexed="true" stored="true"
required="true" />
<field name="HasWord" type="string" indexed="true" stored="true"
required="true" />
<field name="ContractWorkType" type="string" indexed="true"
stored="true" required="true" />
<field name="ExperienceRange" type="string" indexed="true" stored="true"
required="true" />
<field name="Location1" type="string" indexed="true" stored="true"
required="true" />
<field name="Location2Short" type="string" indexed="true" stored="true"
required="true" />
<field name="PreferenceManagerialLevelMin" type="string" indexed="true"
stored="true" required="true" />
<field name="PreferenceManagerialLevelMax" type="string" indexed="true"
stored="true" required="true" />
<field name="IdMaritalStatus" type="string" indexed="true" stored="true"
required="true" />
<field name="MaritalStatus" type="string" indexed="true" stored="true"
required="true" />
<field name="IdNationality" type="sint" indexed="true" stored="true"
required="true" />
<field name="Nationality" type="string" indexed="true" stored="true"
required="true" />
<field name="IdSalaryType" type="sint" indexed="true" stored="true"
required="true" />
<field name="SalaryType" type="string" indexed="true" stored="true"
required="true" />
<field name="Sex" type="string" indexed="true" stored="true"
required="true" />
<field name="Studie1Max" type="string" indexed="true" stored="true"
required="true" />
<field name="IdVisibility" type="string" indexed="true" stored="true"
required="true" />
<field name="WorkingHours" type="string" indexed="true" stored="true"
required="true" />
<field name="Number" type="sint" indexed="true" stored="true"
required="true" />
<field name="ProductCode" type="string" indexed="true" stored="true"
required="true" />
<field name="IdLanguageLevel" type="string" indexed="true" stored="true"
required="false" multiValued="true" />
<field name="Fti" type="text" indexed="true" stored="false"
required="true" />
<field name="Latitude" type="double" indexed="true" stored="true"
required="true" />
<field name="Longitude" type="double" indexed="true" stored="true"
required="true" />
<field name="lat" type="double" indexed="true" stored="true"
required="true" multiValued="false"/>
<field name="lng" type="double" indexed="true" stored="true"
required="true" multiValued="false"/>
<field name="location_latlng" type="location" indexed="true"
stored="true" required="true"/>
<field name="IdCobranded" type="sint" indexed="true" stored="true"
required="false" multiValued="true" />
<field name="IdCandidateStatus" type="sint" indexed="true" stored="true"
required="true" />
<field name="IdManagerialLevelLatest" type="sint" indexed="true"
stored="true" required="false" multiValued="true" />
<dynamicField name="*_coordinate" type="tdouble" indexed="true"
stored="false"/>
*Solconfig.xml:*
<luceneMatchVersion>LUCENE_35</luceneMatchVersion>
...
<directoryFactory name="DirectoryFactory"
class="${solr.directoryFactory:solr.StandardDirectoryFactory}"/>
<useCompoundFile>false</useCompoundFile>
<ramBufferSizeMB>32</ramBufferSizeMB>
<mergeFactor>10</mergeFactor>
<unlockOnStartup>false</unlockOnStartup>
<reopenReaders>true</reopenReaders>
...
<maxBooleanClauses>131072</maxBooleanClauses>
<filterCache class="solr.FastLRUCache" size="262144" initialSize="8048"
autowarmCount="8048"/><queryResultCache class="solr.LRUCache" size="262144"
initialSize="8048" autowarmCount="8048"/><documentCache
class="solr.LRUCache" size="262144" initialSize="8048"
autowarmCount="8048"/><enableLazyFieldLoading>true</enableLazyFieldLoading><queryResultWindowSize>60</queryResultWindowSize>
<queryResultMaxDocsCached>200</queryResultMaxDocsCached>
...
<useColdSearcher>false</useColdSearcher>
<maxWarmingSearchers>2</maxWarmingSearchers>
--
View this message in context: http://lucene.472066.n3.nabble.com/Query-Time-problem-on-Big-Index-Solr-3-5-tp4003660.html
Sent from the Solr - User mailing list archive at Nabble.com.
Re: Query Time problem on Big Index Solr 3.5
Posted by pravesh <su...@yahoo.com>.
How often the documents are added in the index? Try lessen down the optimize
frequency.
BTW do you optimize only on master (which should be the desired way).
Also specifically for dates ranges, try to use the filter queries, this way
it would be cached and would thus be faster. This would also apply to other
fields which require very less analysis or have limited unique fields.
Thanx
Pravesh
--
View this message in context: http://lucene.472066.n3.nabble.com/Query-Time-problem-on-Big-Index-Solr-3-5-tp4003660p4004437.html
Sent from the Solr - User mailing list archive at Nabble.com.
Re: Query Time problem on Big Index Solr 3.5
Posted by Jack Krupansky <ja...@basetechnology.com>.
I did notice that you use a lot of the sint/slong/sdouble field types
instead of the default int/long/double which are now the much more efficient
trie field types, even in Solr 3.5. I have no idea how much of your
lackluster performance is due to sint, et al, but I am sure that is a
contributing factor.
Unfortunately, you can't change those field types without necessitating a
full re-index.
I also noticed that you are using trie for your date fields. As an
experiment, try some queries that use only the date fields and compare the
time to queries that use only sint fields (but not as ranges). This might
give an indication of the kind of gain you could get with trie field types
for your numeric fields.
-- Jack Krupansky
-----Original Message-----
From: mpcmarcos
Sent: Tuesday, August 28, 2012 5:02 AM
To: solr-user@lucene.apache.org
Subject: Query Time problem on Big Index Solr 3.5
Hello,
I have a problem, I'm working with Solr 3.5, with a index that has 8.000.000
of documents (13Gb), each document has a lot of fields, I include the schema
at bottom the message for more information.
The query time is very high, a simple query has a query time of 300-1.000
ms, and a complex query to 10.000 ms. I have a master, and 6 slaves, they
are been syncronized every 10 minutes. And the index always is optimized.
What can I do?
- I think that cache system is working ok, when I do the same query two
times, the query time decrease to 0 ms.
Here a example of query, there are any incorrect o anything that can I
change?
http://xxx:8893/solr/candidate/select/?q=+(IdCandidateStatus:2)+(IdCobranded:3)+(IdLocation1:12))+(LastLoginDate:[2011-08-26T00:00:00Z
TO 2012-08-28T00:00:00Z])
*Schema:*
<field name="IdCandidate" type="slong" indexed="true" stored="true"
required="true" />
<field name="IdUser" type="slong" indexed="true" stored="true"
required="true" />
<field name="Email" type="string" indexed="true" stored="true"
required="true" />
<field name="Name" type="string" indexed="true" stored="true"
required="true" />
<field name="NameFormated" type="alphaOnlySort" indexed="true"
stored="true"/>
<field name="Surname" type="string" indexed="true" stored="true"
required="true" />
<field name="SurnameFormated" type="alphaOnlySort" indexed="true"
stored="true"/>
<field name="IdSex" type="string" indexed="true" stored="true"
required="true" />
<field name="IdWorkingHours" type="sint" indexed="true" stored="true"
required="true" />
<field name="IdContractWorkType" type="sint" indexed="true"
stored="true" required="true" />
<field name="IdLocation1" type="sint" indexed="true" stored="true"
required="true" />
<field name="IdLocation2" type="sint" indexed="true" stored="true"
required="true" />
<field name="Location2" type="string" indexed="true" stored="true"
required="true" />
<field name="IdLocation3" type="slong" indexed="true" stored="true"
required="true" />
<field name="IdLocation4" type="slong" indexed="true" stored="true"
required="true" />
<field name="Location4" type="string" indexed="true" stored="true"
required="true" />
<field name="IdLocation5" type="slong" indexed="true" stored="true"
required="true" />
<field name="Location5" type="string" indexed="true" stored="true"
required="true" />
<field name="IdRegion1" type="slong" indexed="true" stored="true"
required="true" />
<field name="Region1" type="string" indexed="true" stored="true"
required="true" />
<field name="IdRegion2" type="slong" indexed="true" stored="true"
required="true" />
<field name="Region2" type="string" indexed="true" stored="true"
required="true" />
<field name="LastLoginDate" type="tdate" indexed="true" stored="true"
required="true" />
<field name="BirthDate" type="tdate" indexed="true" stored="true"
required="true" />
<field name="InsertDate" type="tdate" indexed="true" stored="true"
required="true" />
<field name="ModifyDate" type="tdate" indexed="true" stored="true"
required="true" />
<field name="IdModifyRangeDate" type="sint" indexed="true" stored="true"
required="true" />
<field name="Age" type="sint" indexed="true" stored="true"
required="true" />
<field name="IdAgeRange" type="sint" indexed="true" stored="true"
required="true" />
<field name="Travel" type="sint" indexed="true" stored="true"
required="true" />
<field name="ChangeResidence" type="sint" indexed="true" stored="true"
required="true" />
<field name="IdEmployed" type="sint" indexed="true" stored="true"
required="true" />
<field name="SalaryMax" type="sdouble" indexed="true" stored="true"
required="true" />
<field name="SalaryMin" type="sdouble" indexed="true" stored="true"
required="true" />
<field name="IdSalaryRange" type="sint" indexed="true" stored="true"
required="true" />
<field name="IdPreferenceManagerialLevelMin" type="sint" indexed="true"
stored="true" required="true" />
<field name="IdPreferenceManagerialLevelMax" type="sint" indexed="true"
stored="true" required="true" />
<field name="IdStudie1Max" type="sint" indexed="true" stored="true"
required="true" />
<field name="IdCategory2Last" type="sint" indexed="true" stored="true"
required="true" />
<field name="Category2Last" type="string" indexed="true" stored="true"
required="true" />
<field name="Category2LastFormated" type="alphaOnlySort" indexed="true"
stored="true" />
<field name="IdCategory1Last" type="sint" indexed="true" stored="true"
required="true" />
<field name="IdExperienceTime" type="sint" indexed="true" stored="true"
required="true" />
<field name="IdExperienceRange" type="sint" indexed="true" stored="true"
required="true" />
<field name="IsDeficiency" type="string" indexed="true" stored="true"
required="true" />
<field name="IdStudie1" type="sint" indexed="true" stored="true"
required="false" multiValued="true" />
<field name="IdStudie2" type="sint" indexed="true" stored="true"
required="false" multiValued="true" />
<field name="IdStudie2Status" type="sint" indexed="true" stored="true"
required="false" multiValued="true" />
<field name="IdLanguage" type="sint" indexed="true" stored="true"
required="false" multiValued="true" />
<field name="IdKnowledge1" type="sint" indexed="true" stored="true"
required="false" multiValued="true" />
<field name="IdKnowledge2" type="sint" indexed="true" stored="true"
required="false" multiValued="true" />
<field name="IdDeficiency1" type="sint" indexed="true" stored="true"
required="false" multiValued="true" />
<field name="IdDeficiency2" type="sint" indexed="true" stored="true"
required="false" multiValued="true" />
<field name="IdCategory1" type="sint" indexed="true" stored="true"
required="false" multiValued="true" />
<field name="IdCategory2" type="sint" indexed="true" stored="true"
required="false" multiValued="true" />
<field name="IdCategory2Latest" type="sint" indexed="true" stored="true"
required="false" multiValued="true" />
<field name="IdPreferenceCategory1" type="sint" indexed="true"
stored="true" required="false" multiValued="true" />
<field name="IdPreferenceCategory2" type="sint" indexed="true"
stored="true" required="false" multiValued="true" />
<field name="IdLicense" type="sint" indexed="true" stored="true"
required="false" multiValued="true" />
<field name="IdVehicle" type="sint" indexed="true" stored="true"
required="false" multiValued="true" />
<field name="Phone1" type="string" indexed="true" stored="true"
required="true" />
<field name="Phone2" type="string" indexed="true" stored="true"
required="true" />
<field name="HasPhoto" type="string" indexed="true" stored="true"
required="true" />
<field name="Photo" type="string" indexed="true" stored="true"
required="true" />
<field name="StudieCentreLast" type="string" indexed="true"
stored="true" required="true" />
<field name="IdManagerialLevelMax" type="sint" indexed="true"
stored="true" required="true" />
<field name="IdManagerialLevel" type="sint" indexed="true" stored="true"
required="false" multiValued="true" />
<field name="CEP" type="string" indexed="true" stored="true"
required="true" />
<field name="Complement" type="string" indexed="true" stored="true"
required="true" />
<field name="DeficiencyInformation" type="string" indexed="true"
stored="true" required="true" />
<field name="Employed" type="string" indexed="true" stored="true"
required="true" />
<field name="HasWord" type="string" indexed="true" stored="true"
required="true" />
<field name="ContractWorkType" type="string" indexed="true"
stored="true" required="true" />
<field name="ExperienceRange" type="string" indexed="true" stored="true"
required="true" />
<field name="Location1" type="string" indexed="true" stored="true"
required="true" />
<field name="Location2Short" type="string" indexed="true" stored="true"
required="true" />
<field name="PreferenceManagerialLevelMin" type="string" indexed="true"
stored="true" required="true" />
<field name="PreferenceManagerialLevelMax" type="string" indexed="true"
stored="true" required="true" />
<field name="IdMaritalStatus" type="string" indexed="true" stored="true"
required="true" />
<field name="MaritalStatus" type="string" indexed="true" stored="true"
required="true" />
<field name="IdNationality" type="sint" indexed="true" stored="true"
required="true" />
<field name="Nationality" type="string" indexed="true" stored="true"
required="true" />
<field name="IdSalaryType" type="sint" indexed="true" stored="true"
required="true" />
<field name="SalaryType" type="string" indexed="true" stored="true"
required="true" />
<field name="Sex" type="string" indexed="true" stored="true"
required="true" />
<field name="Studie1Max" type="string" indexed="true" stored="true"
required="true" />
<field name="IdVisibility" type="string" indexed="true" stored="true"
required="true" />
<field name="WorkingHours" type="string" indexed="true" stored="true"
required="true" />
<field name="Number" type="sint" indexed="true" stored="true"
required="true" />
<field name="ProductCode" type="string" indexed="true" stored="true"
required="true" />
<field name="IdLanguageLevel" type="string" indexed="true" stored="true"
required="false" multiValued="true" />
<field name="Fti" type="text" indexed="true" stored="false"
required="true" />
<field name="Latitude" type="double" indexed="true" stored="true"
required="true" />
<field name="Longitude" type="double" indexed="true" stored="true"
required="true" />
<field name="lat" type="double" indexed="true" stored="true"
required="true" multiValued="false"/>
<field name="lng" type="double" indexed="true" stored="true"
required="true" multiValued="false"/>
<field name="location_latlng" type="location" indexed="true"
stored="true" required="true"/>
<field name="IdCobranded" type="sint" indexed="true" stored="true"
required="false" multiValued="true" />
<field name="IdCandidateStatus" type="sint" indexed="true" stored="true"
required="true" />
<field name="IdManagerialLevelLatest" type="sint" indexed="true"
stored="true" required="false" multiValued="true" />
<dynamicField name="*_coordinate" type="tdouble" indexed="true"
stored="false"/>
*Solconfig.xml:*
<luceneMatchVersion>LUCENE_35</luceneMatchVersion>
...
<directoryFactory name="DirectoryFactory"
class="${solr.directoryFactory:solr.StandardDirectoryFactory}"/>
<useCompoundFile>false</useCompoundFile>
<ramBufferSizeMB>32</ramBufferSizeMB>
<mergeFactor>10</mergeFactor>
<unlockOnStartup>false</unlockOnStartup>
<reopenReaders>true</reopenReaders>
...
<maxBooleanClauses>131072</maxBooleanClauses>
<filterCache class="solr.FastLRUCache" size="262144" initialSize="8048"
autowarmCount="8048"/><queryResultCache class="solr.LRUCache" size="262144"
initialSize="8048" autowarmCount="8048"/><documentCache
class="solr.LRUCache" size="262144" initialSize="8048"
autowarmCount="8048"/><enableLazyFieldLoading>true</enableLazyFieldLoading><queryResultWindowSize>60</queryResultWindowSize>
<queryResultMaxDocsCached>200</queryResultMaxDocsCached>
...
<useColdSearcher>false</useColdSearcher>
<maxWarmingSearchers>2</maxWarmingSearchers>
--
View this message in context:
http://lucene.472066.n3.nabble.com/Query-Time-problem-on-Big-Index-Solr-3-5-tp4003660.html
Sent from the Solr - User mailing list archive at Nabble.com.
Re: Query Time problem on Big Index Solr 3.5
Posted by Alexey Serba <as...@gmail.com>.
1. Use filter queries
> Here a example of query, there are any incorrect o anything that can I
> change?
> http://xxx:8893/solr/candidate/select/?q=+(IdCandidateStatus:2)+(IdCobranded:3)+(IdLocation1:12))+(LastLoginDate:[2011-08-26T00:00:00Z
> TO 2012-08-28T00:00:00Z])
What is the logic here? Are you AND-ing these boolean clauses? If yes,
then I would change queries to
http://xxx:8893/solr/candidate/select/?q=*:*&fq=IdCandidateStatus:2&fq=IdCobranded:3&fq=IdLocation1:12&fq=LastLoginDate:[2011-08-26T00:00:00Z
TO 2012-08-28T00:00:00Z]
I.e. move queries into fq (filter query) parameter.
* it should be faster as it seems you don't need score here. Sort by
id/date instead.
* fq-s will be cached separately thus increasing cache hit rate.
2. Do not optimize your index
> I have a master, and 6 slaves, they are been syncronized every 10 minutes. And the index always is optimized.
DO NOT optimize your index! (unless you re-create the whole index
completely every 10 mins). It basically kills the idea of replication
(after every optimize command slaves download the whole index).
Re: Query Time problem on Big Index Solr 3.5
Posted by pravesh <su...@yahoo.com>.
13 GB index isn't too big, but going forward index sharding is the solutions
for large single core indexes. This way you can scale out.
This links will give you some info:
http://lucidworks.lucidimagination.com/display/solr/Distributed+Search+with+Index+Sharding
http://lucidworks.lucidimagination.com/display/solr/Distributed+Search+with+Index+Sharding
Regards
Pravesh
--
View this message in context: http://lucene.472066.n3.nabble.com/Query-Time-problem-on-Big-Index-Solr-3-5-tp4003660p4004630.html
Sent from the Solr - User mailing list archive at Nabble.com.
Re: Query Time problem on Big Index Solr 3.5
Posted by Kiran Jayakumar <ki...@gmail.com>.
Have you tried sharding ? The best practices recommend sharding when your
index grows too large and query time is slow.
On Tue, Aug 28, 2012 at 2:02 AM, mpcmarcos <mp...@gmail.com> wrote:
> Hello,
>
> I have a problem, I'm working with Solr 3.5, with a index that has
> 8.000.000
> of documents (13Gb), each document has a lot of fields, I include the
> schema
> at bottom the message for more information.
>
> The query time is very high, a simple query has a query time of 300-1.000
> ms, and a complex query to 10.000 ms. I have a master, and 6 slaves, they
> are been syncronized every 10 minutes. And the index always is optimized.
>
> What can I do?
> - I think that cache system is working ok, when I do the same query two
> times, the query time decrease to 0 ms.
>
>
> Here a example of query, there are any incorrect o anything that can I
> change?
>
> http://xxx:8893/solr/candidate/select/?q=+(IdCandidateStatus:2)+(IdCobranded:3)+(IdLocation1:12))+(LastLoginDate:[2011-08-26T00:00:00Z
> TO 2012-08-28T00:00:00Z])
>
>
>
> *Schema:*
> <field name="IdCandidate" type="slong" indexed="true" stored="true"
> required="true" />
> <field name="IdUser" type="slong" indexed="true" stored="true"
> required="true" />
> <field name="Email" type="string" indexed="true" stored="true"
> required="true" />
> <field name="Name" type="string" indexed="true" stored="true"
> required="true" />
> <field name="NameFormated" type="alphaOnlySort" indexed="true"
> stored="true"/>
> <field name="Surname" type="string" indexed="true" stored="true"
> required="true" />
> <field name="SurnameFormated" type="alphaOnlySort" indexed="true"
> stored="true"/>
> <field name="IdSex" type="string" indexed="true" stored="true"
> required="true" />
> <field name="IdWorkingHours" type="sint" indexed="true" stored="true"
> required="true" />
> <field name="IdContractWorkType" type="sint" indexed="true"
> stored="true" required="true" />
> <field name="IdLocation1" type="sint" indexed="true" stored="true"
> required="true" />
> <field name="IdLocation2" type="sint" indexed="true" stored="true"
> required="true" />
> <field name="Location2" type="string" indexed="true" stored="true"
> required="true" />
> <field name="IdLocation3" type="slong" indexed="true" stored="true"
> required="true" />
> <field name="IdLocation4" type="slong" indexed="true" stored="true"
> required="true" />
> <field name="Location4" type="string" indexed="true" stored="true"
> required="true" />
> <field name="IdLocation5" type="slong" indexed="true" stored="true"
> required="true" />
> <field name="Location5" type="string" indexed="true" stored="true"
> required="true" />
> <field name="IdRegion1" type="slong" indexed="true" stored="true"
> required="true" />
> <field name="Region1" type="string" indexed="true" stored="true"
> required="true" />
> <field name="IdRegion2" type="slong" indexed="true" stored="true"
> required="true" />
> <field name="Region2" type="string" indexed="true" stored="true"
> required="true" />
> <field name="LastLoginDate" type="tdate" indexed="true" stored="true"
> required="true" />
> <field name="BirthDate" type="tdate" indexed="true" stored="true"
> required="true" />
> <field name="InsertDate" type="tdate" indexed="true" stored="true"
> required="true" />
> <field name="ModifyDate" type="tdate" indexed="true" stored="true"
> required="true" />
> <field name="IdModifyRangeDate" type="sint" indexed="true"
> stored="true"
> required="true" />
> <field name="Age" type="sint" indexed="true" stored="true"
> required="true" />
> <field name="IdAgeRange" type="sint" indexed="true" stored="true"
> required="true" />
> <field name="Travel" type="sint" indexed="true" stored="true"
> required="true" />
> <field name="ChangeResidence" type="sint" indexed="true" stored="true"
> required="true" />
> <field name="IdEmployed" type="sint" indexed="true" stored="true"
> required="true" />
> <field name="SalaryMax" type="sdouble" indexed="true" stored="true"
> required="true" />
> <field name="SalaryMin" type="sdouble" indexed="true" stored="true"
> required="true" />
> <field name="IdSalaryRange" type="sint" indexed="true" stored="true"
> required="true" />
> <field name="IdPreferenceManagerialLevelMin" type="sint" indexed="true"
> stored="true" required="true" />
> <field name="IdPreferenceManagerialLevelMax" type="sint" indexed="true"
> stored="true" required="true" />
> <field name="IdStudie1Max" type="sint" indexed="true" stored="true"
> required="true" />
> <field name="IdCategory2Last" type="sint" indexed="true" stored="true"
> required="true" />
> <field name="Category2Last" type="string" indexed="true" stored="true"
> required="true" />
> <field name="Category2LastFormated" type="alphaOnlySort" indexed="true"
> stored="true" />
> <field name="IdCategory1Last" type="sint" indexed="true" stored="true"
> required="true" />
> <field name="IdExperienceTime" type="sint" indexed="true" stored="true"
> required="true" />
> <field name="IdExperienceRange" type="sint" indexed="true"
> stored="true"
> required="true" />
> <field name="IsDeficiency" type="string" indexed="true" stored="true"
> required="true" />
> <field name="IdStudie1" type="sint" indexed="true" stored="true"
> required="false" multiValued="true" />
> <field name="IdStudie2" type="sint" indexed="true" stored="true"
> required="false" multiValued="true" />
> <field name="IdStudie2Status" type="sint" indexed="true" stored="true"
> required="false" multiValued="true" />
> <field name="IdLanguage" type="sint" indexed="true" stored="true"
> required="false" multiValued="true" />
> <field name="IdKnowledge1" type="sint" indexed="true" stored="true"
> required="false" multiValued="true" />
> <field name="IdKnowledge2" type="sint" indexed="true" stored="true"
> required="false" multiValued="true" />
> <field name="IdDeficiency1" type="sint" indexed="true" stored="true"
> required="false" multiValued="true" />
> <field name="IdDeficiency2" type="sint" indexed="true" stored="true"
> required="false" multiValued="true" />
> <field name="IdCategory1" type="sint" indexed="true" stored="true"
> required="false" multiValued="true" />
> <field name="IdCategory2" type="sint" indexed="true" stored="true"
> required="false" multiValued="true" />
> <field name="IdCategory2Latest" type="sint" indexed="true"
> stored="true"
> required="false" multiValued="true" />
> <field name="IdPreferenceCategory1" type="sint" indexed="true"
> stored="true" required="false" multiValued="true" />
> <field name="IdPreferenceCategory2" type="sint" indexed="true"
> stored="true" required="false" multiValued="true" />
> <field name="IdLicense" type="sint" indexed="true" stored="true"
> required="false" multiValued="true" />
> <field name="IdVehicle" type="sint" indexed="true" stored="true"
> required="false" multiValued="true" />
> <field name="Phone1" type="string" indexed="true" stored="true"
> required="true" />
> <field name="Phone2" type="string" indexed="true" stored="true"
> required="true" />
> <field name="HasPhoto" type="string" indexed="true" stored="true"
> required="true" />
> <field name="Photo" type="string" indexed="true" stored="true"
> required="true" />
> <field name="StudieCentreLast" type="string" indexed="true"
> stored="true" required="true" />
> <field name="IdManagerialLevelMax" type="sint" indexed="true"
> stored="true" required="true" />
> <field name="IdManagerialLevel" type="sint" indexed="true"
> stored="true"
> required="false" multiValued="true" />
> <field name="CEP" type="string" indexed="true" stored="true"
> required="true" />
> <field name="Complement" type="string" indexed="true" stored="true"
> required="true" />
> <field name="DeficiencyInformation" type="string" indexed="true"
> stored="true" required="true" />
> <field name="Employed" type="string" indexed="true" stored="true"
> required="true" />
> <field name="HasWord" type="string" indexed="true" stored="true"
> required="true" />
> <field name="ContractWorkType" type="string" indexed="true"
> stored="true" required="true" />
> <field name="ExperienceRange" type="string" indexed="true"
> stored="true"
> required="true" />
> <field name="Location1" type="string" indexed="true" stored="true"
> required="true" />
> <field name="Location2Short" type="string" indexed="true" stored="true"
> required="true" />
> <field name="PreferenceManagerialLevelMin" type="string" indexed="true"
> stored="true" required="true" />
> <field name="PreferenceManagerialLevelMax" type="string" indexed="true"
> stored="true" required="true" />
> <field name="IdMaritalStatus" type="string" indexed="true"
> stored="true"
> required="true" />
> <field name="MaritalStatus" type="string" indexed="true" stored="true"
> required="true" />
> <field name="IdNationality" type="sint" indexed="true" stored="true"
> required="true" />
> <field name="Nationality" type="string" indexed="true" stored="true"
> required="true" />
> <field name="IdSalaryType" type="sint" indexed="true" stored="true"
> required="true" />
> <field name="SalaryType" type="string" indexed="true" stored="true"
> required="true" />
> <field name="Sex" type="string" indexed="true" stored="true"
> required="true" />
> <field name="Studie1Max" type="string" indexed="true" stored="true"
> required="true" />
> <field name="IdVisibility" type="string" indexed="true" stored="true"
> required="true" />
> <field name="WorkingHours" type="string" indexed="true" stored="true"
> required="true" />
> <field name="Number" type="sint" indexed="true" stored="true"
> required="true" />
> <field name="ProductCode" type="string" indexed="true" stored="true"
> required="true" />
> <field name="IdLanguageLevel" type="string" indexed="true"
> stored="true"
> required="false" multiValued="true" />
> <field name="Fti" type="text" indexed="true" stored="false"
> required="true" />
> <field name="Latitude" type="double" indexed="true" stored="true"
> required="true" />
> <field name="Longitude" type="double" indexed="true" stored="true"
> required="true" />
> <field name="lat" type="double" indexed="true" stored="true"
> required="true" multiValued="false"/>
> <field name="lng" type="double" indexed="true" stored="true"
> required="true" multiValued="false"/>
> <field name="location_latlng" type="location" indexed="true"
> stored="true" required="true"/>
> <field name="IdCobranded" type="sint" indexed="true" stored="true"
> required="false" multiValued="true" />
> <field name="IdCandidateStatus" type="sint" indexed="true"
> stored="true"
> required="true" />
> <field name="IdManagerialLevelLatest" type="sint" indexed="true"
> stored="true" required="false" multiValued="true" />
> <dynamicField name="*_coordinate" type="tdouble" indexed="true"
> stored="false"/>
>
> *Solconfig.xml:*
> <luceneMatchVersion>LUCENE_35</luceneMatchVersion>
> ...
> <directoryFactory name="DirectoryFactory"
> class="${solr.directoryFactory:solr.StandardDirectoryFactory}"/>
> <useCompoundFile>false</useCompoundFile>
> <ramBufferSizeMB>32</ramBufferSizeMB>
> <mergeFactor>10</mergeFactor>
> <unlockOnStartup>false</unlockOnStartup>
> <reopenReaders>true</reopenReaders>
> ...
> <maxBooleanClauses>131072</maxBooleanClauses>
> <filterCache class="solr.FastLRUCache" size="262144" initialSize="8048"
> autowarmCount="8048"/><queryResultCache class="solr.LRUCache" size="262144"
> initialSize="8048" autowarmCount="8048"/><documentCache
> class="solr.LRUCache" size="262144" initialSize="8048"
>
> autowarmCount="8048"/><enableLazyFieldLoading>true</enableLazyFieldLoading><queryResultWindowSize>60</queryResultWindowSize>
> <queryResultMaxDocsCached>200</queryResultMaxDocsCached>
> ...
> <useColdSearcher>false</useColdSearcher>
> <maxWarmingSearchers>2</maxWarmingSearchers>
>
>
>
>
> --
> View this message in context:
> http://lucene.472066.n3.nabble.com/Query-Time-problem-on-Big-Index-Solr-3-5-tp4003660.html
> Sent from the Solr - User mailing list archive at Nabble.com.
>