You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2013/02/17 01:45:24 UTC
svn commit: r1446985 [1/2] - in /lucene/dev/branches/lucene4765: ./ lucene/
lucene/core/ lucene/core/src/test/org/apache/lucene/search/ lucene/queries/
lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/
solr/ solr/core/ solr/core/s...
Author: rmuir
Date: Sun Feb 17 00:45:22 2013
New Revision: 1446985
URL: http://svn.apache.org/r1446985
Log:
Merged /lucene/dev/trunk:r1446754-1446983
Added:
lucene/dev/branches/lucene4765/lucene/core/src/test/org/apache/lucene/search/TestSort.java
- copied unchanged from r1446983, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestSort.java
lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/request/NumericFacets.java
- copied unchanged from r1446983, lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/NumericFacets.java
lucene/dev/branches/lucene4765/solr/core/src/test-files/solr/collection1/conf/bad-schema-docValues-not-required-no-default.xml
- copied unchanged from r1446983, lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/bad-schema-docValues-not-required-no-default.xml
lucene/dev/branches/lucene4765/solr/core/src/test-files/solr/collection1/conf/bad-schema-unsupported-docValues.xml
- copied unchanged from r1446983, lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/bad-schema-unsupported-docValues.xml
lucene/dev/branches/lucene4765/solr/core/src/test-files/solr/collection1/conf/schema-docValues.xml
- copied unchanged from r1446983, lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/schema-docValues.xml
lucene/dev/branches/lucene4765/solr/core/src/test/org/apache/solr/schema/DocValuesTest.java
- copied unchanged from r1446983, lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/DocValuesTest.java
Removed:
lucene/dev/branches/lucene4765/lucene/core/src/test/org/apache/lucene/search/TestSort2.java
Modified:
lucene/dev/branches/lucene4765/ (props changed)
lucene/dev/branches/lucene4765/lucene/ (props changed)
lucene/dev/branches/lucene4765/lucene/common-build.xml (contents, props changed)
lucene/dev/branches/lucene4765/lucene/core/ (props changed)
lucene/dev/branches/lucene4765/lucene/queries/ (props changed)
lucene/dev/branches/lucene4765/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java
lucene/dev/branches/lucene4765/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java
lucene/dev/branches/lucene4765/solr/ (props changed)
lucene/dev/branches/lucene4765/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/lucene4765/solr/core/ (props changed)
lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/cloud/ZkController.java
lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/core/CoreContainer.java
lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java
lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/core/SolrCore.java
lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java
lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java
lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java
lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/handler/component/StatsValues.java
lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java
lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/request/SimpleFacets.java
lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/request/UnInvertedField.java
lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java
lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/CurrencyField.java
lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/DateField.java
lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/FieldProperties.java
lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/FieldType.java
lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/LatLonType.java
lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/PointType.java
lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/SchemaField.java
lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java
lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java
lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/SortableIntField.java
lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/SortableLongField.java
lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/StrField.java
lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/TrieDateField.java
lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/TrieField.java
lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/UUIDField.java
lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java
lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/update/UpdateHandler.java
lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/update/UpdateLog.java
lucene/dev/branches/lucene4765/solr/core/src/test-files/solr/collection1/conf/schema.xml
lucene/dev/branches/lucene4765/solr/core/src/test-files/solr/collection1/conf/schema_codec.xml
lucene/dev/branches/lucene4765/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
lucene/dev/branches/lucene4765/solr/core/src/test-files/solr/solr.xml
lucene/dev/branches/lucene4765/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java
lucene/dev/branches/lucene4765/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java
lucene/dev/branches/lucene4765/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java
lucene/dev/branches/lucene4765/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java
lucene/dev/branches/lucene4765/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java
lucene/dev/branches/lucene4765/solr/core/src/test/org/apache/solr/schema/BadIndexSchemaTest.java
lucene/dev/branches/lucene4765/solr/core/src/test/org/apache/solr/schema/CurrencyFieldTest.java
lucene/dev/branches/lucene4765/solr/core/src/test/org/apache/solr/schema/PolyFieldTest.java
lucene/dev/branches/lucene4765/solr/example/ (props changed)
lucene/dev/branches/lucene4765/solr/example/solr/collection1/conf/schema.xml
lucene/dev/branches/lucene4765/solr/solrj/ (props changed)
lucene/dev/branches/lucene4765/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrServerTest.java
lucene/dev/branches/lucene4765/solr/test-framework/ (props changed)
lucene/dev/branches/lucene4765/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java
lucene/dev/branches/lucene4765/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
Modified: lucene/dev/branches/lucene4765/lucene/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/common-build.xml?rev=1446985&r1=1446984&r2=1446985&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/common-build.xml (original)
+++ lucene/dev/branches/lucene4765/lucene/common-build.xml Sun Feb 17 00:45:22 2013
@@ -1930,7 +1930,7 @@ ${tests-output}/junit4-*.suites - pe
<!-- Forbidden API Task -->
<target name="install-forbidden-apis" unless="forbidden-apis.loaded" depends="ivy-availability-check,ivy-configure">
- <ivy:cachepath organisation="de.thetaphi" module="forbiddenapis" revision="1.1"
+ <ivy:cachepath organisation="de.thetaphi" module="forbiddenapis" revision="1.2"
inline="true" conf="default" transitive="true" pathid="forbidden-apis.classpath"/>
<taskdef name="forbidden-apis" classname="de.thetaphi.forbiddenapis.AntTask" classpathref="forbidden-apis.classpath"/>
<property name="forbidden-apis.loaded" value="true"/>
Modified: lucene/dev/branches/lucene4765/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java?rev=1446985&r1=1446984&r2=1446985&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java (original)
+++ lucene/dev/branches/lucene4765/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java Sun Feb 17 00:45:22 2013
@@ -85,7 +85,7 @@ public class IntFieldSource extends Fiel
@Override
public String strVal(int doc) {
- return Float.toString(arr.get(doc));
+ return Integer.toString(arr.get(doc));
}
@Override
Modified: lucene/dev/branches/lucene4765/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java?rev=1446985&r1=1446984&r2=1446985&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java (original)
+++ lucene/dev/branches/lucene4765/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java Sun Feb 17 00:45:22 2013
@@ -64,6 +64,10 @@ public class LongFieldSource extends Fie
return val;
}
+ public String longToString(long val) {
+ return longToObject(val).toString();
+ }
+
@Override
public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
final FieldCache.Longs arr = cache.getLongs(readerContext.reader(), field, parser, true);
@@ -86,6 +90,11 @@ public class LongFieldSource extends Fie
}
@Override
+ public String strVal(int doc) {
+ return valid.get(doc) ? longToString(arr.get(doc)) : null;
+ }
+
+ @Override
public ValueSourceScorer getRangeScorer(IndexReader reader, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) {
long lower,upper;
Modified: lucene/dev/branches/lucene4765/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/solr/CHANGES.txt?rev=1446985&r1=1446984&r2=1446985&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/solr/CHANGES.txt (original)
+++ lucene/dev/branches/lucene4765/solr/CHANGES.txt Sun Feb 17 00:45:22 2013
@@ -77,6 +77,8 @@ New Features
under the covers -- allowing many HTTP connection related properties to be
controlled via 'standard' java system properties. (hossman)
+* SOLR-3855: Doc values support. (Adrien Grand)
+
Bug Fixes
----------------------
@@ -107,7 +109,7 @@ Bug Fixes
* SOLR-3926: Solr should support better way of finding active sorts (Eirik Lygre via
Erick Erickson)
-* SOLR-4342: Fix DataImportHandler stats to be a prper Map (hossman)
+* SOLR-4342: Fix DataImportHandler stats to be a proper Map (hossman)
* SOLR-3967: langid.enforceSchema option checks source field instead of target field (janhoy)
@@ -125,6 +127,16 @@ Bug Fixes
* SOLR-4463: Fix SolrCoreState reference counting. (Mark Miller)
+* SOLR-4459: The Replication 'index move' rather than copy optimization doesn't
+ kick in when using NRTCachingDirectory or the rate limiting feature.
+ (Mark Miller)
+
+* SOLR-4421,SOLR-4165: On CoreContainer shutdown, all SolrCores should publish their
+ state as DOWN. (Mark Miller, Markus Jelsma)
+
+* SOLR-4467: Ephemeral directory implementations may not recover correctly
+ because the code to clear the tlog files on startup is off. (Mark Miller)
+
Optimizations
----------------------
@@ -157,9 +169,6 @@ Other Changes
* SOLR-4384: Make post.jar report timing information (Upayavira via janhoy)
-* SOLR-4421: On CoreContainer shutdown, all SolrCores should publish their
- state as DOWN. (Mark Miller)
-
================== 4.1.0 ==================
Versions of Major Components
Modified: lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java?rev=1446985&r1=1446984&r2=1446985&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java (original)
+++ lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java Sun Feb 17 00:45:22 2013
@@ -294,7 +294,7 @@ final class ShardLeaderElectionContext e
Slice slices = zkController.getClusterState().getSlice(collection, shardId);
int cnt = 0;
- while (true && !isClosed) {
+ while (true && !isClosed && !cc.isShutDown()) {
// wait for everyone to be up
if (slices != null) {
int found = 0;
Modified: lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/cloud/ZkController.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/cloud/ZkController.java?rev=1446985&r1=1446984&r2=1446985&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/cloud/ZkController.java (original)
+++ lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/cloud/ZkController.java Sun Feb 17 00:45:22 2013
@@ -480,9 +480,14 @@ public final class ZkController {
}
private void init(CurrentCoreDescriptorProvider registerOnReconnect) {
- boolean alreadyCreatedZkReader = false;
+
try {
- alreadyCreatedZkReader = publishAndWaitForDownStates(alreadyCreatedZkReader);
+ boolean createdWatchesAndUpdated = false;
+ if (zkClient.exists(ZkStateReader.LIVE_NODES_ZKNODE, true)) {
+ zkStateReader.createClusterStateWatchersAndUpdate();
+ createdWatchesAndUpdated = true;
+ publishAndWaitForDownStates();
+ }
// makes nodes zkNode
cmdExecutor.ensureExists(ZkStateReader.LIVE_NODES_ZKNODE, zkClient);
@@ -501,7 +506,7 @@ public final class ZkController {
overseerElector.setup(context);
overseerElector.joinElection(context, false);
- if (!alreadyCreatedZkReader) {
+ if (!createdWatchesAndUpdated) {
zkStateReader.createClusterStateWatchersAndUpdate();
}
@@ -523,93 +528,94 @@ public final class ZkController {
}
- private boolean publishAndWaitForDownStates(boolean alreadyCreatedZkReader)
- throws KeeperException, InterruptedException {
- if (zkClient.exists(ZkStateReader.LIVE_NODES_ZKNODE, true)) {
- alreadyCreatedZkReader = true;
- // try and publish anyone from our node as down
- zkStateReader.createClusterStateWatchersAndUpdate();
- ClusterState clusterState = zkStateReader.getClusterState();
- Set<String> collections = clusterState.getCollections();
- List<String> updatedNodes = new ArrayList<String>();
+ public void publishAndWaitForDownStates() throws KeeperException,
+ InterruptedException {
+
+ ClusterState clusterState = zkStateReader.getClusterState();
+ Set<String> collections = clusterState.getCollections();
+ List<String> updatedNodes = new ArrayList<String>();
+ for (String collectionName : collections) {
+ DocCollection collection = clusterState.getCollection(collectionName);
+ Collection<Slice> slices = collection.getSlices();
+ for (Slice slice : slices) {
+ Collection<Replica> replicas = slice.getReplicas();
+ for (Replica replica : replicas) {
+ if (replica.getNodeName().equals(getNodeName())
+ && !(replica.getStr(ZkStateReader.STATE_PROP)
+ .equals(ZkStateReader.DOWN))) {
+ ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, "state",
+ ZkStateReader.STATE_PROP, ZkStateReader.DOWN,
+ ZkStateReader.BASE_URL_PROP, getBaseUrl(),
+ ZkStateReader.CORE_NAME_PROP,
+ replica.getStr(ZkStateReader.CORE_NAME_PROP),
+ ZkStateReader.ROLES_PROP,
+ replica.getStr(ZkStateReader.ROLES_PROP),
+ ZkStateReader.NODE_NAME_PROP, getNodeName(),
+ ZkStateReader.SHARD_ID_PROP,
+ replica.getStr(ZkStateReader.SHARD_ID_PROP),
+ ZkStateReader.COLLECTION_PROP,
+ replica.getStr(ZkStateReader.COLLECTION_PROP));
+ updatedNodes.add(replica.getStr(ZkStateReader.CORE_NAME_PROP));
+ overseerJobQueue.offer(ZkStateReader.toJSON(m));
+ }
+ }
+ }
+ }
+
+ // now wait till the updates are in our state
+ long now = System.currentTimeMillis();
+ long timeout = now + 1000 * 30;
+ boolean foundStates = false;
+ while (System.currentTimeMillis() < timeout) {
+ clusterState = zkStateReader.getClusterState();
+ collections = clusterState.getCollections();
for (String collectionName : collections) {
DocCollection collection = clusterState.getCollection(collectionName);
Collection<Slice> slices = collection.getSlices();
for (Slice slice : slices) {
Collection<Replica> replicas = slice.getReplicas();
for (Replica replica : replicas) {
- if (replica.getNodeName().equals(getNodeName())
- && !(replica.getStr(ZkStateReader.STATE_PROP)
- .equals(ZkStateReader.DOWN))) {
- ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION,
- "state", ZkStateReader.STATE_PROP, ZkStateReader.DOWN,
- ZkStateReader.BASE_URL_PROP, getBaseUrl(),
- ZkStateReader.CORE_NAME_PROP, replica.getStr(ZkStateReader.CORE_NAME_PROP),
- ZkStateReader.ROLES_PROP,
- replica.getStr(ZkStateReader.ROLES_PROP),
- ZkStateReader.NODE_NAME_PROP, getNodeName(),
- ZkStateReader.SHARD_ID_PROP,
- replica.getStr(ZkStateReader.SHARD_ID_PROP),
- ZkStateReader.COLLECTION_PROP,
- replica.getStr(ZkStateReader.COLLECTION_PROP));
- updatedNodes.add(replica.getStr(ZkStateReader.CORE_NAME_PROP));
- overseerJobQueue.offer(ZkStateReader.toJSON(m));
+ if (replica.getStr(ZkStateReader.STATE_PROP).equals(
+ ZkStateReader.DOWN)) {
+ updatedNodes.remove(replica.getStr(ZkStateReader.CORE_NAME_PROP));
+
}
}
}
}
- // now wait till the updates are in our state
- long now = System.currentTimeMillis();
- long timeout = now + 1000 * 300;
- boolean foundStates = false;
- while (System.currentTimeMillis() < timeout) {
- clusterState = zkStateReader.getClusterState();
- collections = clusterState.getCollections();
- for (String collectionName : collections) {
- DocCollection collection = clusterState
- .getCollection(collectionName);
- Collection<Slice> slices = collection.getSlices();
- for (Slice slice : slices) {
- Collection<Replica> replicas = slice.getReplicas();
- for (Replica replica : replicas) {
- if (replica.getStr(ZkStateReader.STATE_PROP).equals(
- ZkStateReader.DOWN)) {
- updatedNodes.remove(replica
- .getStr(ZkStateReader.CORE_NAME_PROP));
-
- }
- }
- }
- }
-
- if (updatedNodes.size() == 0) {
- foundStates = true;
- break;
- }
- }
- if (!foundStates) {
- log.warn("Timed out waiting to see all nodes published as DOWN in our cluster state.");
+ if (updatedNodes.size() == 0) {
+ foundStates = true;
+ Thread.sleep(1000);
+ break;
}
+ Thread.sleep(1000);
}
- return alreadyCreatedZkReader;
+ if (!foundStates) {
+ log.warn("Timed out waiting to see all nodes published as DOWN in our cluster state.");
+ }
+
}
-
+
/**
- * Validates if the chroot exists in zk (or if it is successfully created). Optionally, if create is set to true this method will create the path
- * in case it doesn't exist
- * @return true if the path exists or is created
- * false if the path doesn't exist and 'create' = false
+ * Validates if the chroot exists in zk (or if it is successfully created).
+ * Optionally, if create is set to true this method will create the path in
+ * case it doesn't exist
+ *
+ * @return true if the path exists or is created false if the path doesn't
+ * exist and 'create' = false
*/
- public static boolean checkChrootPath(String zkHost, boolean create) throws KeeperException, InterruptedException {
- if(!containsChroot(zkHost)) {
+ public static boolean checkChrootPath(String zkHost, boolean create)
+ throws KeeperException, InterruptedException {
+ if (!containsChroot(zkHost)) {
return true;
}
log.info("zkHost includes chroot");
String chrootPath = zkHost.substring(zkHost.indexOf("/"), zkHost.length());
- SolrZkClient tmpClient = new SolrZkClient(zkHost.substring(0, zkHost.indexOf("/")), 60*1000);
+ SolrZkClient tmpClient = new SolrZkClient(zkHost.substring(0,
+ zkHost.indexOf("/")), 60 * 1000);
boolean exists = tmpClient.exists(chrootPath, true);
- if(!exists && create) {
+ if (!exists && create) {
tmpClient.makePath(chrootPath, false, true);
exists = true;
}
@@ -617,7 +623,6 @@ public final class ZkController {
return exists;
}
-
/**
* Validates if zkHost contains a chroot. See http://zookeeper.apache.org/doc/r3.2.2/zookeeperProgrammers.html#ch_zkSessions
*/
Modified: lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/core/CoreContainer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/core/CoreContainer.java?rev=1446985&r1=1446984&r2=1446985&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/core/CoreContainer.java (original)
+++ lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/core/CoreContainer.java Sun Feb 17 00:45:22 2013
@@ -740,12 +740,24 @@ public class CoreContainer
public void shutdown() {
log.info("Shutting down CoreContainer instance="
+ System.identityHashCode(this));
+
+ if (isZooKeeperAware()) {
+ try {
+ zkController.publishAndWaitForDownStates();
+ } catch (KeeperException e) {
+ log.error("", e);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ log.warn("", e);
+ }
+ }
+
isShutDown = true;
if (isZooKeeperAware()) {
- publishCoresAsDown();
cancelCoreRecoveries();
}
+
try {
synchronized (cores) {
@@ -784,20 +796,6 @@ public class CoreContainer
}
}
- private void publishCoresAsDown() {
- synchronized (cores) {
- for (SolrCore core : cores.values()) {
- try {
- zkController.publish(core.getCoreDescriptor(), ZkStateReader.DOWN);
- } catch (KeeperException e) {
- log.error("", e);
- } catch (InterruptedException e) {
- log.error("", e);
- }
- }
- }
- }
-
public void cancelCoreRecoveries() {
ArrayList<SolrCoreState> coreStates = new ArrayList<SolrCoreState>();
synchronized (cores) {
Modified: lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java?rev=1446985&r1=1446984&r2=1446985&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java (original)
+++ lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java Sun Feb 17 00:45:22 2013
@@ -103,6 +103,10 @@ public abstract class DirectoryFactory i
/**
* Override for more efficient moves.
*
+ * Intended for use with replication - use
+ * carefully - some Directory wrappers will
+ * cache files for example.
+ *
* @throws IOException If there is a low-level I/O error.
*/
public void move(Directory fromDir, Directory toDir, String fileName, IOContext ioContext) throws IOException {
Modified: lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java?rev=1446985&r1=1446984&r2=1446985&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java (original)
+++ lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java Sun Feb 17 00:45:22 2013
@@ -1,6 +1,7 @@
package org.apache.solr.core;
import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.lucene42.Lucene42Codec;
import org.apache.solr.schema.IndexSchema;
@@ -55,7 +56,18 @@ public class SchemaCodecFactory extends
}
return super.getPostingsFormatForField(field);
}
- // TODO: when dv support is added to solr, add it here too
+ @Override
+ public DocValuesFormat getDocValuesFormatForField(String field) {
+ final SchemaField fieldOrNull = schema.getFieldOrNull(field);
+ if (fieldOrNull == null) {
+ throw new IllegalArgumentException("no such field " + field);
+ }
+ String docValuesFormatName = fieldOrNull.getType().getDocValuesFormat();
+ if (docValuesFormatName != null) {
+ return DocValuesFormat.forName(docValuesFormatName);
+ }
+ return super.getDocValuesFormatForField(field);
+ }
};
}
Modified: lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/core/SolrCore.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/core/SolrCore.java?rev=1446985&r1=1446984&r2=1446985&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/core/SolrCore.java (original)
+++ lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/core/SolrCore.java Sun Feb 17 00:45:22 2013
@@ -834,6 +834,11 @@ public final class SolrCore implements S
log.error(msg);
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, msg);
}
+ if (null != ft.getDocValuesFormat()) {
+ String msg = "FieldType '" + ft.getTypeName() + "' is configured with a docValues format, but the codec does not support it: " + factory.getClass();
+ log.error(msg);
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, msg);
+ }
}
}
return factory.getCodec();
Modified: lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java?rev=1446985&r1=1446984&r2=1446985&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java (original)
+++ lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java Sun Feb 17 00:45:22 2013
@@ -23,6 +23,8 @@ import org.apache.commons.io.FileUtils;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.NRTCachingDirectory;
+import org.apache.lucene.store.RateLimitedDirectoryWrapper;
/**
* Directory provider which mimics original Solr
@@ -69,15 +71,29 @@ public class StandardDirectoryFactory ex
/**
* Override for more efficient moves.
*
+ * Intended for use with replication - use
+ * carefully - some Directory wrappers will
+ * cache files for example.
+ *
+ * This implementation works with two wrappers:
+ * NRTCachingDirectory and RateLimitedDirectoryWrapper.
+ *
+ * You should first {@link Directory#sync(java.util.Collection)} any file that will be
+ * moved or avoid cached files through settings.
+ *
* @throws IOException
* If there is a low-level I/O error.
*/
@Override
public void move(Directory fromDir, Directory toDir, String fileName, IOContext ioContext)
throws IOException {
- if (fromDir instanceof FSDirectory && toDir instanceof FSDirectory) {
- File dir1 = ((FSDirectory) fromDir).getDirectory();
- File dir2 = ((FSDirectory) toDir).getDirectory();
+
+ Directory baseFromDir = getBaseDir(fromDir);
+ Directory baseToDir = getBaseDir(fromDir);
+
+ if (baseFromDir instanceof FSDirectory && baseToDir instanceof FSDirectory) {
+ File dir1 = ((FSDirectory) baseFromDir).getDirectory();
+ File dir2 = ((FSDirectory) baseToDir).getDirectory();
File indexFileInTmpDir = new File(dir1, fileName);
File indexFileInIndex = new File(dir2, fileName);
boolean success = indexFileInTmpDir.renameTo(indexFileInIndex);
@@ -89,4 +105,18 @@ public class StandardDirectoryFactory ex
super.move(fromDir, toDir, fileName, ioContext);
}
+ // special hack to work with NRTCachingDirectory and RateLimitedDirectoryWrapper
+ private Directory getBaseDir(Directory dir) {
+ Directory baseDir;
+ if (dir instanceof NRTCachingDirectory) {
+ baseDir = ((NRTCachingDirectory)dir).getDelegate();
+ } else if (dir instanceof RateLimitedDirectoryWrapper) {
+ baseDir = ((RateLimitedDirectoryWrapper)dir).getDelegate();
+ } else {
+ baseDir = dir;
+ }
+
+ return baseDir;
+ }
+
}
Modified: lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java?rev=1446985&r1=1446984&r2=1446985&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java (original)
+++ lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java Sun Feb 17 00:45:22 2013
@@ -16,16 +16,22 @@ package org.apache.solr.handler.componen
* limitations under the License.
*/
+import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.lucene.index.AtomicReader;
+import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.queries.function.FunctionValues;
+import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.util.BytesRef;
-import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.SchemaField;
+import org.apache.solr.search.SolrIndexSearcher;
/**
@@ -40,101 +46,76 @@ import org.apache.solr.schema.SchemaFiel
public class FieldFacetStats {
public final String name;
- final SortedDocValues si;
final SchemaField facet_sf;
final SchemaField field_sf;
- final int startTermIndex;
- final int endTermIndex;
- final int nTerms;
+ public final Map<String, StatsValues> facetStatsValues;
- final int numStatsTerms;
+ List<HashMap<String, Integer>> facetStatsTerms;
- public final Map<String, StatsValues> facetStatsValues;
+ final AtomicReader topLevelReader;
+ AtomicReaderContext leave;
+ final ValueSource valueSource;
+ AtomicReaderContext context;
+ FunctionValues values;
- final List<HashMap<String, Integer>> facetStatsTerms;
+ SortedDocValues topLevelSortedValues = null;
private final BytesRef tempBR = new BytesRef();
- public FieldFacetStats(String name, SortedDocValues si, SchemaField field_sf, SchemaField facet_sf, int numStatsTerms) {
+ public FieldFacetStats(SolrIndexSearcher searcher, String name, SchemaField field_sf, SchemaField facet_sf) {
this.name = name;
- this.si = si;
this.field_sf = field_sf;
this.facet_sf = facet_sf;
- this.numStatsTerms = numStatsTerms;
- startTermIndex = 0;
- endTermIndex = si.getValueCount();
- nTerms = endTermIndex - startTermIndex;
+ topLevelReader = searcher.getAtomicReader();
+ valueSource = facet_sf.getType().getValueSource(facet_sf, null);
facetStatsValues = new HashMap<String, StatsValues>();
-
- // for mv stats field, we'll want to keep track of terms
facetStatsTerms = new ArrayList<HashMap<String, Integer>>();
- if (numStatsTerms == 0) return;
- int i = 0;
- for (; i < numStatsTerms; i++) {
- facetStatsTerms.add(new HashMap<String, Integer>());
- }
}
- BytesRef getTermText(int docID, BytesRef ret) {
- final int ord = si.getOrd(docID);
- if (ord == -1) {
- return null;
- } else {
- si.lookupOrd(ord, ret);
- return ret;
+ private StatsValues getStatsValues(String key) throws IOException {
+ StatsValues stats = facetStatsValues.get(key);
+ if (stats == null) {
+ stats = StatsValuesFactory.createStatsValues(field_sf);
+ facetStatsValues.put(key, stats);
+ stats.setNextReader(context);
}
+ return stats;
}
- public boolean facet(int docID, BytesRef v) {
- int term = si.getOrd(docID);
- int arrIdx = term - startTermIndex;
- if (arrIdx >= 0 && arrIdx < nTerms) {
-
- final BytesRef br;
- if (term == -1) {
- br = null;
- } else {
- br = tempBR;
- si.lookupOrd(term, tempBR);
- }
- String key = (br == null)?null:facet_sf.getType().indexedToReadable(br.utf8ToString());
- StatsValues stats = facetStatsValues.get(key);
- if (stats == null) {
- stats = StatsValuesFactory.createStatsValues(field_sf);
- facetStatsValues.put(key, stats);
- }
-
- if (v != null && v.length>0) {
- stats.accumulate(v);
- } else {
- stats.missing();
- return false;
- }
- return true;
- }
- return false;
+ // docID is relative to the context
+ public void facet(int docID) throws IOException {
+ final String key = values.exists(docID)
+ ? values.strVal(docID)
+ : null;
+ final StatsValues stats = getStatsValues(key);
+ stats.accumulate(docID);
}
-
// Function to keep track of facet counts for term number.
// Currently only used by UnInvertedField stats
- public boolean facetTermNum(int docID, int statsTermNum) {
-
- int term = si.getOrd(docID);
- int arrIdx = term - startTermIndex;
- if (arrIdx >= 0 && arrIdx < nTerms) {
+ public boolean facetTermNum(int docID, int statsTermNum) throws IOException {
+ if (topLevelSortedValues == null) {
+ topLevelSortedValues = FieldCache.DEFAULT.getTermsIndex(topLevelReader, name);
+ }
+
+ int term = topLevelSortedValues.getOrd(docID);
+ int arrIdx = term;
+ if (arrIdx >= 0 && arrIdx < topLevelSortedValues.getValueCount()) {
final BytesRef br;
if (term == -1) {
br = null;
} else {
br = tempBR;
- si.lookupOrd(term, tempBR);
+ topLevelSortedValues.lookupOrd(term, tempBR);
}
String key = br == null ? null : br.utf8ToString();
- HashMap<String, Integer> statsTermCounts = facetStatsTerms.get(statsTermNum);
+ while (facetStatsTerms.size() <= statsTermNum) {
+ facetStatsTerms.add(new HashMap<String, Integer>());
+ }
+ final Map<String, Integer> statsTermCounts = facetStatsTerms.get(statsTermNum);
Integer statsTermCount = statsTermCounts.get(key);
if (statsTermCount == null) {
statsTermCounts.put(key, 1);
@@ -148,8 +129,11 @@ public class FieldFacetStats {
//function to accumulate counts for statsTermNum to specified value
- public boolean accumulateTermNum(int statsTermNum, BytesRef value) {
+ public boolean accumulateTermNum(int statsTermNum, BytesRef value) throws IOException {
if (value == null) return false;
+ while (facetStatsTerms.size() <= statsTermNum) {
+ facetStatsTerms.add(new HashMap<String, Integer>());
+ }
for (Map.Entry<String, Integer> stringIntegerEntry : facetStatsTerms.get(statsTermNum).entrySet()) {
Map.Entry pairs = (Map.Entry) stringIntegerEntry;
String key = (String) pairs.getKey();
@@ -166,6 +150,14 @@ public class FieldFacetStats {
return true;
}
+ public void setNextReader(AtomicReaderContext ctx) throws IOException {
+ this.context = ctx;
+ values = valueSource.getValues(Collections.emptyMap(), ctx);
+ for (StatsValues stats : facetStatsValues.values()) {
+ stats.setNextReader(ctx);
+ }
+ }
+
}
Modified: lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java?rev=1446985&r1=1446984&r2=1446985&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java (original)
+++ lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java Sun Feb 17 00:45:22 2013
@@ -20,12 +20,11 @@ package org.apache.solr.handler.componen
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import org.apache.lucene.index.SortedDocValues;
-import org.apache.lucene.search.FieldCache;
-import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.index.AtomicReaderContext;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.params.SolrParams;
@@ -43,14 +42,12 @@ import org.apache.solr.search.SolrIndexS
/**
* Stats component calculates simple statistics on numeric field values
- *
- *
* @since solr 1.4
*/
public class StatsComponent extends SearchComponent {
public static final String COMPONENT_NAME = "stats";
-
+
@Override
public void prepare(ResponseBuilder rb) throws IOException {
if (rb.req.getParams().getBool(StatsParams.STATS,false)) {
@@ -236,25 +233,13 @@ class SimpleStats {
}
return res;
}
-
- // why does this use a top-level field cache?
- public NamedList<?> getFieldCacheStats(String fieldName, String[] facet ) {
- SchemaField sf = searcher.getSchema().getField(fieldName);
-
- SortedDocValues si;
- try {
- si = FieldCache.DEFAULT.getTermsIndex(searcher.getAtomicReader(), fieldName);
- }
- catch (IOException e) {
- throw new RuntimeException( "failed to open field cache for: "+fieldName, e );
- }
- StatsValues allstats = StatsValuesFactory.createStatsValues(sf);
- final int nTerms = si.getValueCount();
- if ( nTerms <= 0 || docs.size() <= 0 ) return allstats.getStatsValues();
- // don't worry about faceting if no documents match...
+ public NamedList<?> getFieldCacheStats(String fieldName, String[] facet) throws IOException {
+ final SchemaField sf = searcher.getSchema().getField(fieldName);
+
+ final StatsValues allstats = StatsValuesFactory.createStatsValues(sf);
+
List<FieldFacetStats> facetStats = new ArrayList<FieldFacetStats>();
- SortedDocValues facetTermsIndex;
for( String facetField : facet ) {
SchemaField fsf = searcher.getSchema().getField(facetField);
@@ -262,40 +247,32 @@ class SimpleStats {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Stats can only facet on single-valued fields, not: " + facetField );
}
-
- try {
- facetTermsIndex = FieldCache.DEFAULT.getTermsIndex(searcher.getAtomicReader(), facetField);
- }
- catch (IOException e) {
- throw new RuntimeException( "failed to open field cache for: "
- + facetField, e );
- }
- facetStats.add(new FieldFacetStats(facetField, facetTermsIndex, sf, fsf, nTerms));
+
+ facetStats.add(new FieldFacetStats(searcher, facetField, sf, fsf));
}
-
- final BytesRef tempBR = new BytesRef();
- DocIterator iter = docs.iterator();
- while (iter.hasNext()) {
- int docID = iter.nextDoc();
- int docOrd = si.getOrd(docID);
- BytesRef raw;
- if (docOrd == -1) {
- allstats.missing();
- tempBR.length = 0;
- raw = tempBR;
- } else {
- raw = tempBR;
- si.lookupOrd(docOrd, tempBR);
- if( tempBR.length > 0 ) {
- allstats.accumulate(tempBR);
- } else {
- allstats.missing();
+
+ final Iterator<AtomicReaderContext> ctxIt = searcher.getIndexReader().leaves().iterator();
+ AtomicReaderContext ctx = null;
+ for (DocIterator docsIt = docs.iterator(); docsIt.hasNext(); ) {
+ final int doc = docsIt.nextDoc();
+ if (ctx == null || doc >= ctx.docBase + ctx.reader().maxDoc()) {
+ // advance
+ do {
+ ctx = ctxIt.next();
+ } while (ctx == null || doc >= ctx.docBase + ctx.reader().maxDoc());
+ assert doc >= ctx.docBase;
+
+ // propagate the context among accumulators.
+ allstats.setNextReader(ctx);
+ for (FieldFacetStats f : facetStats) {
+ f.setNextReader(ctx);
}
}
- // now update the facets
+ // accumulate
+ allstats.accumulate(doc - ctx.docBase);
for (FieldFacetStats f : facetStats) {
- f.facet(docID, raw);
+ f.facet(doc - ctx.docBase);
}
}
@@ -305,5 +282,4 @@ class SimpleStats {
return allstats.getStatsValues();
}
-
}
Modified: lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/handler/component/StatsValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/handler/component/StatsValues.java?rev=1446985&r1=1446984&r2=1446985&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/handler/component/StatsValues.java (original)
+++ lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/handler/component/StatsValues.java Sun Feb 17 00:45:22 2013
@@ -19,14 +19,19 @@
package org.apache.solr.handler.component;
+import java.io.IOException;
import java.util.Map;
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.schema.FieldType;
/**
* StatsValue defines the interface for the collection of statistical values about fields and facets.
*/
+// TODO: should implement Collector?
public interface StatsValues {
/**
@@ -36,12 +41,9 @@ public interface StatsValues {
*/
void accumulate(NamedList stv);
- /**
- * Accumulate the values based on the given value
- *
- * @param value Value to use to accumulate the current values
- */
- void accumulate(BytesRef value);
+ /** Accumulate the value associated with <code>docID</code>.
+ * @see #setNextReader(AtomicReaderContext) */
+ void accumulate(int docID);
/**
* Accumulate the values based on the given value
@@ -77,4 +79,7 @@ public interface StatsValues {
* @return NamedList representation of the current values
*/
NamedList<?> getStatsValues();
+
+ /** Set the context for {@link #accumulate(int)}. */
+ void setNextReader(AtomicReaderContext ctx) throws IOException;
}
Modified: lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java?rev=1446985&r1=1446984&r2=1446985&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java (original)
+++ lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java Sun Feb 17 00:45:22 2013
@@ -17,10 +17,15 @@
package org.apache.solr.handler.component;
+import java.io.IOException;
+import java.util.Collections;
import java.util.Date;
import java.util.Map;
import java.util.HashMap;
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.queries.function.FunctionValues;
+import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.NamedList;
@@ -39,6 +44,7 @@ public class StatsValuesFactory {
* @return Instance of StatsValues that will create statistics from values from a field of the given type
*/
public static StatsValues createStatsValues(SchemaField sf) {
+ // TODO: allow for custom field types
FieldType fieldType = sf.getType();
if (DoubleField.class.isInstance(fieldType) ||
IntField.class.isInstance(fieldType) ||
@@ -77,6 +83,8 @@ abstract class AbstractStatsValues<T> im
protected T min;
protected long missing;
protected long count;
+ private ValueSource valueSource;
+ protected FunctionValues values;
// facetField facetValue
protected Map<String, Map<String, StatsValues>> facets = new HashMap<String, Map<String, StatsValues>>();
@@ -121,29 +129,22 @@ abstract class AbstractStatsValues<T> im
}
}
}
-
+
/**
* {@inheritDoc}
*/
@Override
- public void accumulate(BytesRef value) {
- count++;
+ public void accumulate(BytesRef value, int count) {
T typedValue = (T)ft.toObject(sf, value);
- updateMinMax(typedValue, typedValue);
- updateTypeSpecificStats(typedValue);
+ accumulate(typedValue, count);
}
- /**
- * {@inheritDoc}
- */
- @Override
- public void accumulate(BytesRef value, int count) {
+ public void accumulate(T value, int count) {
this.count += count;
- T typedValue = (T)ft.toObject(sf, value);
- updateMinMax(typedValue, typedValue);
- updateTypeSpecificStats(typedValue, count);
+ updateMinMax(value, value);
+ updateTypeSpecificStats(value, count);
}
-
+
/**
* {@inheritDoc}
*/
@@ -194,6 +195,13 @@ abstract class AbstractStatsValues<T> im
return res;
}
+ public void setNextReader(AtomicReaderContext ctx) throws IOException {
+ if (valueSource == null) {
+ valueSource = ft.getValueSource(sf, null);
+ }
+ values = valueSource.getValues(Collections.emptyMap(), ctx);
+ }
+
/**
* Updates the minimum and maximum statistics based on the given values
*
@@ -206,13 +214,6 @@ abstract class AbstractStatsValues<T> im
* Updates the type specific statistics based on the given value
*
* @param value Value the statistics should be updated against
- */
- protected abstract void updateTypeSpecificStats(T value);
-
- /**
- * Updates the type specific statistics based on the given value
- *
- * @param value Value the statistics should be updated against
* @param count Number of times the value is being accumulated
*/
protected abstract void updateTypeSpecificStats(T value, int count);
@@ -246,23 +247,22 @@ class NumericStatsValues extends Abstrac
max = Double.NEGATIVE_INFINITY;
}
- /**
- * {@inheritDoc}
- */
@Override
- public void updateTypeSpecificStats(NamedList stv) {
- sum += ((Number)stv.get("sum")).doubleValue();
- sumOfSquares += ((Number)stv.get("sumOfSquares")).doubleValue();
+ public void accumulate(int docID) {
+ if (values.exists(docID)) {
+ accumulate((Number) values.objectVal(docID), 1);
+ } else {
+ missing();
+ }
}
/**
* {@inheritDoc}
*/
@Override
- public void updateTypeSpecificStats(Number v) {
- double value = v.doubleValue();
- sumOfSquares += (value * value); // for std deviation
- sum += value;
+ public void updateTypeSpecificStats(NamedList stv) {
+ sum += ((Number)stv.get("sum")).doubleValue();
+ sumOfSquares += ((Number)stv.get("sumOfSquares")).doubleValue();
}
/**
@@ -323,23 +323,22 @@ class DateStatsValues extends AbstractSt
super(sf);
}
- /**
- * {@inheritDoc}
- */
@Override
- protected void updateTypeSpecificStats(NamedList stv) {
- sum += ((Date) stv.get("sum")).getTime();
- sumOfSquares += ((Number)stv.get("sumOfSquares")).doubleValue();
+ public void accumulate(int docID) {
+ if (values.exists(docID)) {
+ accumulate((Date) values.objectVal(docID), 1);
+ } else {
+ missing();
+ }
}
/**
* {@inheritDoc}
*/
@Override
- public void updateTypeSpecificStats(Date v) {
- long value = v.getTime();
- sumOfSquares += (value * value); // for std deviation
- sum += value;
+ protected void updateTypeSpecificStats(NamedList stv) {
+ sum += ((Date) stv.get("sum")).getTime();
+ sumOfSquares += ((Number)stv.get("sumOfSquares")).doubleValue();
}
/**
@@ -407,19 +406,20 @@ class StringStatsValues extends Abstract
super(sf);
}
- /**
- * {@inheritDoc}
- */
@Override
- protected void updateTypeSpecificStats(NamedList stv) {
- // No type specific stats
+ public void accumulate(int docID) {
+ if (values.exists(docID)) {
+ accumulate(values.strVal(docID), 1);
+ } else {
+ missing();
+ }
}
/**
* {@inheritDoc}
*/
@Override
- protected void updateTypeSpecificStats(String value) {
+ protected void updateTypeSpecificStats(NamedList stv) {
// No type specific stats
}
Modified: lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/request/SimpleFacets.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/request/SimpleFacets.java?rev=1446985&r1=1446984&r2=1446985&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/request/SimpleFacets.java (original)
+++ lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/request/SimpleFacets.java Sun Feb 17 00:45:22 2013
@@ -17,37 +17,83 @@
package org.apache.solr.request;
-import org.apache.lucene.index.*;
-import org.apache.lucene.search.*;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Date;
+import java.util.EnumSet;
+import java.util.IdentityHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Executor;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.lucene.index.AtomicReader;
+import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.Fields;
+import org.apache.lucene.index.MultiDocsEnum;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.FieldCache;
+import org.apache.lucene.search.Filter;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TermRangeQuery;
import org.apache.lucene.search.grouping.AbstractAllGroupHeadsCollector;
-import org.apache.lucene.search.grouping.term.TermGroupFacetCollector;
import org.apache.lucene.search.grouping.term.TermAllGroupsCollector;
-import org.apache.lucene.util.*;
-import org.apache.lucene.util.packed.PackedInts;
+import org.apache.lucene.search.grouping.term.TermGroupFacetCollector;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.CharsRef;
+import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.OpenBitSet;
+import org.apache.lucene.util.StringHelper;
+import org.apache.lucene.util.UnicodeUtil;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
-import org.apache.solr.common.params.*;
-import org.apache.solr.common.params.FacetParams.FacetRangeOther;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.FacetParams;
import org.apache.solr.common.params.FacetParams.FacetRangeInclude;
+import org.apache.solr.common.params.FacetParams.FacetRangeOther;
+import org.apache.solr.common.params.GroupParams;
+import org.apache.solr.common.params.RequiredSolrParams;
+import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils;
-import org.apache.solr.schema.*;
-import org.apache.solr.search.*;
+import org.apache.solr.handler.component.ResponseBuilder;
+import org.apache.solr.schema.BoolField;
+import org.apache.solr.schema.DateField;
+import org.apache.solr.schema.FieldType;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.schema.SortableDoubleField;
+import org.apache.solr.schema.SortableFloatField;
+import org.apache.solr.schema.SortableIntField;
+import org.apache.solr.schema.SortableLongField;
+import org.apache.solr.schema.TrieField;
+import org.apache.solr.search.BitDocSet;
+import org.apache.solr.search.DocIterator;
+import org.apache.solr.search.DocSet;
+import org.apache.solr.search.Grouping;
+import org.apache.solr.search.HashDocSet;
+import org.apache.solr.search.QParser;
+import org.apache.solr.search.QueryParsing;
+import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.search.SortedIntDocSet;
+import org.apache.solr.search.SyntaxError;
import org.apache.solr.search.grouping.GroupingSpecification;
import org.apache.solr.util.BoundedTreeSet;
import org.apache.solr.util.DateMathParser;
import org.apache.solr.util.DefaultSolrThreadFactory;
-import org.apache.solr.handler.component.ResponseBuilder;
import org.apache.solr.util.LongPriorityQueue;
-import java.io.IOException;
-import java.util.*;
-import java.util.concurrent.Executor;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
/**
* A class that generates simple Facet information for a request.
*
@@ -300,7 +346,8 @@ public class SimpleFacets {
boolean enumMethod = FacetParams.FACET_METHOD_enum.equals(method);
// TODO: default to per-segment or not?
- boolean per_segment = FacetParams.FACET_METHOD_fcs.equals(method);
+ boolean per_segment = FacetParams.FACET_METHOD_fcs.equals(method) // explicit
+ || (ft.getNumericType() != null && sf.hasDocValues()); // numeric doc values are per-segment by default
if (method == null && ft instanceof BoolField) {
// Always use filters for booleans... we know the number of values is very small.
@@ -329,10 +376,18 @@ public class SimpleFacets {
// TODO: future logic could use filters instead of the fieldcache if
// the number of terms in the field is small enough.
if (per_segment) {
- PerSegmentSingleValuedFaceting ps = new PerSegmentSingleValuedFaceting(searcher, docs, field, offset,limit, mincount, missing, sort, prefix);
- Executor executor = threads == 0 ? directExecutor : facetExecutor;
- ps.setNumThreads(threads);
- counts = ps.getFacetCounts(executor);
+ if (ft.getNumericType() != null && !sf.multiValued()) {
+ // force numeric faceting
+ if (prefix != null && !prefix.isEmpty()) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, FacetParams.FACET_PREFIX + " is not supported on numeric types");
+ }
+ counts = NumericFacets.getCounts(searcher, docs, field, offset, limit, mincount, missing, sort);
+ } else {
+ PerSegmentSingleValuedFaceting ps = new PerSegmentSingleValuedFaceting(searcher, docs, field, offset,limit, mincount, missing, sort, prefix);
+ Executor executor = threads == 0 ? directExecutor : facetExecutor;
+ ps.setNumThreads(threads);
+ counts = ps.getFacetCounts(executor);
+ }
} else {
counts = getFieldCacheCounts(searcher, docs, field, offset,limit, mincount, missing, sort, prefix);
}
Modified: lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/request/UnInvertedField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/request/UnInvertedField.java?rev=1446985&r1=1446984&r2=1446985&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/request/UnInvertedField.java (original)
+++ lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/request/UnInvertedField.java Sun Feb 17 00:45:22 2013
@@ -483,13 +483,7 @@ public class UnInvertedField extends Doc
SortedDocValues si;
for (String f : facet) {
SchemaField facet_sf = searcher.getSchema().getField(f);
- try {
- si = FieldCache.DEFAULT.getTermsIndex(searcher.getAtomicReader(), f);
- }
- catch (IOException e) {
- throw new RuntimeException("failed to open field cache for: " + f, e);
- }
- finfo[i] = new FieldFacetStats(f, si, sf, facet_sf, numTermsInField);
+ finfo[i] = new FieldFacetStats(searcher, f, sf, facet_sf);
i++;
}
Modified: lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java?rev=1446985&r1=1446984&r2=1446985&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java (original)
+++ lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java Sun Feb 17 00:45:22 2013
@@ -51,6 +51,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
@@ -103,7 +107,7 @@ public abstract class AbstractSpatialFie
}
@Override
- public Field[] createFields(SchemaField field, Object val, float boost) {
+ public List<StorableField> createFields(SchemaField field, Object val, float boost) {
String shapeStr = null;
Shape shape = null;
if (val instanceof Shape) {
@@ -114,34 +118,22 @@ public abstract class AbstractSpatialFie
}
if (shape == null) {
log.debug("Field {}: null shape for input: {}", field, val);
- return null;
+ return Collections.emptyList();
}
- Field[] indexableFields = null;
+ List<StorableField> result = new ArrayList<StorableField>();
if (field.indexed()) {
T strategy = getStrategy(field.getName());
- indexableFields = strategy.createIndexableFields(shape);
+ result.addAll(Arrays.asList(strategy.createIndexableFields(shape)));
}
- StoredField storedField = null;
if (field.stored()) {
if (shapeStr == null)
shapeStr = shapeToString(shape);
- storedField = new StoredField(field.getName(), shapeStr);
+ result.add(new StoredField(field.getName(), shapeStr));
}
- if (indexableFields == null) {
- if (storedField == null)
- return null;
- return new Field[]{storedField};
- } else {
- if (storedField == null)
- return indexableFields;
- Field[] result = new Field[indexableFields.length+1];
- System.arraycopy(indexableFields,0,result,0,indexableFields.length);
- result[result.length-1] = storedField;
- return result;
- }
+ return result;
}
protected Shape parseShape(String shapeStr) {
Modified: lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/CurrencyField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/CurrencyField.java?rev=1446985&r1=1446984&r2=1446985&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/CurrencyField.java (original)
+++ lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/CurrencyField.java Sun Feb 17 00:45:22 2013
@@ -46,9 +46,11 @@ import javax.xml.xpath.XPathExpressionEx
import javax.xml.xpath.XPathFactory;
import java.io.IOException;
import java.io.InputStream;
+import java.util.ArrayList;
import java.util.Currency;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -145,14 +147,14 @@ public class CurrencyField extends Field
}
@Override
- public StorableField[] createFields(SchemaField field, Object externalVal, float boost) {
+ public List<StorableField> createFields(SchemaField field, Object externalVal, float boost) {
CurrencyValue value = CurrencyValue.parse(externalVal.toString(), defaultCurrency);
- StorableField[] f = new StorableField[field.stored() ? 3 : 2];
+ List<StorableField> f = new ArrayList<StorableField>();
SchemaField amountField = getAmountField(field);
- f[0] = amountField.createField(String.valueOf(value.getAmount()), amountField.indexed() && !amountField.omitNorms() ? boost : 1F);
+ f.add(amountField.createField(String.valueOf(value.getAmount()), amountField.indexed() && !amountField.omitNorms() ? boost : 1F));
SchemaField currencyField = getCurrencyField(field);
- f[1] = currencyField.createField(value.getCurrencyCode(), currencyField.indexed() && !currencyField.omitNorms() ? boost : 1F);
+ f.add(currencyField.createField(value.getCurrencyCode(), currencyField.indexed() && !currencyField.omitNorms() ? boost : 1F));
if (field.stored()) {
org.apache.lucene.document.FieldType customType = new org.apache.lucene.document.FieldType();
@@ -162,7 +164,7 @@ public class CurrencyField extends Field
if (storedValue.indexOf(",") < 0) {
storedValue += "," + defaultCurrency;
}
- f[2] = createField(field.getName(), storedValue, customType, 1F);
+ f.add(createField(field.getName(), storedValue, customType, 1F));
}
return f;
Modified: lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/DateField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/DateField.java?rev=1446985&r1=1446984&r2=1446985&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/DateField.java (original)
+++ lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/DateField.java Sun Feb 17 00:45:22 2013
@@ -435,7 +435,7 @@ public class DateField extends Primitive
@Override
public ValueSource getValueSource(SchemaField field, QParser parser) {
field.checkFieldCacheSource(parser);
- return new DateFieldSource(field.getName(), field.getType());
+ return new DateFieldSource(field.getName(), field);
}
/** DateField specific range query */
@@ -453,11 +453,13 @@ public class DateField extends Primitive
class DateFieldSource extends FieldCacheSource {
// NOTE: this is bad for serialization... but we currently need the fieldType for toInternal()
+ SchemaField sf;
FieldType ft;
- public DateFieldSource(String name, FieldType ft) {
+ public DateFieldSource(String name, SchemaField sf) {
super(name);
- this.ft = ft;
+ this.sf = sf;
+ this.ft = sf.getType();
}
@Override
@@ -475,6 +477,11 @@ class DateFieldSource extends FieldCache
}
@Override
+ public boolean exists(int doc) {
+ return termsIndex.getOrd(doc) >= 0;
+ }
+
+ @Override
public float floatVal(int doc) {
return (float)intVal(doc);
}
@@ -514,7 +521,7 @@ class DateFieldSource extends FieldCache
} else {
final BytesRef br = new BytesRef();
termsIndex.lookupOrd(ord, br);
- return ft.toObject(null, br);
+ return ft.toObject(sf, br);
}
}
Modified: lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/FieldProperties.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/FieldProperties.java?rev=1446985&r1=1446984&r2=1446985&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/FieldProperties.java (original)
+++ lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/FieldProperties.java Sun Feb 17 00:45:22 2013
@@ -50,6 +50,7 @@ public abstract class FieldProperties {
protected final static int OMIT_POSITIONS = 0x00002000;
protected final static int STORE_OFFSETS = 0x00004000;
+ protected final static int DOC_VALUES = 0x00008000;
static final String[] propertyNames = {
"indexed", "tokenized", "stored",
@@ -57,7 +58,7 @@ public abstract class FieldProperties {
"termVectors", "termPositions", "termOffsets",
"multiValued",
"sortMissingFirst","sortMissingLast","required", "omitPositions",
- "storeOffsetsWithPositions"
+ "storeOffsetsWithPositions", "docValues"
};
static final Map<String,Integer> propertyMap = new HashMap<String,Integer>();
Modified: lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/FieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/FieldType.java?rev=1446985&r1=1446984&r2=1446985&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/FieldType.java (original)
+++ lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/FieldType.java Sun Feb 17 00:45:22 2013
@@ -17,14 +17,20 @@
package org.apache.solr.schema;
+import java.io.IOException;
+import java.io.Reader;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.document.Field;
+import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.FieldInfo.IndexOptions;
-import org.apache.lucene.index.GeneralField;
-import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.StorableField;
import org.apache.lucene.index.Term;
import org.apache.lucene.queries.function.ValueSource;
@@ -45,11 +51,6 @@ import org.apache.solr.search.Sorting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.io.Reader;
-import java.util.HashMap;
-import java.util.Map;
-
/**
* Base class for all field types used by an index schema.
*
@@ -120,14 +121,6 @@ public abstract class FieldType extends
}
- protected String getArg(String n, Map<String,String> args) {
- String s = args.remove(n);
- if (s == null) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Missing parameter '"+n+"' for FieldType=" + typeName +args);
- }
- return s;
- }
-
// Handle additional arguments...
void setArgs(IndexSchema schema, Map<String,String> args) {
// default to STORED, INDEXED, OMIT_TF_POSITIONS and MULTIVALUED depending on schema version
@@ -169,11 +162,8 @@ public abstract class FieldType extends
initArgs.remove("positionIncrementGap");
}
- final String postingsFormat = initArgs.get("postingsFormat");
- if (postingsFormat != null) {
- this.postingsFormat = postingsFormat;
- initArgs.remove("postingsFormat");
- }
+ this.postingsFormat = initArgs.remove("postingsFormat");
+ this.docValuesFormat = initArgs.remove("docValuesFormat");
if (initArgs.size() > 0) {
throw new RuntimeException("schema fieldtype " + typeName
@@ -261,7 +251,7 @@ public abstract class FieldType extends
newType.setStoreTermVectors(field.storeTermVector());
newType.setStoreTermVectorOffsets(field.storeTermOffsets());
newType.setStoreTermVectorPositions(field.storeTermPositions());
-
+
return createField(field.getName(), val, newType, boost);
}
@@ -290,9 +280,15 @@ public abstract class FieldType extends
* @see #createField(SchemaField, Object, float)
* @see #isPolyField()
*/
- public StorableField[] createFields(SchemaField field, Object value, float boost) {
+ public List<StorableField> createFields(SchemaField field, Object value, float boost) {
StorableField f = createField( field, value, boost);
- return f==null ? new StorableField[]{} : new StorableField[]{f};
+ if (field.hasDocValues() && f.fieldType().docValueType() == null) {
+ // field types that support doc values should either override createField
+ // to return a field with doc values or extend createFields if this can't
+ // be done in a single field instance (see StrField for example)
+ throw new UnsupportedOperationException("This field type does not support doc values: " + this);
+ }
+ return f==null ? Collections.<StorableField>emptyList() : Collections.singletonList(f);
}
protected IndexOptions getIndexOptions(SchemaField field, String internalVal) {
@@ -513,7 +509,13 @@ public abstract class FieldType extends
public Similarity getSimilarity() {
return similarity;
}
-
+
+ /** Return the numeric type of this field, or null if this field is not a
+ * numeric field. */
+ public org.apache.lucene.document.FieldType.NumericType getNumericType() {
+ return null;
+ }
+
/**
* Sets the Similarity used when scoring fields of this type
* @lucene.internal
@@ -530,7 +532,16 @@ public abstract class FieldType extends
public String getPostingsFormat() {
return postingsFormat;
}
-
+
+ /**
+ * The docvalues format used for this field type
+ */
+ protected String docValuesFormat;
+
+ public final String getDocValuesFormat() {
+ return docValuesFormat;
+ }
+
/**
* calls back to TextResponseWriter to write the field value
*/
@@ -562,7 +573,6 @@ public abstract class FieldType extends
return new StrFieldSource(field.name);
}
-
/**
* Returns a Query instance for doing range searches on this field type. {@link org.apache.solr.search.SolrQueryParser}
* currently passes part1 and part2 as null if they are '*' respectively. minInclusive and maxInclusive are both true
@@ -615,7 +625,11 @@ public abstract class FieldType extends
* if invariants are violated by the <code>SchemaField.</code>
* </p>
*/
- public void checkSchemaField(final SchemaField field) throws SolrException {
- // :NOOP:
+ public void checkSchemaField(final SchemaField field) {
+ // override if your field type supports doc values
+ if (field.hasDocValues()) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Field type " + this + " does not support doc values");
+ }
}
+
}
Modified: lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/LatLonType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/LatLonType.java?rev=1446985&r1=1446984&r2=1446985&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/LatLonType.java (original)
+++ lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/LatLonType.java Sun Feb 17 00:45:22 2013
@@ -16,30 +16,42 @@ package org.apache.solr.schema;
* limitations under the License.
*/
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
import org.apache.lucene.document.FieldType;
-import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.StorableField;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.VectorValueSource;
-import org.apache.lucene.search.*;
-import com.spatial4j.core.io.ParseUtils;
-import com.spatial4j.core.context.SpatialContext;
-import com.spatial4j.core.distance.DistanceUtils;
-import com.spatial4j.core.exception.InvalidShapeException;
-import com.spatial4j.core.shape.Rectangle;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.ComplexExplanation;
+import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.Weight;
import org.apache.lucene.util.Bits;
import org.apache.solr.common.SolrException;
import org.apache.solr.response.TextResponseWriter;
-import org.apache.solr.search.*;
-import org.apache.solr.search.function.distance.HaversineConstFunction;
+import org.apache.solr.search.DelegatingCollector;
+import org.apache.solr.search.ExtendedQueryBase;
+import org.apache.solr.search.PostFilter;
+import org.apache.solr.search.QParser;
+import org.apache.solr.search.SpatialOptions;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import com.spatial4j.core.context.SpatialContext;
+import com.spatial4j.core.distance.DistanceUtils;
+import com.spatial4j.core.exception.InvalidShapeException;
+import com.spatial4j.core.io.ParseUtils;
+import com.spatial4j.core.shape.Rectangle;
/**
@@ -57,10 +69,10 @@ public class LatLonType extends Abstract
}
@Override
- public StorableField[] createFields(SchemaField field, Object value, float boost) {
+ public List<StorableField> createFields(SchemaField field, Object value, float boost) {
String externalVal = value.toString();
//we could have tileDiff + 3 fields (two for the lat/lon, one for storage)
- StorableField[] f = new StorableField[(field.indexed() ? 2 : 0) + (field.stored() ? 1 : 0)];
+ List<StorableField> f = new ArrayList<StorableField>(3);
if (field.indexed()) {
int i = 0;
double[] latLon;
@@ -71,18 +83,18 @@ public class LatLonType extends Abstract
}
//latitude
SchemaField lat = subField(field, i);
- f[i] = lat.createField(String.valueOf(latLon[LAT]), lat.indexed() && !lat.omitNorms() ? boost : 1f);
+ f.add(lat.createField(String.valueOf(latLon[LAT]), lat.indexed() && !lat.omitNorms() ? boost : 1f));
i++;
//longitude
SchemaField lon = subField(field, i);
- f[i] = lon.createField(String.valueOf(latLon[LON]), lon.indexed() && !lon.omitNorms() ? boost : 1f);
+ f.add(lon.createField(String.valueOf(latLon[LON]), lon.indexed() && !lon.omitNorms() ? boost : 1f));
}
if (field.stored()) {
FieldType customType = new FieldType();
customType.setStored(true);
- f[f.length - 1] = createField(field.getName(), externalVal, customType, 1f);
+ f.add(createField(field.getName(), externalVal, customType, 1f));
}
return f;
}
Modified: lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/PointType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/PointType.java?rev=1446985&r1=1446984&r2=1446985&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/PointType.java (original)
+++ lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/PointType.java Sun Feb 17 00:45:22 2013
@@ -69,7 +69,7 @@ public class PointType extends Coordinat
}
@Override
- public StorableField[] createFields(SchemaField field, Object value, float boost) {
+ public List<StorableField> createFields(SchemaField field, Object value, float boost) {
String externalVal = value.toString();
String[] point = new String[0];
try {
@@ -79,12 +79,12 @@ public class PointType extends Coordinat
}
// TODO: this doesn't currently support polyFields as sub-field types
- StorableField[] f = new StorableField[ (field.indexed() ? dimension : 0) + (field.stored() ? 1 : 0) ];
+ List<StorableField> f = new ArrayList<StorableField>(dimension+1);
if (field.indexed()) {
for (int i=0; i<dimension; i++) {
SchemaField sf = subField(field, i);
- f[i] = sf.createField(point[i], sf.indexed() && !sf.omitNorms() ? boost : 1f);
+ f.add(sf.createField(point[i], sf.indexed() && !sf.omitNorms() ? boost : 1f));
}
}
@@ -92,7 +92,7 @@ public class PointType extends Coordinat
String storedVal = externalVal; // normalize or not?
FieldType customType = new FieldType();
customType.setStored(true);
- f[f.length - 1] = createField(field.getName(), storedVal, customType, 1f);
+ f.add(createField(field.getName(), storedVal, customType, 1f));
}
return f;
Modified: lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/SchemaField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/SchemaField.java?rev=1446985&r1=1446984&r2=1446985&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/SchemaField.java (original)
+++ lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/SchemaField.java Sun Feb 17 00:45:22 2013
@@ -18,13 +18,13 @@
package org.apache.solr.schema;
import org.apache.solr.common.SolrException;
-import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.StorableField;
import org.apache.lucene.search.SortField;
import org.apache.solr.search.QParser;
import org.apache.solr.response.TextResponseWriter;
+import java.util.List;
import java.util.Map;
import java.io.IOException;
@@ -79,6 +79,7 @@ public final class SchemaField extends F
public boolean indexed() { return (properties & INDEXED)!=0; }
public boolean stored() { return (properties & STORED)!=0; }
+ public boolean hasDocValues() { return (properties & DOC_VALUES) != 0; }
public boolean storeTermVector() { return (properties & STORE_TERMVECTORS)!=0; }
public boolean storeTermPositions() { return (properties & STORE_TERMPOSITIONS)!=0; }
public boolean storeTermOffsets() { return (properties & STORE_TERMOFFSETS)!=0; }
@@ -104,8 +105,8 @@ public final class SchemaField extends F
public StorableField createField(Object val, float boost) {
return type.createField(this,val,boost);
}
-
- public StorableField[] createFields(Object val, float boost) {
+
+ public List<StorableField> createFields(Object val, float boost) {
return type.createFields(this,val,boost);
}
@@ -148,9 +149,9 @@ public final class SchemaField extends F
* @see FieldType#getSortField
*/
public void checkSortability() throws SolrException {
- if (! indexed() ) {
+ if (! (indexed() || hasDocValues()) ) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
- "can not sort on unindexed field: "
+ "can not sort on a field which is neither indexed nor has doc values: "
+ getName());
}
if ( multiValued() ) {
@@ -169,9 +170,9 @@ public final class SchemaField extends F
* @see FieldType#getValueSource
*/
public void checkFieldCacheSource(QParser parser) throws SolrException {
- if (! indexed() ) {
+ if (! (indexed() || hasDocValues()) ) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
- "can not use FieldCache on unindexed field: "
+ "can not use FieldCache on a field which is neither indexed nor has doc values: "
+ getName());
}
if ( multiValued() ) {
Modified: lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java?rev=1446985&r1=1446984&r2=1446985&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java (original)
+++ lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java Sun Feb 17 00:45:22 2013
@@ -132,6 +132,11 @@ class SortableDoubleFieldSource extends
}
@Override
+ public boolean exists(int doc) {
+ return termsIndex.getOrd(doc) >= 0;
+ }
+
+ @Override
public float floatVal(int doc) {
return (float)doubleVal(doc);
}
@@ -164,13 +169,7 @@ class SortableDoubleFieldSource extends
@Override
public Object objectVal(int doc) {
- int ord=termsIndex.getOrd(doc);
- if (ord==-1) {
- return null;
- } else {
- termsIndex.lookupOrd(ord, spare);
- return NumberUtils.SortableStr2double(spare);
- }
+ return exists(doc) ? doubleVal(doc) : null;
}
@Override
Modified: lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java?rev=1446985&r1=1446984&r2=1446985&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java (original)
+++ lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java Sun Feb 17 00:45:22 2013
@@ -136,6 +136,11 @@ class SortableFloatFieldSource extends F
}
@Override
+ public boolean exists(int doc) {
+ return termsIndex.getOrd(doc) >= 0;
+ }
+
+ @Override
public float floatVal(int doc) {
int ord=termsIndex.getOrd(doc);
if (ord==-1) {
@@ -173,13 +178,7 @@ class SortableFloatFieldSource extends F
@Override
public Object objectVal(int doc) {
- int ord=termsIndex.getOrd(doc);
- if (ord==-1) {
- return null;
- } else {
- termsIndex.lookupOrd(ord, spare);
- return NumberUtils.SortableStr2float(spare);
- }
+ return exists(doc) ? floatVal(doc) : null;
}
@Override
Modified: lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/SortableIntField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/SortableIntField.java?rev=1446985&r1=1446984&r2=1446985&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/SortableIntField.java (original)
+++ lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/SortableIntField.java Sun Feb 17 00:45:22 2013
@@ -143,6 +143,11 @@ class SortableIntFieldSource extends Fie
}
@Override
+ public boolean exists(int doc) {
+ return termsIndex.getOrd(doc) >= 0;
+ }
+
+ @Override
public int intVal(int doc) {
int ord=termsIndex.getOrd(doc);
if (ord==-1) {
@@ -175,13 +180,7 @@ class SortableIntFieldSource extends Fie
@Override
public Object objectVal(int doc) {
- int ord=termsIndex.getOrd(doc);
- if (ord==-1) {
- return null;
- } else {
- termsIndex.lookupOrd(ord, spare);
- return NumberUtils.SortableStr2int(spare);
- }
+ return exists(doc) ? intVal(doc) : null;
}
@Override
Modified: lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/SortableLongField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/SortableLongField.java?rev=1446985&r1=1446984&r2=1446985&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/SortableLongField.java (original)
+++ lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/schema/SortableLongField.java Sun Feb 17 00:45:22 2013
@@ -136,6 +136,11 @@ class SortableLongFieldSource extends Fi
}
@Override
+ public boolean exists(int doc) {
+ return termsIndex.getOrd(doc) >= 0;
+ }
+
+ @Override
public float floatVal(int doc) {
return (float)longVal(doc);
}
@@ -168,13 +173,7 @@ class SortableLongFieldSource extends Fi
@Override
public Object objectVal(int doc) {
- int ord=termsIndex.getOrd(doc);
- if (ord==-1) {
- return null;
- } else {
- termsIndex.lookupOrd(ord, spare);
- return NumberUtils.SortableStr2long(spare);
- }
+ return exists(doc) ? longVal(doc) : null;
}
@Override