You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2014/10/01 17:20:49 UTC
svn commit: r1628734 - in /lucene/dev/trunk/solr: ./
core/src/java/org/apache/solr/rest/schema/
core/src/java/org/apache/solr/schema/
core/src/test/org/apache/solr/rest/schema/
core/src/test/org/apache/solr/schema/
Author: noble
Date: Wed Oct 1 15:20:48 2014
New Revision: 1628734
URL: http://svn.apache.org/r1628734
Log:
SOLR-6476
Added:
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SchemaManager.java (with props)
lucene/dev/trunk/solr/core/src/test/org/apache/solr/rest/schema/TestBulkSchemaAPI.java (with props)
lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/TestBulkSchemaConcurrent.java (with props)
lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/TestSchemaManager.java (with props)
Modified:
lucene/dev/trunk/solr/CHANGES.txt
lucene/dev/trunk/solr/core/src/java/org/apache/solr/rest/schema/BaseFieldTypeResource.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/rest/schema/CopyFieldCollectionResource.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/rest/schema/DynamicFieldCollectionResource.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/rest/schema/DynamicFieldResource.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/rest/schema/FieldCollectionResource.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/rest/schema/SchemaResource.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java
Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1628734&r1=1628733&r2=1628734&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Wed Oct 1 15:20:48 2014
@@ -145,6 +145,8 @@ New Features
* SOLR-6565: SolrRequest support for query params (Gregory Chanan)
+* SOLR-6476: Create a bulk mode for schema API (Noble Paul, Steve Rowe)
+
Bug Fixes
----------------------
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/rest/schema/BaseFieldTypeResource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/rest/schema/BaseFieldTypeResource.java?rev=1628734&r1=1628733&r2=1628734&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/rest/schema/BaseFieldTypeResource.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/rest/schema/BaseFieldTypeResource.java Wed Oct 1 15:20:48 2014
@@ -70,7 +70,7 @@ abstract class BaseFieldTypeResource ext
while (!success) {
try {
synchronized (oldSchema.getSchemaUpdateLock()) {
- newSchema = oldSchema.addFieldTypes(newFieldTypes);
+ newSchema = oldSchema.addFieldTypes(newFieldTypes, true);
getSolrCore().setLatestSchema(newSchema);
success = true;
}
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/rest/schema/CopyFieldCollectionResource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/rest/schema/CopyFieldCollectionResource.java?rev=1628734&r1=1628733&r2=1628734&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/rest/schema/CopyFieldCollectionResource.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/rest/schema/CopyFieldCollectionResource.java Wed Oct 1 15:20:48 2014
@@ -173,7 +173,7 @@ public class CopyFieldCollectionResource
while (!success) {
try {
synchronized (oldSchema.getSchemaUpdateLock()) {
- newSchema = oldSchema.addCopyFields(fieldsToCopy);
+ newSchema = oldSchema.addCopyFields(fieldsToCopy,true);
if (null != newSchema) {
getSolrCore().setLatestSchema(newSchema);
success = true;
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/rest/schema/DynamicFieldCollectionResource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/rest/schema/DynamicFieldCollectionResource.java?rev=1628734&r1=1628733&r2=1628734&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/rest/schema/DynamicFieldCollectionResource.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/rest/schema/DynamicFieldCollectionResource.java Wed Oct 1 15:20:48 2014
@@ -178,7 +178,7 @@ public class DynamicFieldCollectionResou
}
firstAttempt = false;
synchronized (oldSchema.getSchemaUpdateLock()) {
- newSchema = oldSchema.addDynamicFields(newDynamicFields, copyFields);
+ newSchema = oldSchema.addDynamicFields(newDynamicFields, copyFields, true);
if (null != newSchema) {
getSolrCore().setLatestSchema(newSchema);
success = true;
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/rest/schema/DynamicFieldResource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/rest/schema/DynamicFieldResource.java?rev=1628734&r1=1628733&r2=1628734&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/rest/schema/DynamicFieldResource.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/rest/schema/DynamicFieldResource.java Wed Oct 1 15:20:48 2014
@@ -32,9 +32,14 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.UnsupportedEncodingException;
+import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.Set;
+
+import static java.util.Collections.singletonList;
+import static java.util.Collections.singletonMap;
/**
* This class responds to requests at /solr/(corename)/schema/dynamicfields/(pattern)
@@ -142,12 +147,12 @@ public class DynamicFieldResource extend
} else {
ManagedIndexSchema oldSchema = (ManagedIndexSchema)getSchema();
Object copies = map.get(IndexSchema.COPY_FIELDS);
- List<String> copyFieldNames = null;
+ Collection<String> copyFieldNames = null;
if (copies != null) {
if (copies instanceof List) {
copyFieldNames = (List<String>)copies;
} else if (copies instanceof String) {
- copyFieldNames = Collections.singletonList(copies.toString());
+ copyFieldNames = singletonList(copies.toString());
} else {
String message = "Invalid '" + IndexSchema.COPY_FIELDS + "' type.";
log.error(message);
@@ -163,7 +168,7 @@ public class DynamicFieldResource extend
try {
SchemaField newDynamicField = oldSchema.newDynamicField(fieldNamePattern, fieldType, map);
synchronized (oldSchema.getSchemaUpdateLock()) {
- newSchema = oldSchema.addDynamicField(newDynamicField, copyFieldNames);
+ newSchema = oldSchema.addDynamicFields(singletonList(newDynamicField), singletonMap(newDynamicField.getName(), copyFieldNames), true);
if (null != newSchema) {
getSolrCore().setLatestSchema(newSchema);
success = true;
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/rest/schema/FieldCollectionResource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/rest/schema/FieldCollectionResource.java?rev=1628734&r1=1628733&r2=1628734&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/rest/schema/FieldCollectionResource.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/rest/schema/FieldCollectionResource.java Wed Oct 1 15:20:48 2014
@@ -199,7 +199,7 @@ public class FieldCollectionResource ext
}
firstAttempt = false;
synchronized (oldSchema.getSchemaUpdateLock()) {
- newSchema = oldSchema.addFields(newFields, copyFields);
+ newSchema = oldSchema.addFields(newFields, copyFields, true);
if (null != newSchema) {
getSolrCore().setLatestSchema(newSchema);
success = true;
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/rest/schema/SchemaResource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/rest/schema/SchemaResource.java?rev=1628734&r1=1628733&r2=1628734&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/rest/schema/SchemaResource.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/rest/schema/SchemaResource.java Wed Oct 1 15:20:48 2014
@@ -16,18 +16,26 @@ package org.apache.solr.rest.schema;
* limitations under the License.
*/
+import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.rest.BaseSolrResource;
import org.apache.solr.rest.GETable;
+import org.apache.solr.rest.POSTable;
import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.SchemaManager;
import org.restlet.representation.Representation;
import org.restlet.resource.ResourceException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.Collections;
+import java.util.List;
+
/**
* This class responds to requests at /solr/(corename)/schema
*/
-public class SchemaResource extends BaseSolrResource implements GETable {
+public class SchemaResource extends BaseSolrResource implements GETable,POSTable {
private static final Logger log = LoggerFactory.getLogger(SchemaResource.class);
public SchemaResource() {
@@ -50,4 +58,25 @@ public class SchemaResource extends Base
return new SolrOutputRepresentation();
}
+
+ @Override
+ public Representation post(Representation representation) {
+ SolrRequestInfo requestInfo = SolrRequestInfo.getRequestInfo();
+ List<String> errs = null;
+ try {
+ String text = representation.getText();
+ errs = new SchemaManager(requestInfo.getReq()).performOperations(new StringReader(text));
+ } catch (IOException e) {
+ requestInfo.getRsp().add("errors", Collections.singletonList("Error reading input String " + e.getMessage()));
+ requestInfo.getRsp().setException(e);
+ }
+ if(!errs.isEmpty()){
+ requestInfo.getRsp().add("errors", errs);
+ }
+
+
+ return new BaseSolrResource.SolrOutputRepresentation();
+ }
+
+
}
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/IndexSchema.java?rev=1628734&r1=1628733&r2=1628734&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/IndexSchema.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/IndexSchema.java Wed Oct 1 15:20:48 2014
@@ -74,6 +74,9 @@ import java.util.TreeMap;
import java.util.TreeSet;
import java.util.regex.Pattern;
+import static java.util.Collections.singletonList;
+import static java.util.Collections.singletonMap;
+
/**
* <code>IndexSchema</code> contains information about the valid fields in an index
* and the types of those fields.
@@ -1473,23 +1476,26 @@ public class IndexSchema {
}
/**
- * Copies this schema, adds the given field to the copy, then persists the
- * new schema. Requires synchronizing on the object returned by
+ * Copies this schema, adds the given field to the copy
+ * Requires synchronizing on the object returned by
* {@link #getSchemaUpdateLock()}.
*
* @param newField the SchemaField to add
+ * @param persist to persist the schema or not or not
* @return a new IndexSchema based on this schema with newField added
* @see #newField(String, String, Map)
*/
+ public IndexSchema addField(SchemaField newField, boolean persist) {
+ return addFields(Collections.singletonList(newField),Collections.EMPTY_MAP,persist );
+ }
+
public IndexSchema addField(SchemaField newField) {
- String msg = "This IndexSchema is not mutable.";
- log.error(msg);
- throw new SolrException(ErrorCode.SERVER_ERROR, msg);
+ return addField(newField, true);
}
/**
- * Copies this schema, adds the given field to the copy, then persists the
- * new schema. Requires synchronizing on the object returned by
+ * Copies this schema, adds the given field to the copy
+ * Requires synchronizing on the object returned by
* {@link #getSchemaUpdateLock()}.
*
* @param newField the SchemaField to add
@@ -1498,14 +1504,12 @@ public class IndexSchema {
* @see #newField(String, String, Map)
*/
public IndexSchema addField(SchemaField newField, Collection<String> copyFieldNames) {
- String msg = "This IndexSchema is not mutable.";
- log.error(msg);
- throw new SolrException(ErrorCode.SERVER_ERROR, msg);
+ return addFields(singletonList(newField), singletonMap(newField.getName(), copyFieldNames), true);
}
/**
- * Copies this schema, adds the given fields to the copy, then persists the
- * new schema. Requires synchronizing on the object returned by
+ * Copies this schema, adds the given fields to the copy.
+ * Requires synchronizing on the object returned by
* {@link #getSchemaUpdateLock()}.
*
* @param newFields the SchemaFields to add
@@ -1513,99 +1517,57 @@ public class IndexSchema {
* @see #newField(String, String, Map)
*/
public IndexSchema addFields(Collection<SchemaField> newFields) {
- String msg = "This IndexSchema is not mutable.";
- log.error(msg);
- throw new SolrException(ErrorCode.SERVER_ERROR, msg);
+ return addFields(newFields, Collections.<String, Collection<String>>emptyMap(), true);
}
/**
- * Copies this schema, adds the given fields to the copy, then persists the
- * new schema. Requires synchronizing on the object returned by
+ * Copies this schema, adds the given fields to the copy
+ * Requires synchronizing on the object returned by
* {@link #getSchemaUpdateLock()}.
*
* @param newFields the SchemaFields to add
* @param copyFieldNames 0 or more names of targets to copy this field to. The target fields must already exist.
+ * @param persist Persist the schema or not
* @return a new IndexSchema based on this schema with newFields added
* @see #newField(String, String, Map)
*/
- public IndexSchema addFields(Collection<SchemaField> newFields, Map<String, Collection<String>> copyFieldNames) {
+ public IndexSchema addFields(Collection<SchemaField> newFields, Map<String, Collection<String>> copyFieldNames, boolean persist) {
String msg = "This IndexSchema is not mutable.";
log.error(msg);
throw new SolrException(ErrorCode.SERVER_ERROR, msg);
}
- /**
- * Copies this schema, adds the given dynamic field to the copy, then persists the
- * new schema. Requires synchronizing on the object returned by
- * {@link #getSchemaUpdateLock()}.
- *
- * @param newDynamicField the SchemaField to add
- * @return a new IndexSchema based on this schema with newField added
- * @see #newDynamicField(String, String, Map)
- */
- public IndexSchema addDynamicField(SchemaField newDynamicField) {
- String msg = "This IndexSchema is not mutable.";
- log.error(msg);
- throw new SolrException(ErrorCode.SERVER_ERROR, msg);
- }
/**
- * Copies this schema, adds the given dynamic field to the copy, then persists the
- * new schema. Requires synchronizing on the object returned by
- * {@link #getSchemaUpdateLock()}.
- *
- * @param newDynamicField the SchemaField to add
- * @param copyFieldNames 0 or more names of targets to copy this field to. The targets must already exist.
- * @return a new IndexSchema based on this schema with newDynamicField added
- * @see #newDynamicField(String, String, Map)
- */
- public IndexSchema addDynamicField(SchemaField newDynamicField, Collection<String> copyFieldNames) {
- String msg = "This IndexSchema is not mutable.";
- log.error(msg);
- throw new SolrException(ErrorCode.SERVER_ERROR, msg);
- }
-
- /**
- * Copies this schema, adds the given dynamic fields to the copy, then persists the
- * new schema. Requires synchronizing on the object returned by
- * {@link #getSchemaUpdateLock()}.
- *
- * @param newDynamicFields the SchemaFields to add
- * @return a new IndexSchema based on this schema with newDynamicFields added
- * @see #newDynamicField(String, String, Map)
- */
- public IndexSchema addDynamicFields(Collection<SchemaField> newDynamicFields) {
- String msg = "This IndexSchema is not mutable.";
- log.error(msg);
- throw new SolrException(ErrorCode.SERVER_ERROR, msg);
- }
-
- /**
- * Copies this schema, adds the given dynamic fields to the copy, then persists the
- * new schema. Requires synchronizing on the object returned by
+ * Copies this schema, adds the given dynamic fields to the copy,
+ * Requires synchronizing on the object returned by
* {@link #getSchemaUpdateLock()}.
*
* @param newDynamicFields the SchemaFields to add
* @param copyFieldNames 0 or more names of targets to copy this field to. The target fields must already exist.
+ * @param persist to persist the schema or not or not
* @return a new IndexSchema based on this schema with newDynamicFields added
* @see #newDynamicField(String, String, Map)
*/
public IndexSchema addDynamicFields
- (Collection<SchemaField> newDynamicFields, Map<String, Collection<String>> copyFieldNames) {
+ (Collection<SchemaField> newDynamicFields,
+ Map<String, Collection<String>> copyFieldNames,
+ boolean persist) {
String msg = "This IndexSchema is not mutable.";
log.error(msg);
throw new SolrException(ErrorCode.SERVER_ERROR, msg);
}
/**
- * Copies this schema and adds the new copy fields to the copy, then
- * persists the new schema. Requires synchronizing on the object returned by
+ * Copies this schema and adds the new copy fields to the copy
+ * Requires synchronizing on the object returned by
* {@link #getSchemaUpdateLock()}.
*
* @param copyFields Key is the name of the source field name, value is a collection of target field names. Fields must exist.
+ * @param persist to persist the schema or not or not
* @return The new Schema with the copy fields added
*/
- public IndexSchema addCopyFields(Map<String, Collection<String>> copyFields){
+ public IndexSchema addCopyFields(Map<String, Collection<String>> copyFields, boolean persist){
String msg = "This IndexSchema is not mutable.";
log.error(msg);
throw new SolrException(ErrorCode.SERVER_ERROR, msg);
@@ -1660,30 +1622,16 @@ public class IndexSchema {
}
/**
- * Copies this schema, adds the given field type to the copy, then persists the
- * new schema. Requires synchronizing on the object returned by
- * {@link #getSchemaUpdateLock()}.
- *
- * @param fieldType the FieldType to add
- * @return a new IndexSchema based on this schema with the new FieldType added
- * @see #newFieldType(String, String, Map)
- */
- public IndexSchema addFieldType(FieldType fieldType) {
- String msg = "This IndexSchema is not mutable.";
- log.error(msg);
- throw new SolrException(ErrorCode.SERVER_ERROR, msg);
- }
-
- /**
- * Copies this schema, adds the given field type to the copy, then persists the
- * new schema. Requires synchronizing on the object returned by
+ * Copies this schema, adds the given field type to the copy,
+ * Requires synchronizing on the object returned by
* {@link #getSchemaUpdateLock()}.
*
* @param fieldTypeList a list of FieldTypes to add
+ * @param persist to persist the schema or not or not
* @return a new IndexSchema based on this schema with the new types added
* @see #newFieldType(String, String, Map)
*/
- public IndexSchema addFieldTypes(List<FieldType> fieldTypeList) {
+ public IndexSchema addFieldTypes(List<FieldType> fieldTypeList, boolean persist) {
String msg = "This IndexSchema is not mutable.";
log.error(msg);
throw new SolrException(ErrorCode.SERVER_ERROR, msg);
@@ -1692,13 +1640,13 @@ public class IndexSchema {
/**
* Returns a FieldType if the given typeName does not already
* exist in this schema. The resulting FieldType can be used in a call
- * to {@link #addFieldType(FieldType)}.
+ * to {@link #addFieldTypes(java.util.List, boolean)}.
*
* @param typeName the name of the type to add
* @param className the name of the FieldType class
* @param options the options to use when creating the FieldType
* @return The created FieldType
- * @see #addFieldType(FieldType)
+ * @see #addFieldTypes(java.util.List, boolean)
*/
public FieldType newFieldType(String typeName, String className, Map<String,?> options) {
String msg = "This IndexSchema is not mutable.";
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java?rev=1628734&r1=1628733&r2=1628734&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java Wed Oct 1 15:20:48 2014
@@ -80,6 +80,9 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
+import static java.util.Collections.singletonList;
+import static java.util.Collections.singletonMap;
+
/** Solr-managed schema - non-user-editable, but can be mutable via internal and external REST API requests. */
public final class ManagedIndexSchema extends IndexSchema {
@@ -386,23 +389,11 @@ public final class ManagedIndexSchema ex
}
}
- @Override
- public ManagedIndexSchema addField(SchemaField newField) {
- return addFields(Arrays.asList(newField));
- }
-
- @Override
- public ManagedIndexSchema addField(SchemaField newField, Collection<String> copyFieldNames) {
- return addFields(Arrays.asList(newField), Collections.singletonMap(newField.getName(), copyFieldNames));
- }
@Override
- public ManagedIndexSchema addFields(Collection<SchemaField> newFields) {
- return addFields(newFields, Collections.<String, Collection<String>>emptyMap());
- }
-
- @Override
- public ManagedIndexSchema addFields(Collection<SchemaField> newFields, Map<String, Collection<String>> copyFieldNames) {
+ public ManagedIndexSchema addFields(Collection<SchemaField> newFields,
+ Map<String, Collection<String>> copyFieldNames,
+ boolean persist) {
ManagedIndexSchema newSchema = null;
if (isMutable) {
boolean success = false;
@@ -439,12 +430,15 @@ public final class ManagedIndexSchema ex
aware.inform(newSchema);
}
newSchema.refreshAnalyzers();
- success = newSchema.persistManagedSchema(false); // don't just create - update it if it already exists
- if (success) {
- log.debug("Added field(s): {}", newFields);
- } else {
- log.error("Failed to add field(s): {}", newFields);
- newSchema = null;
+
+ if(persist) {
+ success = newSchema.persistManagedSchema(false); // don't just create - update it if it already exists
+ if (success) {
+ log.debug("Added field(s): {}", newFields);
+ } else {
+ log.error("Failed to add field(s): {}", newFields);
+ newSchema = null;
+ }
}
} else {
String msg = "This ManagedIndexSchema is not mutable.";
@@ -454,25 +448,10 @@ public final class ManagedIndexSchema ex
return newSchema;
}
- @Override
- public IndexSchema addDynamicField(SchemaField newDynamicField) {
- return addDynamicFields(Arrays.asList(newDynamicField));
- }
-
- @Override
- public IndexSchema addDynamicField(SchemaField newDynamicField, Collection<String> copyFieldNames) {
- return addDynamicFields(Arrays.asList(newDynamicField),
- Collections.singletonMap(newDynamicField.getName(), copyFieldNames));
- }
-
- @Override
- public ManagedIndexSchema addDynamicFields(Collection<SchemaField> newDynamicFields) {
- return addDynamicFields(newDynamicFields, Collections.<String,Collection<String>>emptyMap());
- }
@Override
public ManagedIndexSchema addDynamicFields(Collection<SchemaField> newDynamicFields,
- Map<String,Collection<String>> copyFieldNames) {
+ Map<String,Collection<String>> copyFieldNames, boolean persist) {
ManagedIndexSchema newSchema = null;
if (isMutable) {
boolean success = false;
@@ -489,7 +468,7 @@ public final class ManagedIndexSchema ex
}
dFields.add(new DynamicField(newDynamicField));
newSchema.dynamicFields = dynamicFieldListToSortedArray(dFields);
-
+
Collection<String> copyFields = copyFieldNames.get(newDynamicField.getName());
if (copyFields != null) {
for (String copyField : copyFields) {
@@ -503,11 +482,13 @@ public final class ManagedIndexSchema ex
aware.inform(newSchema);
}
newSchema.refreshAnalyzers();
- success = newSchema.persistManagedSchema(false); // don't just create - update it if it already exists
- if (success) {
- log.debug("Added dynamic field(s): {}", newDynamicFields);
- } else {
- log.error("Failed to add dynamic field(s): {}", newDynamicFields);
+ if(persist) {
+ success = newSchema.persistManagedSchema(false); // don't just create - update it if it already exists
+ if (success) {
+ log.debug("Added dynamic field(s): {}", newDynamicFields);
+ } else {
+ log.error("Failed to add dynamic field(s): {}", newDynamicFields);
+ }
}
} else {
String msg = "This ManagedIndexSchema is not mutable.";
@@ -518,7 +499,7 @@ public final class ManagedIndexSchema ex
}
@Override
- public ManagedIndexSchema addCopyFields(Map<String, Collection<String>> copyFields) {
+ public ManagedIndexSchema addCopyFields(Map<String, Collection<String>> copyFields, boolean persist) {
ManagedIndexSchema newSchema = null;
if (isMutable) {
boolean success = false;
@@ -536,21 +517,19 @@ public final class ManagedIndexSchema ex
aware.inform(newSchema);
}
newSchema.refreshAnalyzers();
- success = newSchema.persistManagedSchema(false); // don't just create - update it if it already exists
- if (success) {
- log.debug("Added copy fields for {} sources", copyFields.size());
- } else {
- log.error("Failed to add copy fields for {} sources", copyFields.size());
+ if(persist) {
+ success = newSchema.persistManagedSchema(false); // don't just create - update it if it already exists
+ if (success) {
+ log.debug("Added copy fields for {} sources", copyFields.size());
+ } else {
+ log.error("Failed to add copy fields for {} sources", copyFields.size());
+ }
}
}
return newSchema;
}
-
- public ManagedIndexSchema addFieldType(FieldType fieldType) {
- return addFieldTypes(Collections.singletonList(fieldType));
- }
- public ManagedIndexSchema addFieldTypes(List<FieldType> fieldTypeList) {
+ public ManagedIndexSchema addFieldTypes(List<FieldType> fieldTypeList, boolean persist) {
if (!isMutable) {
String msg = "This ManagedIndexSchema is not mutable.";
log.error(msg);
@@ -611,24 +590,26 @@ public final class ManagedIndexSchema ex
}
newSchema.refreshAnalyzers();
-
- boolean success = newSchema.persistManagedSchema(false);
- if (success) {
- if (log.isDebugEnabled()) {
- StringBuilder fieldTypeNames = new StringBuilder();
- for (int i=0; i < fieldTypeList.size(); i++) {
- if (i > 0) fieldTypeNames.append(", ");
- fieldTypeNames.append(fieldTypeList.get(i).typeName);
+
+ if (persist) {
+ boolean success = newSchema.persistManagedSchema(false);
+ if (success) {
+ if (log.isDebugEnabled()) {
+ StringBuilder fieldTypeNames = new StringBuilder();
+ for (int i=0; i < fieldTypeList.size(); i++) {
+ if (i > 0) fieldTypeNames.append(", ");
+ fieldTypeNames.append(fieldTypeList.get(i).typeName);
+ }
+ log.debug("Added field types: {}", fieldTypeNames.toString());
}
- log.debug("Added field types: {}", fieldTypeNames.toString());
+ } else {
+ // this is unlikely to happen as most errors are handled as exceptions in the persist code
+ log.error("Failed to add field types: {}", fieldTypeList);
+ throw new SolrException(ErrorCode.SERVER_ERROR,
+ "Failed to persist updated schema due to underlying storage issue; check log for more details!");
}
- } else {
- // this is unlikely to happen as most errors are handled as exceptions in the persist code
- log.error("Failed to add field types: {}", fieldTypeList);
- throw new SolrException(ErrorCode.SERVER_ERROR,
- "Failed to persist updated schema due to underlying storage issue; check log for more details!");
}
-
+
return newSchema;
}
@@ -834,7 +815,7 @@ public final class ManagedIndexSchema ex
* are copied; otherwise, they are not.
* @return A shallow copy of this schema
*/
- private ManagedIndexSchema shallowCopy(boolean includeFieldDataStructures) {
+ ManagedIndexSchema shallowCopy(boolean includeFieldDataStructures) {
ManagedIndexSchema newSchema = null;
try {
newSchema = new ManagedIndexSchema
Added: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SchemaManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SchemaManager.java?rev=1628734&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SchemaManager.java (added)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SchemaManager.java Wed Oct 1 15:20:48 2014
@@ -0,0 +1,380 @@
+package org.apache.solr.schema;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+import org.apache.solr.cloud.ZkSolrResourceLoader;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.core.CoreDescriptor;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.rest.BaseSolrResource;
+import org.noggit.JSONParser;
+import org.noggit.ObjectBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static java.util.Collections.EMPTY_LIST;
+import static java.util.Collections.EMPTY_MAP;
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.singletonList;
+import static java.util.Collections.singletonMap;
+import static org.apache.solr.common.cloud.ZkNodeProps.makeMap;
+import static org.apache.solr.schema.FieldType.CLASS_NAME;
+import static org.apache.solr.schema.IndexSchema.DESTINATION;
+import static org.apache.solr.schema.IndexSchema.NAME;
+import static org.apache.solr.schema.IndexSchema.SOURCE;
+import static org.apache.solr.schema.IndexSchema.TYPE;
+
+/**A utility class to manipulate schema using the bulk mode.
+ * This class takes in all the commands and process them completely. It is an all or none
+ * operation
+ */
+public class SchemaManager {
+ private static final Logger log = LoggerFactory.getLogger(SchemaManager.class);
+
+ final SolrQueryRequest req;
+ ManagedIndexSchema managedIndexSchema;
+
+ public static final String ADD_FIELD = "add-field";
+ public static final String ADD_COPY_FIELD = "add-copy-field";
+ public static final String ADD_DYNAMIC_FIELD = "add-dynamic-field";
+ public static final String ADD_FIELD_TYPE = "add-field-type";
+
+ private static final Set<String> KNOWN_OPS = new HashSet<>();
+ static {
+ KNOWN_OPS.add(ADD_COPY_FIELD);
+ KNOWN_OPS.add(ADD_FIELD);
+ KNOWN_OPS.add(ADD_DYNAMIC_FIELD);
+ KNOWN_OPS.add(ADD_FIELD_TYPE);
+ }
+
+ public SchemaManager(SolrQueryRequest req){
+ this.req = req;
+
+ }
+
+ /**Take in a JSON command set and execute them . It tries to capture as many errors
+ * as possible instead of failing at the frst error it encounters
+ * @param rdr The input as a Reader
+ * @return Lis of errors . If the List is empty then the operation is successful.
+ */
+ public List performOperations(Reader rdr) {
+ List<Operation> ops = null;
+ try {
+ ops = SchemaManager.parse(rdr);
+ } catch (Exception e) {
+ String msg= "Error parsing schema operations ";
+ log.warn(msg ,e );
+ return Collections.singletonList(singletonMap(ERR_MSGS, msg + ":" + e.getMessage()));
+ }
+ List errs = captureErrors(ops);
+ if(!errs.isEmpty()) return errs;
+
+ IndexSchema schema = req.getCore().getLatestSchema();
+ if (!(schema instanceof ManagedIndexSchema)) {
+ return singletonList( singletonMap(ERR_MSGS,"schema is not editable"));
+ }
+
+ synchronized (schema.getSchemaUpdateLock()) {
+ return doOperations(ops);
+ }
+
+ }
+
+ private List<String> doOperations(List<Operation> operations){
+ int timeout = req.getParams().getInt(BaseSolrResource.UPDATE_TIMEOUT_SECS, -1);
+ long startTime = System.nanoTime();
+ long endTime = timeout >0 ? System.nanoTime()+ (timeout * 1000*1000) : Long.MAX_VALUE;
+ SolrCore core = req.getCore();
+ for(;System.nanoTime() < endTime ;) {
+ managedIndexSchema = (ManagedIndexSchema) core.getLatestSchema();
+ for (Operation op : operations) {
+ if (ADD_FIELD.equals(op.name) || ADD_DYNAMIC_FIELD.equals(op.name)) {
+ applyAddField(op);
+ } else if(ADD_COPY_FIELD.equals(op.name)) {
+ applyAddCopyField(op);
+ } else if(ADD_FIELD_TYPE.equals(op.name)) {
+ applyAddType(op);
+
+ } else {
+ op.addError("No such operation : " + op.name);
+ }
+ }
+ List errs = captureErrors(operations);
+ if (!errs.isEmpty()) return errs;
+
+ try {
+ managedIndexSchema.persistManagedSchema(false);
+ core.setLatestSchema(managedIndexSchema);
+ waitForOtherReplicasToUpdate(timeout, startTime);
+ return EMPTY_LIST;
+ } catch (ManagedIndexSchema.SchemaChangedInZkException e) {
+ String s = "Failed to update schema because schema is modified";
+ log.warn(s, e);
+ continue;
+ } catch (Exception e){
+ String s = "Exception persisting schema";
+ log.warn(s, e);
+ return singletonList(s + e.getMessage());
+ }
+ }
+
+ return singletonList("Unable to persist schema");
+
+ }
+
+ private void waitForOtherReplicasToUpdate(int timeout, long startTime) {
+ if(timeout > 0 && managedIndexSchema.getResourceLoader()instanceof ZkSolrResourceLoader){
+ CoreDescriptor cd = req.getCore().getCoreDescriptor();
+ String collection = cd.getCollectionName();
+ if (collection != null) {
+ ZkSolrResourceLoader zkLoader = (ZkSolrResourceLoader) managedIndexSchema.getResourceLoader();
+ long timeLeftSecs1 = timeout - ((System.nanoTime() - startTime) /1000000);
+ int secsLeft = (int) (timeLeftSecs1 > 0 ? timeLeftSecs1 : -1);
+ if(secsLeft<=0) throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Not enough time left to update replicas. However the schema is updated already");
+ long timeLeftSecs = timeout - ((System.nanoTime() - startTime) /1000000);
+ ManagedIndexSchema.waitForSchemaZkVersionAgreement(collection,
+ cd.getCloudDescriptor().getCoreNodeName(),
+ (managedIndexSchema).getSchemaZkVersion(),
+ zkLoader.getZkController(),
+ (int) (timeLeftSecs > 0 ? timeLeftSecs : -1));
+ }
+
+ }
+ }
+
+ private boolean applyAddType(Operation op) {
+ String name = op.getStr(NAME);
+ String clz = op.getStr(CLASS_NAME);
+ if(op.hasError())
+ return false;
+ try {
+ FieldType fieldType = managedIndexSchema.newFieldType(name, clz, (Map<String, ?>) op.commandData);
+ managedIndexSchema = managedIndexSchema.addFieldTypes(singletonList(fieldType), false);
+ return true;
+ } catch (Exception e) {
+ op.addError(getErrorStr(e));
+ return false;
+ }
+ }
+
+ private String getErrorStr(Exception e) {
+ StringBuilder sb = new StringBuilder();
+ Throwable cause= e;
+ for(int i =0;i<5;i++) {
+ sb.append(cause.getMessage()).append("\n");
+ if(cause.getCause() == null || cause.getCause() == cause) break;
+ cause = cause.getCause();
+ }
+ return sb.toString();
+ }
+
+ private boolean applyAddCopyField(Operation op) {
+ String src = op.getStr(SOURCE);
+ List<String> dest = op.getStrs(DESTINATION);
+ if(op.hasError())
+ return false;
+ try {
+ managedIndexSchema = managedIndexSchema.addCopyFields(Collections.<String,Collection<String>>singletonMap(src,dest), false);
+ return true;
+ } catch (Exception e) {
+ op.addError(getErrorStr(e));
+ return false;
+ }
+ }
+
+
+ private boolean applyAddField( Operation op) {
+ String name = op.getStr(NAME);
+ String type = op.getStr(TYPE);
+ if(op.hasError())
+ return false;
+ FieldType ft = managedIndexSchema.getFieldTypeByName(type);
+ if(ft==null){
+ op.addError("No such field type '"+type+"'");
+ return false;
+ }
+ try {
+ if(ADD_DYNAMIC_FIELD.equals(op.name)){
+ managedIndexSchema = managedIndexSchema.addDynamicFields(
+ singletonList(SchemaField.create(name, ft, op.getValuesExcluding(NAME, TYPE))),
+ EMPTY_MAP,false);
+ } else {
+ managedIndexSchema = managedIndexSchema.addFields(
+ singletonList( SchemaField.create(name, ft, op.getValuesExcluding(NAME, TYPE))),
+ EMPTY_MAP,
+ false);
+ }
+ } catch (Exception e) {
+ op.addError(getErrorStr(e));
+ return false;
+ }
+ return true;
+ }
+
+
+ public static class Operation {
+ public final String name;
+ private Object commandData;//this is most often a map
+ private List<String> errors = new ArrayList<>();
+
+ Operation(String operationName, Object metaData) {
+ commandData = metaData;
+ this.name = operationName;
+ if(!KNOWN_OPS.contains(this.name)) errors.add("Unknown Operation :"+this.name);
+ }
+
+ public String getStr(String key, String def){
+ String s = (String) getMapVal(key);
+ return s == null ? def : s;
+ }
+
+ private Object getMapVal(String key) {
+ if (commandData instanceof Map) {
+ Map metaData = (Map) commandData;
+ return metaData.get(key);
+ } else {
+ String msg= " value has to be an object for operation :"+name;
+ if(!errors.contains(msg)) errors.add(msg);
+ return null;
+ }
+ }
+
+ public List<String> getStrs(String key){
+ List<String> val = getStrs(key, null);
+ if(val == null) errors.add("'"+key + "' is a required field");
+ return val;
+
+ }
+
+ /**Get collection of values for a key. If only one val is present a
+ * single value collection is returned
+ */
+ public List<String> getStrs(String key, List<String> def){
+ Object v = getMapVal(key);
+ if(v == null){
+ return def;
+ } else {
+ if (v instanceof List) {
+ ArrayList<String> l = new ArrayList<>();
+ for (Object o : (List)v) {
+ l.add(String.valueOf(o));
+ }
+ if(l.isEmpty()) return def;
+ return l;
+ } else {
+ return singletonList(String.valueOf(v));
+ }
+ }
+
+ }
+
+ /**Get a required field. If missing it adds to the errors
+ */
+ public String getStr(String key){
+ String s = getStr(key,null);
+ if(s==null) errors.add("'"+key + "' is a required field");
+ return s;
+ }
+
+ private Map errorDetails(){
+ return makeMap(name, commandData, ERR_MSGS, errors);
+ }
+
+ public boolean hasError() {
+ return !errors.isEmpty();
+ }
+
+ public void addError(String s) {
+ errors.add(s);
+ }
+
+ /**Get all the values from the metadata for the command
+ * without the specified keys
+ */
+ public Map getValuesExcluding(String... keys) {
+ getMapVal(null);
+ if(hasError()) return emptyMap();//just to verify the type is Map
+ LinkedHashMap<String, Object> cp = new LinkedHashMap<>((Map<String,?>) commandData);
+ if(keys == null) return cp;
+ for (String key : keys) {
+ cp.remove(key);
+ }
+ return cp;
+ }
+
+
+ public List<String> getErrors() {
+ return errors;
+ }
+ }
+
+ /**Parse the command operations into command objects
+ */
+ static List<Operation> parse(Reader rdr ) throws IOException {
+ JSONParser parser = new JSONParser(rdr);
+
+ ObjectBuilder ob = new ObjectBuilder(parser);
+
+ if(parser.lastEvent() != JSONParser.OBJECT_START) {
+ throw new RuntimeException("The JSON must be an Object of the form {\"command\": {...},...");
+ }
+ List<Operation> operations = new ArrayList<>();
+ for(;;) {
+ int ev = parser.nextEvent();
+ if (ev==JSONParser.OBJECT_END) return operations;
+ Object key = ob.getKey();
+ ev = parser.nextEvent();
+ Object val = ob.getVal();
+ if (val instanceof List) {
+ List list = (List) val;
+ for (Object o : list) {
+ operations.add(new Operation(String.valueOf(key), o));
+ }
+ } else {
+ operations.add(new Operation(String.valueOf(key), val));
+ }
+ }
+
+ }
+
+ static List<Map> captureErrors(List<Operation> ops){
+ List<Map> errors = new ArrayList<>();
+ for (SchemaManager.Operation op : ops) {
+ if(op.hasError()) {
+ errors.add(op.errorDetails());
+ }
+ }
+ return errors;
+ }
+ public static final String ERR_MSGS = "errorMessages";
+
+
+}
Added: lucene/dev/trunk/solr/core/src/test/org/apache/solr/rest/schema/TestBulkSchemaAPI.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/rest/schema/TestBulkSchemaAPI.java?rev=1628734&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/rest/schema/TestBulkSchemaAPI.java (added)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/rest/schema/TestBulkSchemaAPI.java Wed Oct 1 15:20:48 2014
@@ -0,0 +1,236 @@
+package org.apache.solr.rest.schema;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.commons.io.FileUtils;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.schema.SchemaManager;
+import org.apache.solr.util.RestTestBase;
+import org.apache.solr.util.RestTestHarness;
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.junit.After;
+import org.junit.Before;
+import org.noggit.JSONParser;
+import org.noggit.ObjectBuilder;
+import org.restlet.ext.servlet.ServerServlet;
+
+import java.io.File;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+
+public class TestBulkSchemaAPI extends RestTestBase {
+
+ private static File tmpSolrHome;
+ private static File tmpConfDir;
+
+ private static final String collection = "collection1";
+ private static final String confDir = collection + "/conf";
+
+
+ @Before
+ public void before() throws Exception {
+ tmpSolrHome = createTempDir().toFile();
+ tmpConfDir = new File(tmpSolrHome, confDir);
+ FileUtils.copyDirectory(new File(TEST_HOME()), tmpSolrHome.getAbsoluteFile());
+
+ final SortedMap<ServletHolder,String> extraServlets = new TreeMap<>();
+ final ServletHolder solrRestApi = new ServletHolder("SolrSchemaRestApi", ServerServlet.class);
+ solrRestApi.setInitParameter("org.restlet.application", "org.apache.solr.rest.SolrSchemaRestApi");
+ extraServlets.put(solrRestApi, "/schema/*"); // '/schema/*' matches '/schema', '/schema/', and '/schema/whatever...'
+
+ System.setProperty("managed.schema.mutable", "true");
+ System.setProperty("enable.update.log", "false");
+
+ createJettyAndHarness(tmpSolrHome.getAbsolutePath(), "solrconfig-managed-schema.xml", "schema-rest.xml",
+ "/solr", true, extraServlets);
+ }
+
+ @After
+ public void after() throws Exception {
+ if (jetty != null) {
+ jetty.stop();
+ jetty = null;
+ }
+ server = null;
+ restTestHarness = null;
+ }
+
+ public void testMultipleAddFieldWithErrors() throws Exception {
+
+ String payload = SolrTestCaseJ4.json( "{\n" +
+ " 'add-field' : {\n" +
+ " 'name':'a1',\n" +
+ " 'type': 'string1',\n" +
+ " 'stored':true,\n" +
+ " 'indexed':false\n" +
+ " },\n" +
+ " 'add-field' : {\n" +
+ " 'type': 'string',\n" +
+ " 'stored':true,\n" +
+ " 'indexed':true\n" +
+ " }\n" +
+ " \n" +
+ " }");
+
+ String response = restTestHarness.post("/schema?wt=json", payload);
+ Map map = (Map) ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
+ List l = (List) map.get("errors");
+
+ List errorList = (List) ((Map) l.get(0)).get(SchemaManager.ERR_MSGS);
+ assertEquals(1, errorList.size());
+ assertTrue (((String)errorList.get(0)).contains("No such field type"));
+ errorList = (List) ((Map) l.get(1)).get(SchemaManager.ERR_MSGS);
+ assertEquals(1, errorList.size());
+ assertTrue (((String)errorList.get(0)).contains("is a required field"));
+
+ }
+
+
+ public void testMultipleCommands() throws Exception{
+ String payload = "{\n" +
+ " 'add-field' : {\n" +
+ " 'name':'a1',\n" +
+ " 'type': 'string',\n" +
+ " 'stored':true,\n" +
+ " 'indexed':false\n" +
+ " },\n" +
+ " 'add-field' : {\n" +
+ " 'name':'a2',\n" +
+ " 'type': 'string',\n" +
+ " 'stored':true,\n" +
+ " 'indexed':true\n" +
+ " },\n" +
+ " 'add-dynamic-field' : {\n" +
+ " 'name' :'*_lol',\n" +
+ " 'type':'string',\n" +
+ " 'stored':true,\n" +
+ " 'indexed':true\n" +
+ " },\n" +
+ " 'add-copy-field' : {\n" +
+ " 'source' :'a1',\n" +
+ " 'dest':['a2','hello_lol']\n" +
+ " },\n" +
+ " 'add-field-type' : {\n" +
+ " 'name' :'mystr',\n" +
+ " 'class' : 'solr.StrField',\n" +
+ " 'sortMissingLast':'true'\n" +
+ " },\n" +
+ " 'add-field-type' : {" +
+ " 'name' : 'myNewTxtField',\n" +
+ " 'class':'solr.TextField','positionIncrementGap':'100',\n" +
+ " 'analyzer' : {\n" +
+ " 'charFilters':[\n" +
+ " {'class':'solr.PatternReplaceCharFilterFactory','replacement':'$1$1','pattern':'([a-zA-Z])\\\\\\\\1+'}\n" +
+ " ],\n" +
+ " 'tokenizer':{'class':'solr.WhitespaceTokenizerFactory'},\n" +
+ " 'filters':[\n" +
+ " {'class':'solr.WordDelimiterFilterFactory','preserveOriginal':'0'},\n" +
+ " {'class':'solr.StopFilterFactory','words':'stopwords.txt','ignoreCase':'true'},\n" +
+ " {'class':'solr.LowerCaseFilterFactory'},\n" +
+ " {'class':'solr.ASCIIFoldingFilterFactory'},\n" +
+ " {'class':'solr.KStemFilterFactory'}\n" +
+ " ]\n" +
+ " }\n" +
+ " }"+
+ " }";
+
+ RestTestHarness harness = restTestHarness;
+
+
+ String response = harness.post("/schema?wt=json", SolrTestCaseJ4.json( payload));
+
+ Map map = (Map) ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
+ assertNull(response, map.get("errors"));
+
+
+ Map m = getObj(harness, "a1", "fields");
+ assertNotNull("field a1 not created", m);
+
+ assertEquals("string", m.get("type"));
+ assertEquals(Boolean.TRUE, m.get("stored"));
+ assertEquals(Boolean.FALSE, m.get("indexed"));
+
+ m = getObj(harness,"a2", "fields");
+ assertNotNull("field a2 not created", m);
+
+ assertEquals("string", m.get("type"));
+ assertEquals(Boolean.TRUE, m.get("stored"));
+ assertEquals(Boolean.TRUE, m.get("indexed"));
+
+ m = getObj(harness,"*_lol", "dynamicFields");
+ assertNotNull("field *_lol not created",m );
+
+ assertEquals("string", m.get("type"));
+ assertEquals(Boolean.TRUE, m.get("stored"));
+ assertEquals(Boolean.TRUE, m.get("indexed"));
+
+ List l = getCopyFields(harness,"a1");
+ Set s =new HashSet();
+ assertEquals(2,l.size());
+ s.add(((Map) l.get(0)).get("dest"));
+ s.add(((Map) l.get(1)).get("dest"));
+ assertTrue(s.contains("hello_lol"));
+ assertTrue(s.contains("a2"));
+
+ m = getObj(harness,"mystr", "fieldTypes");
+ assertNotNull(m);
+ assertEquals("solr.StrField",m.get("class"));
+ assertEquals("true",String.valueOf(m.get("sortMissingLast")));
+
+ m = getObj(harness,"myNewTxtField", "fieldTypes");
+ assertNotNull(m);
+
+
+ }
+
+ public static Map getObj(RestTestHarness restHarness, String fld, String key) throws Exception {
+ Map map = getRespMap(restHarness);
+ List l = (List) ((Map)map.get("schema")).get(key);
+ for (Object o : l) {
+ Map m = (Map) o;
+ if(fld.equals(m.get("name"))) return m;
+ }
+ return null;
+ }
+
+ public static Map getRespMap(RestTestHarness restHarness) throws Exception {
+ String response = restHarness.query("/schema?wt=json");
+ return (Map) ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
+ }
+
+ public static List getCopyFields(RestTestHarness harness, String src) throws Exception {
+ Map map = getRespMap(harness);
+ List l = (List) ((Map)map.get("schema")).get("copyFields");
+ List result = new ArrayList();
+ for (Object o : l) {
+ Map m = (Map) o;
+ if(src.equals(m.get("source"))) result.add(m);
+ }
+ return result;
+
+ }
+
+
+}
Added: lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/TestBulkSchemaConcurrent.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/TestBulkSchemaConcurrent.java?rev=1628734&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/TestBulkSchemaConcurrent.java (added)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/TestBulkSchemaConcurrent.java Wed Oct 1 15:20:48 2014
@@ -0,0 +1,194 @@
+package org.apache.solr.schema;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.SolrServer;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.util.RESTfulServerProvider;
+import org.apache.solr.util.RestTestHarness;
+import org.noggit.JSONParser;
+import org.noggit.ObjectBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import static java.text.MessageFormat.format;
+import static org.apache.solr.rest.schema.TestBulkSchemaAPI.getCopyFields;
+import static org.apache.solr.rest.schema.TestBulkSchemaAPI.getObj;
+
+public class TestBulkSchemaConcurrent extends AbstractFullDistribZkTestBase {
+ static final Logger log = LoggerFactory.getLogger(TestBulkSchemaConcurrent.class);
+ private List<RestTestHarness> restTestHarnesses = new ArrayList<>();
+
+ private void setupHarnesses() {
+ for (final SolrServer client : clients) {
+ RestTestHarness harness = new RestTestHarness(new RESTfulServerProvider() {
+ @Override
+ public String getBaseURL() {
+ return ((HttpSolrServer)client).getBaseURL();
+ }
+ });
+ restTestHarnesses.add(harness);
+ }
+ }
+ @Override
+ public void doTest() throws Exception {
+
+ final int threadCount = 5;
+ setupHarnesses();
+ Thread[] threads = new Thread[threadCount];
+ final List<List> collectErrors = new ArrayList<>();
+
+
+ for(int i=0;i<threadCount;i++){
+ final int finalI = i;
+ threads[i] = new Thread(){
+ @Override
+ public void run() {
+ try {
+ ArrayList errs = new ArrayList();
+ collectErrors.add(errs);
+ invokeBulkCall(finalI,errs);
+ } catch (IOException e) {
+ e.printStackTrace();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ }
+ };
+
+ threads[i].start();
+ }
+
+ for (Thread thread : threads) thread.join();
+
+ boolean success = true;
+
+ for (List e : collectErrors) {
+ if(!e.isEmpty()){
+ success = false;
+ log.error(e.toString());
+ }
+
+ }
+
+ assertTrue(success);
+
+
+ }
+
+ private void invokeBulkCall(int seed, ArrayList<String> errs) throws Exception {
+ String payload = "{\n" +
+ " 'add-field' : {\n" +
+ " 'name':'replaceFieldA',\n" +
+ " 'type': 'string',\n" +
+ " 'stored':true,\n" +
+ " 'indexed':false\n" +
+ " },\n" +
+ " 'add-dynamic-field' : {\n" +
+ " 'name' :'replaceDynamicField',\n" +
+ " 'type':'string',\n" +
+ " 'stored':true,\n" +
+ " 'indexed':true\n" +
+ " },\n" +
+ " 'add-copy-field' : {\n" +
+ " 'source' :'replaceFieldA',\n" +
+ " 'dest':['replaceDynamicCopyFieldDest']\n" +
+ " },\n" +
+ " 'add-field-type' : {\n" +
+ " 'name' :'myNewFieldTypeName',\n" +
+ " 'class' : 'solr.StrField',\n" +
+ " 'sortMissingLast':'true'\n" +
+ " }\n" +
+ "\n" +
+ " }";
+ String aField = "a" + seed;
+ String dynamicFldName = "*_lol" + seed;
+ String dynamicCopyFldDest = "hello_lol"+seed;
+ String newFieldTypeName = "mystr" + seed;
+
+
+ RestTestHarness publisher = restTestHarnesses.get(r.nextInt(restTestHarnesses.size()));
+ payload = payload.replace("replaceFieldA1", aField);
+
+ payload = payload.replace("replaceDynamicField", dynamicFldName);
+ payload = payload.replace("dynamicFieldLol","lol"+seed);
+
+ payload = payload.replace("replaceDynamicCopyFieldDest",dynamicCopyFldDest);
+ payload = payload.replace("myNewFieldTypeName", newFieldTypeName);
+ String response = publisher.post("/schema?wt=json", SolrTestCaseJ4.json(payload));
+ Map map = (Map) ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
+ Object errors = map.get("errors");
+ if(errors!= null){
+ errs.add(new String(ZkStateReader.toJSON(errors), StandardCharsets.UTF_8));
+ return;
+ }
+
+ //get another node
+ RestTestHarness harness = restTestHarnesses.get(r.nextInt(restTestHarnesses.size()));
+ long startTime = System.nanoTime();
+ boolean success = false;
+ long maxTimeoutMillis = 100000;
+ Set<String> errmessages = new HashSet<>();
+ while ( ! success
+ && TimeUnit.MILLISECONDS.convert(System.nanoTime() - startTime, TimeUnit.NANOSECONDS) < maxTimeoutMillis) {
+ errmessages.clear();
+ Map m = getObj(harness, aField, "fields");
+ if(m== null) errmessages.add(format("field {0} not created", aField));
+
+ m = getObj(harness, dynamicFldName, "dynamicFields");
+ if(m== null) errmessages.add(format("dynamic field {0} not created", dynamicFldName));
+
+ List l = getCopyFields(harness, "a1");
+ if(!checkCopyField(l,aField,dynamicCopyFldDest))
+ errmessages.add(format("CopyField source={0},dest={1} not created" , aField,dynamicCopyFldDest));
+
+ m = getObj(harness, "mystr", "fieldTypes");
+ if(m == null) errmessages.add(format("new type {} not created" , newFieldTypeName));
+ Thread.sleep(10);
+ }
+ if(!errmessages.isEmpty()){
+ errs.addAll(errmessages);
+ }
+ }
+
+ private boolean checkCopyField(List<Map> l, String src, String dest) {
+ if(l == null) return false;
+ for (Map map : l) {
+ if(src.equals(map.get("source")) &&
+ dest.equals(map.get("dest"))) return true;
+ }
+ return false;
+ }
+
+
+}
Added: lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/TestSchemaManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/TestSchemaManager.java?rev=1628734&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/TestSchemaManager.java (added)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/TestSchemaManager.java Wed Oct 1 15:20:48 2014
@@ -0,0 +1,77 @@
+package org.apache.solr.schema;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.List;
+
+public class TestSchemaManager extends SolrTestCaseJ4 {
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ initCore("solrconfig.xml","schema-tiny.xml");
+ }
+
+ @Test
+ public void testParsing() throws IOException {
+ String x = "{\n" +
+ " \"add-field\" : {\n" +
+ " \"name\":\"a\",\n" +
+ " \"type\": \"string\",\n" +
+ " \"stored\":true,\n" +
+ " \"indexed\":false\n" +
+ " },\n" +
+ " \"add-field\" : {\n" +
+ " \"name\":\"b\",\n" +
+ " \"type\": \"string\",\n" +
+ " \"stored\":true,\n" +
+ " \"indexed\":false\n" +
+ " }\n" +
+ "\n" +
+ "}";
+
+ List<SchemaManager.Operation> ops = SchemaManager.parse(new StringReader(x));
+ assertEquals(2,ops.size());
+ assertTrue( SchemaManager.captureErrors(ops).isEmpty());
+
+ x = " {\"add-field\" : [{\n" +
+ " \"name\":\"a1\",\n" +
+ " \"type\": \"string\",\n" +
+ " \"stored\":true,\n" +
+ " \"indexed\":false\n" +
+ " },\n" +
+ " {\n" +
+ " \"name\":\"a2\",\n" +
+ " \"type\": \"string\",\n" +
+ " \"stored\":true,\n" +
+ " \"indexed\":true\n" +
+ " }]\n" +
+ " }";
+ ops = SchemaManager.parse(new StringReader(x));
+ assertEquals(2,ops.size());
+ assertTrue( SchemaManager.captureErrors(ops).isEmpty());
+
+ }
+
+
+
+}