You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by up...@apache.org on 2016/06/15 17:22:10 UTC
[1/2] incubator-geode git commit: GEODE-11: Support indexing values
that are Strings or Numbers
Repository: incubator-geode
Updated Branches:
refs/heads/develop a7b6b25a2 -> f59e8d0b2
GEODE-11: Support indexing values that are Strings or Numbers
Adding support to index values that are strings or numbers, by providing
a special field name LuceneIndex.REGION_VALUE_FIELD that indicates the
entire value should be indexed.
Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/f59e8d0b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/f59e8d0b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/f59e8d0b
Branch: refs/heads/develop
Commit: f59e8d0b22c7ebb4600b8cfa9c1e0b00d3925f05
Parents: 3de21f5
Author: Dan Smith <up...@apache.org>
Authored: Mon Jun 13 16:49:23 2016 -0700
Committer: Dan Smith <up...@apache.org>
Committed: Wed Jun 15 10:09:14 2016 -0700
----------------------------------------------------------------------
.../gemfire/cache/lucene/LuceneService.java | 16 +++++-
.../lucene/internal/LuceneServiceImpl.java | 6 +++
.../HeterogeneousLuceneSerializer.java | 22 +++++++-
.../serializer/PrimitiveSerializer.java | 35 +++++++++++++
.../repository/serializer/SerializerUtil.java | 30 +++++++++--
.../lucene/LuceneQueriesIntegrationTest.java | 13 +++++
.../internal/LuceneServiceImplJUnitTest.java | 53 ++++++++++++++++++++
.../HeterogeneousLuceneSerializerJUnitTest.java | 21 ++++++++
8 files changed, 188 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/f59e8d0b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/LuceneService.java
----------------------------------------------------------------------
diff --git a/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/LuceneService.java b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/LuceneService.java
index 462aa7e..974cf96 100644
--- a/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/LuceneService.java
+++ b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/LuceneService.java
@@ -68,9 +68,20 @@ import com.gemstone.gemfire.cache.GemFireCache;
*/
@Experimental
public interface LuceneService {
-
+
+ /**
+ * A special field name that indicates that the entire region value should
+ * be indexed. This will only work if the region value is a String or Number, in
+ * which case a lucene document will be created with a single field with this name.
+ */
+ String REGION_VALUE_FIELD = "__REGION_VALUE_FIELD";
+
/**
* Create a lucene index using default analyzer.
+ * @param fields The fields of the object to index. Only fields listed here will be stored
+ * in the index. Fields should map to PDX fieldNames if the object is serialized with PDX, or
+ * to java fields on the object otherwise. The special field name {{@link #REGION_VALUE_FIELD}}
+ * indicates that the entire value should be stored as a single field in the index.
*/
public void createIndex(String indexName, String regionPath, String... fields);
@@ -79,7 +90,8 @@ public interface LuceneService {
*
* @param indexName index name
* @param regionPath region name
- * @param analyzerPerField analyzer per field map
+ * @param analyzerPerField A map of fields to analyzers. See {{@link #createIndex(String, String, String...)}}
+ * for details on valid values for fields. Each field will be tokenized using the provided Analyzer.
*/
public void createIndex(String indexName, String regionPath,
Map<String, Analyzer> analyzerPerField);
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/f59e8d0b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/LuceneServiceImpl.java
----------------------------------------------------------------------
diff --git a/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/LuceneServiceImpl.java b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/LuceneServiceImpl.java
index 47c4d76..6bfaa78 100644
--- a/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/LuceneServiceImpl.java
+++ b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/LuceneServiceImpl.java
@@ -102,6 +102,9 @@ public class LuceneServiceImpl implements InternalLuceneService {
@Override
public void createIndex(String indexName, String regionPath, String... fields) {
+ if(fields == null || fields.length == 0) {
+ throw new IllegalArgumentException("At least one field must be indexed");
+ }
StandardAnalyzer analyzer = new StandardAnalyzer();
createIndex(indexName, regionPath, analyzer, null, fields);
@@ -109,6 +112,9 @@ public class LuceneServiceImpl implements InternalLuceneService {
@Override
public void createIndex(String indexName, String regionPath, Map<String, Analyzer> fieldAnalyzers) {
+ if(fieldAnalyzers == null || fieldAnalyzers.isEmpty()) {
+ throw new IllegalArgumentException("At least one field must be indexed");
+ }
Analyzer analyzer = new PerFieldAnalyzerWrapper(new StandardAnalyzer(), fieldAnalyzers);
Set<String> fieldsSet = fieldAnalyzers.keySet();
String[] fields = (String[])fieldsSet.toArray(new String[fieldsSet.size()]);
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/f59e8d0b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/repository/serializer/HeterogeneousLuceneSerializer.java
----------------------------------------------------------------------
diff --git a/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/repository/serializer/HeterogeneousLuceneSerializer.java b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/repository/serializer/HeterogeneousLuceneSerializer.java
index d2b1db1..271f0fd 100644
--- a/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/repository/serializer/HeterogeneousLuceneSerializer.java
+++ b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/repository/serializer/HeterogeneousLuceneSerializer.java
@@ -18,11 +18,14 @@
*/
package com.gemstone.gemfire.cache.lucene.internal.repository.serializer;
+import java.util.Arrays;
import java.util.Map;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.document.Document;
+import com.gemstone.gemfire.cache.lucene.LuceneIndex;
+import com.gemstone.gemfire.cache.lucene.LuceneService;
import com.gemstone.gemfire.internal.logging.LogService;
import com.gemstone.gemfire.internal.util.concurrent.CopyOnWriteWeakHashMap;
import com.gemstone.gemfire.pdx.PdxInstance;
@@ -42,7 +45,7 @@ public class HeterogeneousLuceneSerializer implements LuceneSerializer {
* A mapper for converting a PDX object into a document
*/
private LuceneSerializer pdxMapper;
-
+
/**
* Mappers for each individual class type that this class has seen.
*
@@ -55,8 +58,23 @@ public class HeterogeneousLuceneSerializer implements LuceneSerializer {
public HeterogeneousLuceneSerializer(String[] indexedFields) {
this.indexedFields = indexedFields;
pdxMapper = new PdxLuceneSerializer(indexedFields);
+
+
+ addSerializersForPrimitiveValues();
}
-
+
+ /**
+ * Add serializers for the primitive value types (String, Number, etc.)
+ * if the user has requested that the whole value be serialized
+ */
+ private void addSerializersForPrimitiveValues() {
+ if(Arrays.asList(indexedFields).contains(LuceneService.REGION_VALUE_FIELD)) {
+ final PrimitiveSerializer primitiveSerializer = new PrimitiveSerializer();
+ SerializerUtil.supportedPrimitiveTypes().stream()
+ .forEach(type -> mappers.put(type, primitiveSerializer));
+ }
+ }
+
@Override
public void toDocument(Object value, Document doc) {
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/f59e8d0b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/repository/serializer/PrimitiveSerializer.java
----------------------------------------------------------------------
diff --git a/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/repository/serializer/PrimitiveSerializer.java b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/repository/serializer/PrimitiveSerializer.java
new file mode 100644
index 0000000..d096637
--- /dev/null
+++ b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/repository/serializer/PrimitiveSerializer.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.gemstone.gemfire.cache.lucene.internal.repository.serializer;
+
+import com.gemstone.gemfire.cache.lucene.LuceneService;
+
+import org.apache.lucene.document.Document;
+
+/**
+ * A LuceneSerializer that can serialize a primitive value (String, int, long, double)
+ * by creating a document with a special field containing the value
+ */
+public class PrimitiveSerializer implements LuceneSerializer {
+
+ @Override
+ public void toDocument(final Object value, final Document doc) {
+ SerializerUtil.addField(doc, LuceneService.REGION_VALUE_FIELD, value);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/f59e8d0b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/repository/serializer/SerializerUtil.java
----------------------------------------------------------------------
diff --git a/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/repository/serializer/SerializerUtil.java b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/repository/serializer/SerializerUtil.java
index 0ed9d5d..4d563c1 100644
--- a/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/repository/serializer/SerializerUtil.java
+++ b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/repository/serializer/SerializerUtil.java
@@ -22,6 +22,10 @@ package com.gemstone.gemfire.cache.lucene.internal.repository.serializer;
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.DoublePoint;
@@ -44,6 +48,23 @@ import com.gemstone.gemfire.internal.util.BlobHelper;
*/
public class SerializerUtil {
private static final String KEY_FIELD = "_KEY";
+
+ private static final Set<Class> SUPPORTED_PRIMITIVE_TYPES;
+
+ static {
+ HashSet<Class> primitiveTypes = new HashSet<>();
+ primitiveTypes.add(String.class);
+ primitiveTypes.add(long.class);
+ primitiveTypes.add(int.class);
+ primitiveTypes.add(float.class);
+ primitiveTypes.add(double.class);
+ primitiveTypes.add(Long.class);
+ primitiveTypes.add(Integer.class);
+ primitiveTypes.add(Float.class);
+ primitiveTypes.add(Double.class);
+
+ SUPPORTED_PRIMITIVE_TYPES = Collections.unmodifiableSet(primitiveTypes);
+ }
/**
* A small buffer for converting keys to byte[] arrays.
@@ -97,10 +118,11 @@ public class SerializerUtil {
* Return true if a field type can be written to a lucene document.
*/
public static boolean isSupported(Class<?> type) {
- return type == String.class || type == long.class || type == int.class
- || type == float.class || type == double.class
- || type == Long.class || type == Integer.class
- || type == Float.class || type == Double.class;
+ return SUPPORTED_PRIMITIVE_TYPES.contains(type);
+ }
+
+ public static Collection<Class> supportedPrimitiveTypes() {
+ return SUPPORTED_PRIMITIVE_TYPES;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/f59e8d0b/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/LuceneQueriesIntegrationTest.java
----------------------------------------------------------------------
diff --git a/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/LuceneQueriesIntegrationTest.java b/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/LuceneQueriesIntegrationTest.java
index 587e4db..674bc9c 100644
--- a/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/LuceneQueriesIntegrationTest.java
+++ b/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/LuceneQueriesIntegrationTest.java
@@ -186,6 +186,19 @@ public class LuceneQueriesIntegrationTest extends LuceneIntegrationTest {
}
@Test()
+ public void shouldAllowQueryOnRegionWithStringValue() throws ParseException {
+ luceneService.createIndex(INDEX_NAME, REGION_NAME, LuceneService.REGION_VALUE_FIELD);
+ Region region = cache.createRegionFactory(RegionShortcut.PARTITION)
+ .create(REGION_NAME);
+ final LuceneIndex index = luceneService.getIndex(INDEX_NAME, REGION_NAME);
+
+ region.put("A", "one three");
+ index.waitUntilFlushed(60000);
+
+ verifyQuery("one", LuceneService.REGION_VALUE_FIELD, "A");
+ }
+
+ @Test()
public void throwFunctionExceptionWhenGivenBadQuery() {
LuceneService luceneService = LuceneServiceProvider.get(cache);
luceneService.createIndex(INDEX_NAME, REGION_NAME, "text");
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/f59e8d0b/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/internal/LuceneServiceImplJUnitTest.java
----------------------------------------------------------------------
diff --git a/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/internal/LuceneServiceImplJUnitTest.java b/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/internal/LuceneServiceImplJUnitTest.java
new file mode 100644
index 0000000..3f96323
--- /dev/null
+++ b/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/internal/LuceneServiceImplJUnitTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.gemstone.gemfire.cache.lucene.internal;
+
+import static org.junit.Assert.*;
+
+import java.util.Collections;
+
+import com.gemstone.gemfire.cache.lucene.LuceneService;
+import com.gemstone.gemfire.test.junit.categories.UnitTest;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.ExpectedException;
+
+@Category(UnitTest.class)
+public class LuceneServiceImplJUnitTest {
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ @Test
+ public void shouldThrowIllegalArgumentExceptionIfFieldsAreMissing() {
+ LuceneServiceImpl service = new LuceneServiceImpl();
+ thrown.expect(IllegalArgumentException.class);
+ service.createIndex("index", "region");
+ }
+
+ @Test
+ public void shouldThrowIllegalArgumentExceptionIfFieldsMapIsMissing() {
+ LuceneServiceImpl service = new LuceneServiceImpl();
+ thrown.expect(IllegalArgumentException.class);
+ service.createIndex("index", "region", Collections.emptyMap());
+ }
+
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/f59e8d0b/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/internal/repository/serializer/HeterogeneousLuceneSerializerJUnitTest.java
----------------------------------------------------------------------
diff --git a/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/internal/repository/serializer/HeterogeneousLuceneSerializerJUnitTest.java b/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/internal/repository/serializer/HeterogeneousLuceneSerializerJUnitTest.java
index 3bce904..7b32ab2 100644
--- a/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/internal/repository/serializer/HeterogeneousLuceneSerializerJUnitTest.java
+++ b/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/internal/repository/serializer/HeterogeneousLuceneSerializerJUnitTest.java
@@ -25,6 +25,7 @@ import org.apache.lucene.document.Document;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import com.gemstone.gemfire.cache.lucene.LuceneService;
import com.gemstone.gemfire.pdx.PdxInstance;
import com.gemstone.gemfire.test.junit.categories.UnitTest;
@@ -83,4 +84,24 @@ public class HeterogeneousLuceneSerializerJUnitTest {
assertEquals(5, doc3.getField("i").numericValue());
}
+ @Test
+ public void shouldIndexPrimitiveStringIfRequested() {
+ HeterogeneousLuceneSerializer mapper = new HeterogeneousLuceneSerializer(new String[] {LuceneService.REGION_VALUE_FIELD});
+ Document doc = new Document();
+ mapper.toDocument("sample value", doc);
+ assertEquals(1, doc.getFields().size());
+ assertEquals("sample value", doc.getField(LuceneService.REGION_VALUE_FIELD).stringValue());
+ }
+
+ @Test
+ public void shouldIndexPrimitiveNumberIfRequested() {
+ HeterogeneousLuceneSerializer mapper = new HeterogeneousLuceneSerializer(new String[] {LuceneService.REGION_VALUE_FIELD});
+ Document doc = new Document();
+ mapper.toDocument(53, doc);
+
+ assertEquals(1, doc.getFields().size());
+
+ assertEquals(53, doc.getField(LuceneService.REGION_VALUE_FIELD).numericValue());
+ }
+
}
[2/2] incubator-geode git commit: GEODE-1542: Removing a bogus static
import of javafx class
Posted by up...@apache.org.
GEODE-1542: Removing a bogus static import of javafx class
This class picked up a bad import of a javafx class. With openjdk this
causes compile errors.
Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/3de21f56
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/3de21f56
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/3de21f56
Branch: refs/heads/develop
Commit: 3de21f56a7381cea926be9fe33c8e1a1456c564d
Parents: a7b6b25
Author: Dan Smith <up...@apache.org>
Authored: Wed Jun 15 10:05:24 2016 -0700
Committer: Dan Smith <up...@apache.org>
Committed: Wed Jun 15 10:09:14 2016 -0700
----------------------------------------------------------------------
.../src/main/java/com/gemstone/gemfire/internal/SocketCreator.java | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/3de21f56/geode-core/src/main/java/com/gemstone/gemfire/internal/SocketCreator.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/com/gemstone/gemfire/internal/SocketCreator.java b/geode-core/src/main/java/com/gemstone/gemfire/internal/SocketCreator.java
index 739028c..fded3c3 100755
--- a/geode-core/src/main/java/com/gemstone/gemfire/internal/SocketCreator.java
+++ b/geode-core/src/main/java/com/gemstone/gemfire/internal/SocketCreator.java
@@ -16,8 +16,6 @@
*/
package com.gemstone.gemfire.internal;
-import static javafx.scene.input.KeyCode.L;
-
import com.gemstone.gemfire.GemFireConfigException;
import com.gemstone.gemfire.SystemConnectException;
import com.gemstone.gemfire.SystemFailure;