You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by bo...@apache.org on 2018/08/09 16:27:01 UTC
[geode] branch develop updated: GEODE-5534: Forced index key to be
a string when a compressed region contains PDX values
This is an automated email from the ASF dual-hosted git repository.
boglesby pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/geode.git
The following commit(s) were added to refs/heads/develop by this push:
new 6ef68c1 GEODE-5534: Forced index key to be a string when a compressed region contains PDX values
6ef68c1 is described below
commit 6ef68c11fdd49e02ded0bfa2bfc072f96f7ab250
Author: Barry Oglesby <bo...@users.noreply.github.com>
AuthorDate: Thu Aug 9 09:26:56 2018 -0700
GEODE-5534: Forced index key to be a string when a compressed region contains PDX values
---
.../AbstractIndexMaintenanceIntegrationTest.java | 16 ++
.../QueryOnCompressedRegionWithIndexTest.java | 238 +++++++++++++++++++++
.../cache/query/internal/index/AbstractIndex.java | 2 +-
3 files changed, 255 insertions(+), 1 deletion(-)
diff --git a/geode-core/src/integrationTest/java/org/apache/geode/cache/query/internal/index/AbstractIndexMaintenanceIntegrationTest.java b/geode-core/src/integrationTest/java/org/apache/geode/cache/query/internal/index/AbstractIndexMaintenanceIntegrationTest.java
index ec1dd7d..9917dd1 100644
--- a/geode-core/src/integrationTest/java/org/apache/geode/cache/query/internal/index/AbstractIndexMaintenanceIntegrationTest.java
+++ b/geode-core/src/integrationTest/java/org/apache/geode/cache/query/internal/index/AbstractIndexMaintenanceIntegrationTest.java
@@ -29,6 +29,7 @@ import org.apache.geode.cache.query.IndexNameConflictException;
import org.apache.geode.cache.query.QueryService;
import org.apache.geode.cache.query.RegionNotFoundException;
import org.apache.geode.cache.query.data.PortfolioPdx;
+import org.apache.geode.compression.SnappyCompressor;
import org.apache.geode.internal.cache.LocalRegion;
import org.apache.geode.internal.cache.RegionEntry;
import org.apache.geode.pdx.internal.PdxString;
@@ -82,6 +83,21 @@ public abstract class AbstractIndexMaintenanceIntegrationTest {
}
@Test
+ public void setPdxStringFlagWithAPdxStringOnACompressedRegionShouldBeFalse() throws Exception {
+ CacheUtils.startCache();
+ Cache cache = CacheUtils.getCache();
+ cache.createRegionFactory(RegionShortcut.REPLICATE).setCompressor(new SnappyCompressor())
+ .create("portfolios");
+ QueryService qs = cache.getQueryService();
+ AbstractIndex statusIndex =
+ createIndex(qs, "statusIndex", "value.status", "/portfolios.entrySet()");
+
+ statusIndex.setPdxStringFlag(new PdxString("PdxString Key"));
+ assertTrue(statusIndex.isIndexedPdxKeysFlagSet);
+ assertFalse(statusIndex.isIndexOnPdxKeys());
+ }
+
+ @Test
public void whenRemovingRegionEntryFromIndexIfEntryDestroyedIsThrownCorrectlyRemoveFromIndexAndNotThrowException()
throws Exception {
CacheUtils.startCache();
diff --git a/geode-core/src/integrationTest/java/org/apache/geode/cache/query/internal/index/QueryOnCompressedRegionWithIndexTest.java b/geode-core/src/integrationTest/java/org/apache/geode/cache/query/internal/index/QueryOnCompressedRegionWithIndexTest.java
new file mode 100644
index 0000000..6ed14ae
--- /dev/null
+++ b/geode-core/src/integrationTest/java/org/apache/geode/cache/query/internal/index/QueryOnCompressedRegionWithIndexTest.java
@@ -0,0 +1,238 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.geode.cache.query.internal.index;
+
+import static org.apache.geode.cache.RegionShortcut.PARTITION;
+import static org.apache.geode.cache.RegionShortcut.PARTITION_OVERFLOW;
+import static org.apache.geode.cache.RegionShortcut.PARTITION_PERSISTENT;
+import static org.apache.geode.cache.RegionShortcut.PARTITION_PERSISTENT_OVERFLOW;
+import static org.apache.geode.cache.RegionShortcut.PARTITION_REDUNDANT;
+import static org.apache.geode.cache.RegionShortcut.PARTITION_REDUNDANT_OVERFLOW;
+import static org.apache.geode.cache.RegionShortcut.PARTITION_REDUNDANT_PERSISTENT;
+import static org.apache.geode.cache.RegionShortcut.PARTITION_REDUNDANT_PERSISTENT_OVERFLOW;
+import static org.apache.geode.cache.RegionShortcut.REPLICATE;
+import static org.apache.geode.cache.RegionShortcut.REPLICATE_OVERFLOW;
+import static org.apache.geode.cache.RegionShortcut.REPLICATE_PERSISTENT;
+import static org.apache.geode.cache.RegionShortcut.REPLICATE_PERSISTENT_OVERFLOW;
+import static org.apache.geode.distributed.ConfigurationProperties.LOCATORS;
+import static org.apache.geode.distributed.ConfigurationProperties.MCAST_PORT;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.regex.Pattern;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.geode.cache.Cache;
+import org.apache.geode.cache.CacheFactory;
+import org.apache.geode.cache.Region;
+import org.apache.geode.cache.RegionShortcut;
+import org.apache.geode.cache.query.Index;
+import org.apache.geode.cache.query.Query;
+import org.apache.geode.cache.query.SelectResults;
+import org.apache.geode.cache.query.data.Portfolio;
+import org.apache.geode.cache.query.data.PortfolioPdx;
+import org.apache.geode.cache.query.internal.QueryObserverAdapter;
+import org.apache.geode.cache.query.internal.QueryObserverHolder;
+import org.apache.geode.compression.SnappyCompressor;
+import org.apache.geode.test.junit.categories.OQLIndexTest;
+import org.apache.geode.test.junit.runners.CategoryWithParameterizedRunnerFactory;
+
+@Category({OQLIndexTest.class})
+@RunWith(Parameterized.class)
+@Parameterized.UseParametersRunnerFactory(CategoryWithParameterizedRunnerFactory.class)
+public class QueryOnCompressedRegionWithIndexTest {
+
+ private Cache cache;
+
+ private Region region;
+
+ @Parameterized.Parameters(name = "{0}")
+ public static Collection<RegionShortcut> getRegionShortcuts() {
+ List<RegionShortcut> shortcuts = new ArrayList<>();
+ shortcuts.add(PARTITION);
+ shortcuts.add(PARTITION_REDUNDANT);
+ shortcuts.add(PARTITION_PERSISTENT);
+ shortcuts.add(PARTITION_REDUNDANT_PERSISTENT);
+ shortcuts.add(PARTITION_OVERFLOW);
+ shortcuts.add(PARTITION_REDUNDANT_OVERFLOW);
+ shortcuts.add(PARTITION_PERSISTENT_OVERFLOW);
+ shortcuts.add(PARTITION_REDUNDANT_PERSISTENT_OVERFLOW);
+ shortcuts.add(REPLICATE);
+ shortcuts.add(REPLICATE_PERSISTENT);
+ shortcuts.add(REPLICATE_OVERFLOW);
+ shortcuts.add(REPLICATE_PERSISTENT_OVERFLOW);
+ return shortcuts;
+ }
+
+ @Parameterized.Parameter
+ public RegionShortcut shortcut;
+
+ @Before
+ public void createCache() {
+ // Create cache
+ CacheFactory cf = new CacheFactory();
+ cf.set(MCAST_PORT, "0");
+ cf.set(LOCATORS, "");
+ cf.setPdxPersistent(true);
+ this.cache = cf.create();
+
+ // Create region with compression enabled
+ this.region = createRegion("region_" + shortcut, shortcut);
+ }
+
+ @After
+ public void closeCache() {
+ // Destroy the region
+ if (this.region != null) {
+ this.region.destroyRegion();
+ }
+
+ // Destroy the cache
+ if (this.cache != null) {
+ this.cache.close();
+ }
+
+ // Delete backup files
+ deleteBackupFiles();
+ }
+
+ private void deleteBackupFiles() {
+ File backupBaseDir = new File(".");
+ Pattern pattern = Pattern.compile("BACKUP.*");
+ File[] files = backupBaseDir.listFiles((dir, name) -> pattern.matcher(name).matches());
+ for (File file : files) {
+ file.delete();
+ }
+ }
+
+ @Test
+ public void testCreateIndexThenAddEntries() throws Exception {
+ // Create index
+ String indexName = this.region.getName() + "_index";
+ createIndex(indexName, "status", this.region.getFullPath());
+
+ // Load entries
+ int numObjects = 10;
+ loadEntries(this.region, numObjects, false);
+
+ // Execute queries and validate results
+ executeQueriesAndValidateResults(numObjects, indexName);
+ }
+
+ @Test
+ public void testCreateIndexThenAddPdxEntries() throws Exception {
+ // Create index
+ String indexName = this.region.getName() + "_index";
+ createIndex(indexName, "status", this.region.getFullPath());
+
+ // Load entries
+ int numObjects = 10;
+ loadEntries(this.region, numObjects, true);
+
+ // Execute queries and validate results
+ executeQueriesAndValidateResults(numObjects, indexName);
+ }
+
+ @Test
+ public void testAddEntriesThenCreateIndex() throws Exception {
+ // Load entries
+ int numObjects = 10;
+ loadEntries(this.region, numObjects, false);
+
+ // Create index
+ String indexName = this.region.getName() + "_index";
+ createIndex(indexName, "status", this.region.getFullPath());
+
+ // Execute queries and validate results
+ executeQueriesAndValidateResults(numObjects, indexName);
+ }
+
+ @Test
+ public void testAddPdxEntriesThenCreateIndex() throws Exception {
+ // Load entries
+ int numObjects = 10;
+ loadEntries(this.region, numObjects, true);
+
+ // Create index
+ String indexName = this.region.getName() + "_index";
+ createIndex(indexName, "status", this.region.getFullPath());
+
+ // Execute queries and validate results
+ executeQueriesAndValidateResults(numObjects, indexName);
+ }
+
+ private Region createRegion(String regionName, RegionShortcut shortcut) {
+ return this.cache.createRegionFactory(shortcut).setCompressor(new SnappyCompressor())
+ .create(regionName);
+ }
+
+ private Index createIndex(String indexName, String indexedExpression, String regionPath)
+ throws Exception {
+ return this.cache.getQueryService().createIndex(indexName, indexedExpression, regionPath);
+ }
+
+ private void loadEntries(Region region, int numObjects, boolean usePdx) {
+ for (int i = 0; i < numObjects; i++) {
+ region.put(i, usePdx ? new PortfolioPdx(i) : new Portfolio(i));
+ }
+ }
+
+ private void executeQueriesAndValidateResults(int numObjects, String indexName) throws Exception {
+ executeQuery("select * from " + this.region.getFullPath() + " where status = 'inactive'",
+ numObjects / 2, indexName);
+ executeQuery("select * from " + this.region.getFullPath() + " where status = 'active'",
+ numObjects / 2, indexName);
+ executeQuery("select * from " + this.region.getFullPath() + " where status = null", 0,
+ indexName);
+ }
+
+ private void executeQuery(String queryStr, int expectedResults, String indexName)
+ throws Exception {
+ // Set query observer
+ QueryObserverImpl observer = new QueryObserverImpl();
+ QueryObserverHolder.setInstance(observer);
+
+ // Execute query
+ Query query = this.cache.getQueryService().newQuery(queryStr);
+ SelectResults results = (SelectResults) query.execute();
+
+ // Validate results size
+ assertThat(results.size()).isEqualTo(expectedResults);
+
+ // Validate index was used
+ assertThat(observer.wasIndexUsed(indexName)).isTrue();
+ }
+
+ class QueryObserverImpl extends QueryObserverAdapter {
+ List indexesUsed = new ArrayList();
+
+ public void beforeIndexLookup(Index index, int oper, Object key) {
+ indexesUsed.add(index.getName());
+ }
+
+ public boolean wasIndexUsed(String indexName) {
+ return indexesUsed.contains(indexName);
+ }
+ }
+}
diff --git a/geode-core/src/main/java/org/apache/geode/cache/query/internal/index/AbstractIndex.java b/geode-core/src/main/java/org/apache/geode/cache/query/internal/index/AbstractIndex.java
index 94ac8a8..30b1401 100644
--- a/geode-core/src/main/java/org/apache/geode/cache/query/internal/index/AbstractIndex.java
+++ b/geode-core/src/main/java/org/apache/geode/cache/query/internal/index/AbstractIndex.java
@@ -2013,7 +2013,7 @@ public abstract class AbstractIndex implements IndexProtocol {
return;
}
if (!this.isIndexedPdxKeys) {
- if (key instanceof PdxString) {
+ if (key instanceof PdxString && this.region.getAttributes().getCompressor() == null) {
this.isIndexedPdxKeys = true;
}
}