You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gora.apache.org by dj...@apache.org on 2017/05/14 11:42:54 UTC
[1/2] gora git commit: GORA-310: Fix partial field delete in solr
Repository: gora
Updated Branches:
refs/heads/master 3a2e1ccfe -> 976a5c1db
GORA-310: Fix partial field delete in solr
Project: http://git-wip-us.apache.org/repos/asf/gora/repo
Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/9b632c5e
Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/9b632c5e
Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/9b632c5e
Branch: refs/heads/master
Commit: 9b632c5e1a951fbf06126063a3cee9ed65bea9a9
Parents: 3a2e1cc
Author: madhawa-gunasekara <ma...@wso2.com>
Authored: Wed May 10 16:17:36 2017 +0530
Committer: madhawa-gunasekara <ma...@wso2.com>
Committed: Wed May 10 20:07:19 2017 +0530
----------------------------------------------------------------------
.../org/apache/gora/solr/store/SolrStore.java | 44 +++++++++---
.../apache/gora/solr/store/TestSolrStore.java | 76 ++++++++++++++++++--
2 files changed, 105 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/gora/blob/9b632c5e/gora-solr-5/src/main/java/org/apache/gora/solr/store/SolrStore.java
----------------------------------------------------------------------
diff --git a/gora-solr-5/src/main/java/org/apache/gora/solr/store/SolrStore.java b/gora-solr-5/src/main/java/org/apache/gora/solr/store/SolrStore.java
index 5d34cf0..9303f76 100644
--- a/gora-solr-5/src/main/java/org/apache/gora/solr/store/SolrStore.java
+++ b/gora-solr-5/src/main/java/org/apache/gora/solr/store/SolrStore.java
@@ -17,11 +17,7 @@ package org.apache.gora.solr.store;
import java.io.IOException;
import java.net.MalformedURLException;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Properties;
+import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.avro.Schema;
@@ -741,11 +737,41 @@ public class SolrStore<K, T extends PersistentBase> extends DataStoreBase<K, T>
@Override
public long deleteByQuery(Query<K, T> query) {
- String q = ((SolrQuery<K, T>) query).toSolrQuery();
+ UpdateResponse rsp;
try {
- UpdateResponse rsp = server.deleteByQuery(q);
- server.commit();
- LOG.info(rsp.toString());
+ /*
+ In this If block we check whether, user needs to delete full document or some fields in the document. We can't delete fields in a document by using solr deleteByQuery method.
+ therefore what we have done here is setting the particular fields values into null.
+ */
+ if (query.getFields() != null && query.getFields().length < mapping.mapping.size() && !(Arrays.asList(query.getFields()).contains(mapping.getPrimaryKey()))) {
+ Result<K, T> result = query.execute();
+ Map<String, String> partialUpdateNull = new HashMap<>();
+ partialUpdateNull.put("set", null);
+ while (result.next()) {
+ SolrInputDocument inputDoc = new SolrInputDocument();
+ inputDoc.setField(mapping.getPrimaryKey(), result.getKey());
+ for (String field : query.getFields()) {
+ inputDoc.setField(field, partialUpdateNull);
+ }
+ batch.add(inputDoc);
+ }
+ if (commitWithin == 0) {
+ rsp = server.add(batch);
+ server.commit(false, true, true);
+ batch.clear();
+ LOG.info(rsp.toString());
+ } else {
+ rsp = server.add(batch, commitWithin);
+ batch.clear();
+ LOG.info(rsp.toString());
+ }
+ } else {
+ SolrQuery<K, T> solrQuery = (SolrQuery<K, T>) query;
+ String q = solrQuery.toSolrQuery();
+ rsp = server.deleteByQuery(q);
+ server.commit();
+ LOG.info(rsp.toString());
+ }
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
http://git-wip-us.apache.org/repos/asf/gora/blob/9b632c5e/gora-solr-5/src/test/java/org/apache/gora/solr/store/TestSolrStore.java
----------------------------------------------------------------------
diff --git a/gora-solr-5/src/test/java/org/apache/gora/solr/store/TestSolrStore.java b/gora-solr-5/src/test/java/org/apache/gora/solr/store/TestSolrStore.java
index da33e6b..4c86e1c 100644
--- a/gora-solr-5/src/test/java/org/apache/gora/solr/store/TestSolrStore.java
+++ b/gora-solr-5/src/test/java/org/apache/gora/solr/store/TestSolrStore.java
@@ -17,17 +17,30 @@
*/
package org.apache.gora.solr.store;
-import java.io.IOException;
-
+import org.apache.gora.examples.WebPageDataCreator;
import org.apache.gora.examples.generated.Employee;
import org.apache.gora.examples.generated.WebPage;
+import org.apache.gora.query.Query;
import org.apache.gora.solr.GoraSolrTestDriver;
import org.apache.gora.store.DataStore;
import org.apache.gora.store.DataStoreFactory;
import org.apache.gora.store.DataStoreTestBase;
-import org.junit.Ignore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+import static junit.framework.Assert.assertNull;
+import static org.apache.gora.examples.WebPageDataCreator.SORTED_URLS;
+import static org.apache.gora.examples.WebPageDataCreator.URLS;
+import static org.apache.gora.store.DataStoreTestUtil.assertNumResults;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
public class TestSolrStore extends DataStoreTestBase {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestSolrStore.class);
+ private static final int NUM_KEYS = 4;
static {
setTestDriver(new GoraSolrTestDriver());
@@ -49,8 +62,59 @@ public class TestSolrStore extends DataStoreTestBase {
return store;
}
-
- @Ignore("GORA-310 and GORA-311 issues are not fixed at SolrStore")
@Override
- public void testDeleteByQueryFields() throws IOException {}
+ public void testDeleteByQueryFields()
+ throws Exception {
+ Query<String, WebPage> query;
+
+ //test 5 - delete all with some fields
+ WebPageDataCreator.createWebPageData(this.webPageStore);
+
+ query = this.webPageStore.newQuery();
+ query.setFields("outlinks"
+ , "parsedContent", "content");
+
+ assertNumResults(this.webPageStore.newQuery(), URLS.length);
+ this.webPageStore.deleteByQuery(query);
+
+ this.webPageStore.flush();
+
+ assertNumResults(this.webPageStore.newQuery(), URLS.length);
+
+ //assert that data is deleted
+ for (String SORTED_URL : SORTED_URLS) {
+ WebPage page = this.webPageStore.get(SORTED_URL);
+ assertNotNull(page);
+
+ assertNotNull(page.getUrl());
+ assertEquals(page.getUrl().toString(), SORTED_URL);
+ assertEquals("Map of Outlinks should have a size of '0' as the deleteByQuery "
+ + "not only removes the data but also the data structure.", 0, page.getOutlinks().size());
+ assertEquals(0, page.getParsedContent().size());
+ if (page.getContent() != null) {
+ LOG.info("url:" + page.getUrl().toString());
+ LOG.info("limit:" + page.getContent().limit());
+ } else {
+ assertNull(page.getContent());
+ }
+ }
+
+ //test 6 - delete some with some fields
+ WebPageDataCreator.createWebPageData(this.webPageStore);
+
+ query = this.webPageStore.newQuery();
+ query.setFields("url");
+ String startKey = SORTED_URLS[NUM_KEYS];
+ String endKey = SORTED_URLS[SORTED_URLS.length - NUM_KEYS];
+ query.setStartKey(startKey);
+ query.setEndKey(endKey);
+
+ assertNumResults(this.webPageStore.newQuery(), URLS.length);
+ this.webPageStore.deleteByQuery(query);
+
+ this.webPageStore.flush();
+
+ assertNumResults(query,0);
+
+ }
}
[2/2] gora git commit: Fix imports properly in SolrStore
Posted by dj...@apache.org.
Fix imports properly in SolrStore
Project: http://git-wip-us.apache.org/repos/asf/gora/repo
Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/976a5c1d
Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/976a5c1d
Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/976a5c1d
Branch: refs/heads/master
Commit: 976a5c1db12d61ac2345f5aa6afcda8c1f3f4ed8
Parents: 9b632c5
Author: madhawa-gunasekara <ma...@wso2.com>
Authored: Wed May 10 23:44:26 2017 +0530
Committer: madhawa-gunasekara <ma...@wso2.com>
Committed: Wed May 10 23:44:26 2017 +0530
----------------------------------------------------------------------
.../src/main/java/org/apache/gora/solr/store/SolrStore.java | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/gora/blob/976a5c1d/gora-solr-5/src/main/java/org/apache/gora/solr/store/SolrStore.java
----------------------------------------------------------------------
diff --git a/gora-solr-5/src/main/java/org/apache/gora/solr/store/SolrStore.java b/gora-solr-5/src/main/java/org/apache/gora/solr/store/SolrStore.java
index 9303f76..ac44a06 100644
--- a/gora-solr-5/src/main/java/org/apache/gora/solr/store/SolrStore.java
+++ b/gora-solr-5/src/main/java/org/apache/gora/solr/store/SolrStore.java
@@ -17,7 +17,13 @@ package org.apache.gora.solr.store;
import java.io.IOException;
import java.net.MalformedURLException;
import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.avro.Schema;