You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by rm...@apache.org on 2019/07/02 13:47:43 UTC
[metron] branch master updated: METRON-2061 Solr documents with
date fields cannot be updated with Dao classes (merrimanr) closes
apache/metron#1374
This is an automated email from the ASF dual-hosted git repository.
rmerriman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 8792882 METRON-2061 Solr documents with date fields cannot be updated with Dao classes (merrimanr) closes apache/metron#1374
8792882 is described below
commit 87928824509c5f6ab314375be720dd64d8361cf4
Author: merrimanr <me...@gmail.com>
AuthorDate: Tue Jul 2 08:47:27 2019 -0500
METRON-2061 Solr documents with date fields cannot be updated with Dao classes (merrimanr) closes apache/metron#1374
---
.../metron/indexing/dao/update/PatchException.java | 27 +++
.../metron/indexing/dao/update/PatchOperation.java | 28 +++
.../metron/indexing/dao/update/PatchUtils.java | 105 ++++++++
.../metron/indexing/dao/update/UpdateDao.java | 3 +-
.../metron/indexing/dao/update/PatchUtilsTest.java | 263 +++++++++++++++++++++
5 files changed, 424 insertions(+), 2 deletions(-)
diff --git a/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/update/PatchException.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/update/PatchException.java
new file mode 100644
index 0000000..c621cfa
--- /dev/null
+++ b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/update/PatchException.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.metron.indexing.dao.update;
+
+public class PatchException extends RuntimeException {
+
+ public PatchException(String message) {
+ super(message);
+ }
+}
diff --git a/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/update/PatchOperation.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/update/PatchOperation.java
new file mode 100644
index 0000000..9b0e92d
--- /dev/null
+++ b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/update/PatchOperation.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.indexing.dao.update;
+
+enum PatchOperation {
+ ADD,
+ REPLACE,
+ REMOVE,
+ COPY,
+ MOVE,
+ TEST;
+}
diff --git a/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/update/PatchUtils.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/update/PatchUtils.java
new file mode 100644
index 0000000..e5c5117
--- /dev/null
+++ b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/update/PatchUtils.java
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.indexing.dao.update;
+
+import java.util.*;
+
+import static org.apache.metron.indexing.dao.update.PatchOperation.*;
+
+public enum PatchUtils {
+ INSTANCE;
+
+ public static final String OP = "op";
+ public static final String VALUE = "value";
+ public static final String PATH = "path";
+ public static final String FROM = "from";
+ private static final String PATH_SEPARATOR = "/";
+
+ public Map<String, Object> applyPatch(List<Map<String, Object>> patches, Map<String, Object> source) {
+ Map<String, Object> patchedObject = new HashMap<>(source);
+ for(Map<String, Object> patch: patches) {
+
+ // parse patch request parameters
+ String operation = (String) patch.get(OP);
+ PatchOperation patchOperation;
+ try {
+ patchOperation = PatchOperation.valueOf(operation.toUpperCase());
+ } catch(IllegalArgumentException e) {
+ throw new UnsupportedOperationException(String.format("The %s operation is not supported", operation));
+ }
+
+ Object value = patch.get(VALUE);
+ String path = (String) patch.get(PATH);
+
+ // locate the nested object
+ List<String> fieldNames = getFieldNames(path);
+ String nestedFieldName = fieldNames.get(fieldNames.size() - 1);
+ Map<String, Object> nestedObject = getNestedObject(fieldNames, patchedObject);
+
+ // apply the patch operation
+ if (ADD.equals(patchOperation) || REPLACE.equals(patchOperation)) {
+ nestedObject.put(nestedFieldName, value);
+ } else if (REMOVE.equals(patchOperation)) {
+ nestedObject.remove(nestedFieldName);
+ } else if (COPY.equals(patchOperation) || MOVE.equals(patchOperation)) {
+
+ // locate the nested object to copy/move the value from
+ String from = (String) patch.get(FROM);
+ List<String> fromFieldNames = getFieldNames(from);
+ String fromNestedFieldName = fromFieldNames.get(fromFieldNames.size() - 1);
+ Map<String, Object> fromNestedObject = getNestedObject(fromFieldNames, patchedObject);
+
+ // copy the value
+ Object copyValue = fromNestedObject.get(fromNestedFieldName);
+ nestedObject.put(nestedFieldName, copyValue);
+ if (MOVE.equals(patchOperation)) {
+
+ // remove the from value in case of a move
+ nestedObject.remove(fromNestedFieldName);
+ }
+ } else if (TEST.equals(patchOperation)) {
+
+ Object testValue = nestedObject.get(nestedFieldName);
+ if (!Objects.equals(value, testValue)) {
+ throw new PatchException(String.format("TEST operation failed: supplied value [%s] != target value [%s]", value, testValue));
+ }
+ }
+ }
+ return patchedObject;
+ }
+
+ private List<String> getFieldNames(String path) {
+ String[] parts = path.split(PATH_SEPARATOR);
+ return new ArrayList<>(Arrays.asList(parts).subList(1, parts.length));
+ }
+
+ @SuppressWarnings("unchecked")
+ private Map<String, Object> getNestedObject(List<String> fieldNames, Map<String, Object> patchedObject) {
+ Map<String, Object> nestedObject = patchedObject;
+ for(int i = 0; i < fieldNames.size() - 1; i++) {
+ Object object = nestedObject.get(fieldNames.get(i));
+ if (object == null || !(object instanceof Map)) {
+ throw new IllegalArgumentException(String.format("Invalid path: /%s", String.join(PATH_SEPARATOR, fieldNames)));
+ } else {
+ nestedObject = (Map<String, Object>) object;
+ }
+ }
+ return nestedObject;
+ }
+}
diff --git a/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/update/UpdateDao.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/update/UpdateDao.java
index ef1d298..efbbb04 100644
--- a/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/update/UpdateDao.java
+++ b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/update/UpdateDao.java
@@ -17,7 +17,6 @@
*/
package org.apache.metron.indexing.dao.update;
-import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.indexing.dao.RetrieveLatestDao;
import java.io.IOException;
@@ -95,7 +94,7 @@ public interface UpdateDao {
}
}
- Map<String, Object> patchedSource = JSONUtils.INSTANCE.applyPatch(request.getPatch(), originalSource);
+ Map<String, Object> patchedSource = PatchUtils.INSTANCE.applyPatch(request.getPatch(), originalSource);
return new Document(patchedSource, guid, sensorType, timestamp, documentID);
}
}
diff --git a/metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/update/PatchUtilsTest.java b/metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/update/PatchUtilsTest.java
new file mode 100644
index 0000000..b2bb173
--- /dev/null
+++ b/metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/update/PatchUtilsTest.java
@@ -0,0 +1,263 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.metron.indexing.dao.update;
+
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class PatchUtilsTest {
+
+ @Rule
+ public final ExpectedException exception = ExpectedException.none();
+
+ @Test
+ public void addOperationShouldAddValue() {
+ List<Map<String, Object>> patches = new ArrayList<>();
+ patches.add(new HashMap<String, Object>() {{
+ put(PatchUtils.OP, PatchOperation.ADD.name());
+ put(PatchUtils.PATH, "/path");
+ put(PatchUtils.VALUE, "value");
+ }});
+
+ Map<String, Object> expected = new HashMap<String, Object>() {{
+ put("path", "value");
+ }};
+
+ Assert.assertEquals(expected, PatchUtils.INSTANCE.applyPatch(patches, new HashMap<>()));
+ }
+
+ @Test
+ public void removeOperationShouldRemoveValue() {
+ List<Map<String, Object>> patches = new ArrayList<>();
+ patches.add(new HashMap<String, Object>() {{
+ put(PatchUtils.OP, PatchOperation.REMOVE.name());
+ put(PatchUtils.PATH, "/remove/path");
+ }});
+
+ Map<String, Object> expected = new HashMap<String, Object>() {{
+ put("path", "value");
+ put("remove", new HashMap<>());
+ }};
+
+ Assert.assertEquals(expected, PatchUtils.INSTANCE.applyPatch(patches, new HashMap<String, Object>() {{
+ put("path", "value");
+ put("remove", new HashMap<String, Object>() {{
+ put("path", "removeValue");
+ }});
+ }}));
+ }
+
+ @Test
+ public void copyOperationShouldCopyValue() {
+ List<Map<String, Object>> patches = new ArrayList<>();
+ patches.add(new HashMap<String, Object>() {{
+ put(PatchUtils.OP, PatchOperation.COPY.name());
+ put(PatchUtils.FROM, "/from");
+ put(PatchUtils.PATH, "/path");
+ }});
+
+ Map<String, Object> expected = new HashMap<String, Object>() {{
+ put("from", "value");
+ put("path", "value");
+ }};
+
+ Assert.assertEquals(expected, PatchUtils.INSTANCE.applyPatch(patches, new HashMap<String, Object>() {{
+ put("from", "value");
+ }}));
+ }
+
+ @Test
+ public void copyOperationShouldCopyNestedValue() {
+ List<Map<String, Object>> patches = new ArrayList<>();
+ patches.add(new HashMap<String, Object>() {{
+ put(PatchUtils.OP, PatchOperation.COPY.name());
+ put(PatchUtils.FROM, "/nested/from");
+ put(PatchUtils.PATH, "/nested/path");
+ }});
+
+ Map<String, Object> expected = new HashMap<String, Object>() {{
+ put("nested", new HashMap<String, Object>() {{
+ put("from", "value");
+ put("path", "value");
+ }});
+ }};
+
+ Assert.assertEquals(expected, PatchUtils.INSTANCE.applyPatch(patches, new HashMap<String, Object>() {{
+ put("nested", new HashMap<String, Object>() {{
+ put("from", "value");
+ }});
+ }}));
+ }
+
+ @Test
+ public void moveOperationShouldMoveValue() {
+ List<Map<String, Object>> patches = new ArrayList<>();
+ patches.add(new HashMap<String, Object>() {{
+ put(PatchUtils.OP, PatchOperation.MOVE.name());
+ put(PatchUtils.FROM, "/from");
+ put(PatchUtils.PATH, "/path");
+ }});
+
+ Map<String, Object> expected = new HashMap<String, Object>() {{
+ put("path", "value");
+ }};
+
+ Assert.assertEquals(expected, PatchUtils.INSTANCE.applyPatch(patches, new HashMap<String, Object>() {{
+ put("from", "value");
+ }}));
+ }
+
+ @Test
+ public void testOperationShouldCompareStrings() {
+ List<Map<String, Object>> patches = new ArrayList<>();
+ patches.add(new HashMap<String, Object>() {{
+ put(PatchUtils.OP, PatchOperation.TEST.name());
+ put(PatchUtils.PATH, "/path");
+ put(PatchUtils.VALUE, "value");
+ }});
+
+ Map<String, Object> expected = new HashMap<String, Object>() {{
+ put("path", "value");
+ }};
+
+ Assert.assertEquals(expected, PatchUtils.INSTANCE.applyPatch(patches, new HashMap<String, Object>() {{
+ put("path", "value");
+ }}));
+ }
+
+ @Test
+ public void testOperationShouldCompareNumbers() {
+ List<Map<String, Object>> patches = new ArrayList<>();
+ patches.add(new HashMap<String, Object>() {{
+ put(PatchUtils.OP, PatchOperation.TEST.name());
+ put(PatchUtils.PATH, "/path");
+ put(PatchUtils.VALUE, 100);
+ }});
+
+ Map<String, Object> expected = new HashMap<String, Object>() {{
+ put("path", 100);
+ }};
+
+ Assert.assertEquals(expected, PatchUtils.INSTANCE.applyPatch(patches, new HashMap<String, Object>() {{
+ put("path", 100);
+ }}));
+ }
+
+ @Test
+ public void testOperationShouldCompareArrays() {
+ List<Map<String, Object>> patches = new ArrayList<>();
+ patches.add(new HashMap<String, Object>() {{
+ put(PatchUtils.OP, PatchOperation.TEST.name());
+ put(PatchUtils.PATH, "/path");
+ put(PatchUtils.VALUE, Arrays.asList(1, 2, 3));
+ }});
+
+ Map<String, Object> expected = new HashMap<String, Object>() {{
+ put("path", Arrays.asList(1, 2, 3));
+ }};
+
+ Assert.assertEquals(expected, PatchUtils.INSTANCE.applyPatch(patches, new HashMap<String, Object>() {{
+ put("path", Arrays.asList(1, 2, 3));
+ }}));
+ }
+
+ @Test
+ public void testOperationShouldCompareObjects() {
+ List<Map<String, Object>> patches = new ArrayList<>();
+ patches.add(new HashMap<String, Object>() {{
+ put(PatchUtils.OP, PatchOperation.TEST.name());
+ put(PatchUtils.PATH, "/path");
+ put(PatchUtils.VALUE, new HashMap<String, Object>() {{
+ put("key", "value");
+ }});
+ }});
+
+ Map<String, Object> expected = new HashMap<String, Object>() {{
+ put("path", new HashMap<String, Object>() {{
+ put("key", "value");
+ }});
+ }};
+
+ Assert.assertEquals(expected, PatchUtils.INSTANCE.applyPatch(patches, new HashMap<String, Object>() {{
+ put("path", new HashMap<String, Object>() {{
+ put("key", "value");
+ }});
+ }}));
+ }
+
+ @Test
+ public void testOperationShouldThrowExceptionOnFailedCompare() {
+ exception.expect(PatchException.class);
+ exception.expectMessage("TEST operation failed: supplied value [value1] != target value [value2]");
+
+ List<Map<String, Object>> patches = new ArrayList<>();
+ patches.add(new HashMap<String, Object>() {{
+ put(PatchUtils.OP, PatchOperation.TEST.name());
+ put(PatchUtils.PATH, "/path");
+ put(PatchUtils.VALUE, "value1");
+ }});
+
+ PatchUtils.INSTANCE.applyPatch(patches, new HashMap<String, Object>() {{
+ put("path", "value2");
+ }});
+ }
+
+ @Test
+ public void shouldThrowExceptionOnInvalidPath() {
+ exception.expect(IllegalArgumentException.class);
+ exception.expectMessage("Invalid path: /missing/path");
+
+ List<Map<String, Object>> patches = new ArrayList<>();
+ patches.add(new HashMap<String, Object>() {{
+ put(PatchUtils.OP, PatchOperation.REMOVE.name());
+ put(PatchUtils.PATH, "/missing/path");
+ }});
+
+ PatchUtils.INSTANCE.applyPatch(patches, new HashMap<String, Object>() {{
+ put("path", "value");
+ }});
+
+ }
+
+ @Test
+ public void shouldThrowExceptionOnInvalidOperation() {
+ exception.expect(UnsupportedOperationException.class);
+ exception.expectMessage("The invalid operation is not supported");
+
+ List<Map<String, Object>> patches = new ArrayList<>();
+ patches.add(new HashMap<String, Object>() {{
+ put(PatchUtils.OP, "invalid");
+ put(PatchUtils.PATH, "/path");
+ }});
+
+ PatchUtils.INSTANCE.applyPatch(patches, new HashMap<String, Object>() {{
+ put("path", "value");
+ }});
+
+ }
+}