You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by mm...@apache.org on 2018/11/07 01:10:06 UTC
metron git commit: METRON-1853: Add shutdown hook to Stellar
BaseFunctionResolver (mmiklavc via mmiklavc) closes apache/metron#1251
Repository: metron
Updated Branches:
refs/heads/master b9461e765 -> 85cd21aa0
METRON-1853: Add shutdown hook to Stellar BaseFunctionResolver (mmiklavc via mmiklavc) closes apache/metron#1251
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/85cd21aa
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/85cd21aa
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/85cd21aa
Branch: refs/heads/master
Commit: 85cd21aa0f5045184c168248dc2b81c1cfd41ddd
Parents: b9461e7
Author: mmiklavc <mi...@gmail.com>
Authored: Tue Nov 6 18:09:56 2018 -0700
Committer: Michael Miklavcic <mi...@gmail.com>
Committed: Tue Nov 6 18:09:56 2018 -0700
----------------------------------------------------------------------
.../ElasticsearchSearchIntegrationTest.java | 1 -
.../metron/stellar/dsl/StellarFunction.java | 9 +-
.../metron/stellar/dsl/StellarFunctions.java | 5 +
.../resolver/BaseFunctionResolver.java | 44 +++++
.../functions/resolver/FunctionResolver.java | 14 +-
.../stellar/dsl/functions/BasicStellarTest.java | 20 ++-
.../resolver/BaseFunctionResolverTest.java | 169 +++++++++++++++++++
7 files changed, 251 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/85cd21aa/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
index 1d2d48e..8187468 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
@@ -25,7 +25,6 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.common.Constants;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.elasticsearch.dao.ElasticsearchDao;
http://git-wip-us.apache.org/repos/asf/metron/blob/85cd21aa/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunction.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunction.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunction.java
index efdd185..4fabfaf 100644
--- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunction.java
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunction.java
@@ -17,10 +17,17 @@
*/
package org.apache.metron.stellar.dsl;
+import java.io.Closeable;
+import java.io.IOException;
import java.util.List;
-public interface StellarFunction {
+public interface StellarFunction extends Closeable {
Object apply(List<Object> args, Context context) throws ParseException;
void initialize(Context context);
boolean isInitialized();
+
+ @Override
+ default void close() throws IOException {
+
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/85cd21aa/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunctions.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunctions.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunctions.java
index dfec90e..73df82f 100644
--- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunctions.java
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunctions.java
@@ -18,6 +18,7 @@
package org.apache.metron.stellar.dsl;
+import java.io.IOException;
import org.apache.metron.stellar.dsl.functions.resolver.FunctionResolver;
import org.apache.metron.stellar.dsl.functions.resolver.SingletonFunctionResolver;
@@ -30,4 +31,8 @@ public class StellarFunctions {
public static void initialize(Context context) {
SingletonFunctionResolver.getInstance().initialize(context);
}
+
+ public static void close() throws IOException {
+ SingletonFunctionResolver.getInstance().close();
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/85cd21aa/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/BaseFunctionResolver.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/BaseFunctionResolver.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/BaseFunctionResolver.java
index aeed9d9..38a32d1 100644
--- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/BaseFunctionResolver.java
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/BaseFunctionResolver.java
@@ -23,6 +23,7 @@ import static java.lang.String.format;
import com.google.common.base.Joiner;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
+import java.io.IOException;
import java.io.Serializable;
import java.lang.invoke.MethodHandles;
import java.util.Arrays;
@@ -58,9 +59,15 @@ public abstract class BaseFunctionResolver implements FunctionResolver, Serializ
*/
protected Context context;
+ /**
+ * Indicates if closed has been called on this resolver.
+ */
+ private boolean closed;
+
public BaseFunctionResolver() {
// memoize provides lazy initialization and thread-safety (the ugly cast is necessary for serialization)
functions = Suppliers.memoize((Supplier<Map<String, StellarFunctionInfo>> & Serializable) this::resolveFunctions);
+ closed = false;
}
/**
@@ -95,6 +102,43 @@ public abstract class BaseFunctionResolver implements FunctionResolver, Serializ
}
/**
+ * Makes an attempt to close all Stellar functions. Calling close multiple times has no effect.
+ * @throws IOException Catches all exceptions and summarizes them.
+ */
+ @Override
+ public void close() throws IOException {
+ if (!closed) {
+ LOG.info("Calling close() on Stellar functions.");
+ Map<String, Throwable> errors = new HashMap<>();
+ for (StellarFunctionInfo info : getFunctionInfo()) {
+ try {
+ info.getFunction().close();
+ } catch (Throwable t) {
+ errors.put(info.getName(), t);
+ }
+ }
+ if (!errors.isEmpty()) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("Unable to close Stellar functions:");
+ for (Map.Entry<String, Throwable> e : errors.entrySet()) {
+ Throwable throwable = e.getValue();
+ String eText = String
+ .format("Exception - Function: %s; Message: %s; Cause: %s", e.getKey(),
+ throwable.getMessage(),
+ throwable.getCause());
+ sb.append(System.lineSeparator());
+ sb.append(eText);
+ }
+ closed = true;
+ throw new IOException(sb.toString());
+ }
+ closed = true;
+ } else {
+ LOG.info("close() already called on Stellar functions - skipping.");
+ }
+ }
+
+ /**
* Resolves a function by name.
* @param functionName The name of the function to resolve.
* @return The executable StellarFunction.
http://git-wip-us.apache.org/repos/asf/metron/blob/85cd21aa/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/FunctionResolver.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/FunctionResolver.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/FunctionResolver.java
index 5acb42c..4047586 100644
--- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/FunctionResolver.java
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/FunctionResolver.java
@@ -17,16 +17,17 @@
*/
package org.apache.metron.stellar.dsl.functions.resolver;
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.function.Function;
import org.apache.metron.stellar.dsl.Context;
import org.apache.metron.stellar.dsl.StellarFunction;
import org.apache.metron.stellar.dsl.StellarFunctionInfo;
-import java.util.function.Function;
-
/**
* Responsible for function resolution in Stellar.
*/
-public interface FunctionResolver extends Function<String, StellarFunction> {
+public interface FunctionResolver extends Function<String, StellarFunction>, Closeable {
/**
* Provides metadata about each Stellar function that is resolvable.
@@ -43,4 +44,11 @@ public interface FunctionResolver extends Function<String, StellarFunction> {
* @param context Context used to initialize.
*/
void initialize(Context context);
+
+ /**
+ * Perform any cleanup necessary for the loaded Stellar functions.
+ */
+ @Override
+ default void close() throws IOException {}
+
}
http://git-wip-us.apache.org/repos/asf/metron/blob/85cd21aa/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
index dec05a8..79f97bc 100644
--- a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
+++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
@@ -18,9 +18,20 @@
package org.apache.metron.stellar.dsl.functions;
+import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run;
+import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.runPredicate;
+import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.validate;
+
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.apache.metron.stellar.common.StellarProcessor;
import org.apache.metron.stellar.dsl.Context;
@@ -37,12 +48,6 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
-import java.util.*;
-
-import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run;
-import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.validate;
-import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.runPredicate;
-
@SuppressWarnings("ALL")
public class BasicStellarTest {
@@ -70,6 +75,7 @@ public class BasicStellarTest {
public boolean isInitialized() {
return true;
}
+
}
@Stellar(
@@ -96,6 +102,7 @@ public class BasicStellarTest {
public boolean isInitialized() {
return true;
}
+
}
@Test
@@ -1000,4 +1007,5 @@ public class BasicStellarTest {
checkFalsey("{}");
checkFalsey("LIST_ADD([])");
}
+
}
http://git-wip-us.apache.org/repos/asf/metron/blob/85cd21aa/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/resolver/BaseFunctionResolverTest.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/resolver/BaseFunctionResolverTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/resolver/BaseFunctionResolverTest.java
new file mode 100644
index 0000000..47cbda3
--- /dev/null
+++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/resolver/BaseFunctionResolverTest.java
@@ -0,0 +1,169 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.stellar.dsl.functions.resolver;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.junit.Assert.assertThat;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import org.apache.metron.stellar.dsl.BaseStellarFunction;
+import org.apache.metron.stellar.dsl.Stellar;
+import org.apache.metron.stellar.dsl.StellarFunction;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+public class BaseFunctionResolverTest {
+
+ public static class TestResolver extends BaseFunctionResolver {
+
+ Set<Class<? extends StellarFunction>> classesToResolve = new HashSet<>();
+
+ @Override
+ public Set<Class<? extends StellarFunction>> resolvables() {
+ return classesToResolve;
+ }
+
+ /**
+ * Will attempt to resolve any Stellar functions defined within the specified class.
+ * @param clazz The class which may contain a Stellar function.
+ */
+ public TestResolver withClass(Class<? extends StellarFunction> clazz) {
+ this.classesToResolve.add(clazz);
+ return this;
+ }
+ }
+
+ /**
+ * Often imitated, never duplicated.
+ */
+ @Stellar(namespace = "namespace", name = "afunction", description = "description", returns = "returns", params = {
+ "param1"})
+ private static class IAmAFunction extends BaseStellarFunction {
+
+ public static int closeCallCount;
+ public static boolean throwException = false; // init here bc of reflection in resolver.
+
+ public IAmAFunction() {
+ closeCallCount = 0;
+ }
+
+ @Override
+ public Object apply(List<Object> args) {
+ return null;
+ }
+
+ @Override
+ public void close() throws IOException {
+ closeCallCount++;
+ if (throwException) {
+ Throwable cause = new Throwable("Some nasty nasty cause.");
+ throw new IOException("Bad things happened", cause);
+ }
+ }
+ }
+
+ /**
+ * Scratch that. I was wrong.
+ */
+ @Stellar(namespace = "namespace", name = "anotherfunction", description = "description", returns = "returns", params = {
+ "param1"})
+ private static class IAmAnotherFunction extends BaseStellarFunction {
+
+ public static int closeCallCount;
+ public static boolean throwException = false; // init here bc of reflection in resolver.
+
+ public IAmAnotherFunction() {
+ closeCallCount = 0;
+ }
+
+ @Override
+ public Object apply(List<Object> args) {
+ return null;
+ }
+
+ @Override
+ public void close() throws IOException {
+ closeCallCount++;
+ if (throwException) {
+ throw new NullPointerException("A most annoying exception.");
+ }
+ }
+ }
+
+ private TestResolver resolver;
+
+ @Before
+ public void setup() {
+ resolver = new TestResolver();
+ IAmAFunction.throwException = false;
+ IAmAnotherFunction.throwException = false;
+ }
+
+ @Test
+ public void close_calls_all_loaded_function_close_methods() throws IOException {
+ resolver.withClass(IAmAFunction.class);
+ resolver.withClass(IAmAnotherFunction.class);
+ resolver.close();
+ assertThat(IAmAFunction.closeCallCount, equalTo(1));
+ assertThat(IAmAnotherFunction.closeCallCount, equalTo(1));
+ }
+
+ @Rule
+ public final ExpectedException exception = ExpectedException.none();
+
+ @Test
+ public void close_collects_all_exceptions_thrown_on_loaded_function_close_methods()
+ throws IOException {
+ IAmAFunction.throwException = true;
+ IAmAnotherFunction.throwException = true;
+ resolver.withClass(IAmAFunction.class);
+ resolver.withClass(IAmAnotherFunction.class);
+ exception.expect(IOException.class);
+ resolver.close();
+ }
+
+ @Test
+ public void close_only_throws_exceptions_on_first_invocation()
+ throws IOException {
+ IAmAFunction.throwException = true;
+ IAmAnotherFunction.throwException = true;
+ resolver.withClass(IAmAFunction.class);
+ resolver.withClass(IAmAnotherFunction.class);
+ try {
+ resolver.close();
+ Assert.fail("Should have thrown an exception.");
+ } catch (IOException e) {
+ // intentionally empty
+ }
+ assertThat(IAmAFunction.closeCallCount, equalTo(1));
+ assertThat(IAmAnotherFunction.closeCallCount, equalTo(1));
+ // should not throw exceptions or call any function's close again.
+ resolver.close();
+ resolver.close();
+ resolver.close();
+ assertThat(IAmAFunction.closeCallCount, equalTo(1));
+ assertThat(IAmAnotherFunction.closeCallCount, equalTo(1));
+ }
+}