You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@apex.apache.org by vr...@apache.org on 2015/10/11 18:05:41 UTC
[26/29] incubator-apex-core git commit: APEX-24 #resolve took out
rhino dependency as it is GPL
APEX-24 #resolve took out rhino dependency as it is GPL
Project: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/commit/48cd2e8f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/tree/48cd2e8f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/diff/48cd2e8f
Branch: refs/heads/feature-module
Commit: 48cd2e8ffd7afd9cc7aa2aa8a8a44235616ece43
Parents: 3d0dd62
Author: David Yan <da...@datatorrent.com>
Authored: Thu Oct 8 12:37:36 2015 -0700
Committer: David Yan <da...@datatorrent.com>
Committed: Thu Oct 8 12:37:36 2015 -0700
----------------------------------------------------------------------
engine/pom.xml | 5 --
.../java/com/datatorrent/stram/StramClient.java | 1 -
.../stram/client/StramClientUtils.java | 92 +++++++++++---------
.../stram/client/EvalPropertiesTest.java | 68 ---------------
.../stram/client/StramClientUtilsTest.java | 37 ++++++++
5 files changed, 87 insertions(+), 116 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/48cd2e8f/engine/pom.xml
----------------------------------------------------------------------
diff --git a/engine/pom.xml b/engine/pom.xml
index 752fc0d..7974313 100644
--- a/engine/pom.xml
+++ b/engine/pom.xml
@@ -278,11 +278,6 @@
<type>jar</type>
</dependency>
<dependency>
- <groupId>org.mozilla</groupId>
- <artifactId>rhino</artifactId>
- <version>1.7R4</version>
- </dependency>
- <dependency>
<!-- We need this direct dependency because without this, it throws method not found exception on Amazon EMR version of hadoop 2.4.
Amazon EMR hadoop has commons-codec-1.2 in its classpath. -->
<groupId>commons-codec</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/48cd2e8f/engine/src/main/java/com/datatorrent/stram/StramClient.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/StramClient.java b/engine/src/main/java/com/datatorrent/stram/StramClient.java
index 7abfc82..9a570e0 100644
--- a/engine/src/main/java/com/datatorrent/stram/StramClient.java
+++ b/engine/src/main/java/com/datatorrent/stram/StramClient.java
@@ -121,7 +121,6 @@ public class StramClient
org.apache.http.message.BasicHeaderValueParser.class,
com.esotericsoftware.minlog.Log.class,
org.apache.xbean.asm5.tree.ClassNode.class,
- org.mozilla.javascript.Scriptable.class,
// The jersey client inclusion is only for Hadoop-2.2 and should be removed when we upgrade our Hadoop
// dependency version since Hadoop-2.3 onwards has jersey client bundled
com.sun.jersey.api.client.ClientHandler.class,
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/48cd2e8f/engine/src/main/java/com/datatorrent/stram/client/StramClientUtils.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/client/StramClientUtils.java b/engine/src/main/java/com/datatorrent/stram/client/StramClientUtils.java
index 4ac6487..27f58f0 100644
--- a/engine/src/main/java/com/datatorrent/stram/client/StramClientUtils.java
+++ b/engine/src/main/java/com/datatorrent/stram/client/StramClientUtils.java
@@ -25,14 +25,17 @@ import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.mozilla.javascript.Scriptable;
+import javax.script.ScriptEngine;
+import javax.script.ScriptEngineManager;
+import javax.script.ScriptException;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.StringEscapeUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileSystem;
@@ -647,58 +650,63 @@ public class StramClientUtils
public static void evalProperties(Properties target, Configuration vars)
{
- Pattern substitionPattern = Pattern.compile("\\$\\{(.+?)\\}");
+ ScriptEngine engine = new ScriptEngineManager().getEngineByName("javascript");
+
+ Pattern substitutionPattern = Pattern.compile("\\$\\{(.+?)\\}");
Pattern evalPattern = Pattern.compile("\\{% (.+?) %\\}");
- org.mozilla.javascript.Context context = org.mozilla.javascript.Context.enter();
- context.setOptimizationLevel(-1);
- Scriptable scope = context.initStandardObjects();
try {
- context.evaluateString(scope, "var _prop = {}", "EvalLaunchProperties", 0, null);
+ engine.eval("var _prop = {}");
for (Map.Entry<String, String> entry : vars) {
- LOG.info("Evaluating: {}", "_prop[\"" + entry.getKey() + "\"] = " + entry.getValue());
- context.evaluateString(scope, "_prop[\"" + StringEscapeUtils.escapeJava(entry.getKey()) + "\"] = \"" + StringEscapeUtils.escapeJava(entry.getValue()) + "\"", "EvalLaunchProperties", 0, null);
+ String evalString = String.format("_prop[\"%s\"] = \"%s\"", StringEscapeUtils.escapeJava(entry.getKey()), StringEscapeUtils.escapeJava(entry.getValue()));
+ LOG.debug("Evaluating: {}", evalString);
+ engine.eval(evalString);
+ }
+ } catch (ScriptException ex) {
+ LOG.warn("Javascript error: {}", ex.getMessage());
+ }
+
+ for (Map.Entry<Object, Object> entry : target.entrySet()) {
+ String value = entry.getValue().toString();
+
+ Matcher matcher = substitutionPattern.matcher(value);
+ if (matcher.find()) {
+ StringBuilder newValue = new StringBuilder();
+ int cursor = 0;
+ do {
+ newValue.append(value.substring(cursor, matcher.start()));
+ String subst = vars.get(matcher.group(1));
+ if (subst != null) {
+ newValue.append(subst);
+ }
+ cursor = matcher.end();
+ } while (matcher.find());
+ newValue.append(value.substring(cursor));
+ target.put(entry.getKey(), newValue.toString());
}
- for (Map.Entry<Object, Object> entry : target.entrySet()) {
- String value = entry.getValue().toString();
-
- Matcher matcher = substitionPattern.matcher(value);
- if (matcher.find()) {
- StringBuilder newValue = new StringBuilder();
- int cursor = 0;
- do {
- newValue.append(value.substring(cursor, matcher.start()));
- String subst = vars.get(matcher.group(1));
- if (subst != null) {
- newValue.append(subst);
- }
- cursor = matcher.end();
- } while (matcher.find());
- newValue.append(value.substring(cursor));
- target.put(entry.getKey(), newValue.toString());
- }
+ matcher = evalPattern.matcher(value);
+ if (matcher.find()) {
+ StringBuilder newValue = new StringBuilder();
+ int cursor = 0;
+ do {
+ newValue.append(value.substring(cursor, matcher.start()));
+ try {
+ Object result = engine.eval(matcher.group(1));
+ String eval = result.toString();
- matcher = evalPattern.matcher(value);
- if (matcher.find()) {
- StringBuilder newValue = new StringBuilder();
- int cursor = 0;
- do {
- newValue.append(value.substring(cursor, matcher.start()));
- String eval = context.evaluateString(scope, matcher.group(1), "EvalLaunchProperties", 0, null).toString();
if (eval != null) {
newValue.append(eval);
}
- cursor = matcher.end();
- } while (matcher.find());
- newValue.append(value.substring(cursor));
- target.put(entry.getKey(), newValue.toString());
- }
+ } catch (ScriptException ex) {
+ LOG.warn("JavaScript exception {}", ex.getMessage());
+ }
+ cursor = matcher.end();
+ } while (matcher.find());
+ newValue.append(value.substring(cursor));
+ target.put(entry.getKey(), newValue.toString());
}
}
- finally {
- org.mozilla.javascript.Context.exit();
- }
}
public static <T> T doAs(String userName, PrivilegedExceptionAction<T> action) throws Exception
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/48cd2e8f/engine/src/test/java/com/datatorrent/stram/client/EvalPropertiesTest.java
----------------------------------------------------------------------
diff --git a/engine/src/test/java/com/datatorrent/stram/client/EvalPropertiesTest.java b/engine/src/test/java/com/datatorrent/stram/client/EvalPropertiesTest.java
deleted file mode 100644
index 816e705..0000000
--- a/engine/src/test/java/com/datatorrent/stram/client/EvalPropertiesTest.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.stram.client;
-
-import java.util.Properties;
-import org.apache.hadoop.conf.Configuration;
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- *
- */
-
-
-public class EvalPropertiesTest
-{
- @Test
- public void testEvalExpression() throws Exception
- {
- Configuration conf = new Configuration();
- conf.set("a.b.c", "123");
- conf.set("d.e.f", "456");
- conf.set("x.y.z", "foobar");
-
- Properties prop = new Properties();
- prop.put("product.result", "Product result is {% (_prop[\"a.b.c\"] * _prop[\"d.e.f\"]).toFixed(0) %}...");
- prop.put("concat.result", "Concat result is {% _prop[\"x.y.z\"] %} ... {% _prop[\"a.b.c\"] %} blah");
-
- StramClientUtils.evalProperties(prop, conf);
-
- Assert.assertEquals("Product result is " + (123 * 456) + "...", prop.get("product.result"));
- Assert.assertEquals("Concat result is foobar ... 123 blah", prop.get("concat.result"));
- Assert.assertEquals("123", conf.get("a.b.c"));
- Assert.assertEquals("456", conf.get("d.e.f"));
- Assert.assertEquals("foobar", conf.get("x.y.z"));
- }
-
- @Test
- public void testVariableSubstitution() throws Exception
- {
- Configuration conf = new Configuration();
- conf.set("a.b.c", "123");
- conf.set("x.y.z", "foobar");
-
- Properties prop = new Properties();
- prop.put("var.result", "1111 ${a.b.c} xxx ${x.y.z} yyy");
-
- StramClientUtils.evalProperties(prop, conf);
-
- Assert.assertEquals("1111 123 xxx foobar yyy", prop.get("var.result"));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/48cd2e8f/engine/src/test/java/com/datatorrent/stram/client/StramClientUtilsTest.java
----------------------------------------------------------------------
diff --git a/engine/src/test/java/com/datatorrent/stram/client/StramClientUtilsTest.java b/engine/src/test/java/com/datatorrent/stram/client/StramClientUtilsTest.java
index f9fca05..51193ef 100644
--- a/engine/src/test/java/com/datatorrent/stram/client/StramClientUtilsTest.java
+++ b/engine/src/test/java/com/datatorrent/stram/client/StramClientUtilsTest.java
@@ -22,6 +22,7 @@ import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.util.List;
+import java.util.Properties;
import org.junit.Assert;
import org.junit.Test;
@@ -39,6 +40,42 @@ import com.datatorrent.stram.util.ConfigUtils;
public class StramClientUtilsTest
{
+ @Test
+ public void testEvalExpression() throws Exception
+ {
+ Configuration conf = new Configuration();
+ conf.set("a.b.c", "123");
+ conf.set("d.e.f", "456");
+ conf.set("x.y.z", "foobar");
+
+ Properties prop = new Properties();
+ prop.put("product.result", "Product result is {% (_prop[\"a.b.c\"] * _prop[\"d.e.f\"]).toFixed(0) %}...");
+ prop.put("concat.result", "Concat result is {% _prop[\"x.y.z\"] %} ... {% _prop[\"a.b.c\"] %} blah");
+
+ StramClientUtils.evalProperties(prop, conf);
+
+ Assert.assertEquals("Product result is " + (123 * 456) + "...", prop.get("product.result"));
+ Assert.assertEquals("Concat result is foobar ... 123 blah", prop.get("concat.result"));
+ Assert.assertEquals("123", conf.get("a.b.c"));
+ Assert.assertEquals("456", conf.get("d.e.f"));
+ Assert.assertEquals("foobar", conf.get("x.y.z"));
+ }
+
+ @Test
+ public void testVariableSubstitution() throws Exception
+ {
+ Configuration conf = new Configuration();
+ conf.set("a.b.c", "123");
+ conf.set("x.y.z", "foobar");
+
+ Properties prop = new Properties();
+ prop.put("var.result", "1111 ${a.b.c} xxx ${x.y.z} yyy");
+
+ StramClientUtils.evalProperties(prop, conf);
+
+ Assert.assertEquals("1111 123 xxx foobar yyy", prop.get("var.result"));
+ }
+
private String getHostString(String host) throws UnknownHostException
{
InetAddress address = InetAddress.getByName(host);