You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ot...@apache.org on 2018/01/09 14:10:34 UTC
[25/35] metron git commit: METRON-1377: Stellar function to generate
typosquatted domains (similar to dnstwist) closes apache/incubator-metron#878
METRON-1377: Stellar function to generate typosquatted domains (similar to dnstwist) closes apache/incubator-metron#878
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/0996b734
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/0996b734
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/0996b734
Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual
Commit: 0996b7348eca14fea1b1b3c4dd57861b3a30bdeb
Parents: 9108072
Author: cstella <ce...@gmail.com>
Authored: Mon Jan 8 09:30:58 2018 -0500
Committer: cstella <ce...@gmail.com>
Committed: Mon Jan 8 09:30:58 2018 -0500
----------------------------------------------------------------------
.../common/typosquat/AdditionStrategy.java | 41 +++
.../common/typosquat/BitsquattingStrategy.java | 61 +++++
.../common/typosquat/HomoglyphStrategy.java | 115 ++++++++
.../common/typosquat/HyphenationStrategy.java | 41 +++
.../common/typosquat/InsertionStrategy.java | 56 ++++
.../metron/common/typosquat/Keyboards.java | 151 +++++++++++
.../common/typosquat/OmissionStrategy.java | 41 +++
.../common/typosquat/RepetitionStrategy.java | 48 ++++
.../common/typosquat/ReplacementStrategy.java | 49 ++++
.../common/typosquat/SubdomainStrategy.java | 48 ++++
.../common/typosquat/TranspositionStrategy.java | 46 ++++
.../typosquat/TyposquattingStrategies.java | 109 ++++++++
.../common/typosquat/TyposquattingStrategy.java | 25 ++
.../common/typosquat/VowelSwapStrategy.java | 57 ++++
.../typosquat/TyposquattingStrategiesTest.java | 126 +++++++++
.../src/test/resources/typosquat/amazon.csv | 259 +++++++++++++++++++
.../src/test/resources/typosquat/github.csv | 227 ++++++++++++++++
.../dsl/functions/FunctionalFunctions.java | 9 +-
18 files changed, 1504 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/0996b734/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/AdditionStrategy.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/AdditionStrategy.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/AdditionStrategy.java
new file mode 100644
index 0000000..871052d
--- /dev/null
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/AdditionStrategy.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.common.typosquat;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * A typo strategy based on adding characters between ascii 97 and 123.
+ */
+public class AdditionStrategy implements TyposquattingStrategy {
+ @Override
+ public Set<String> generateCandidates(String domain) {
+ Set<String> ret = new HashSet<>();
+ for(int i = 97;i < 123;++i) {
+ char c = Character.toChars(i)[0];
+ ret.add(domain + c);
+ }
+ return ret;
+ }
+
+ @Override
+ public String name() {
+ return "Addition";
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/0996b734/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/BitsquattingStrategy.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/BitsquattingStrategy.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/BitsquattingStrategy.java
new file mode 100644
index 0000000..32d4184
--- /dev/null
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/BitsquattingStrategy.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.common.typosquat;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * See http://dinaburg.org/bitsquatting.html for more
+ */
+public class BitsquattingStrategy implements TyposquattingStrategy {
+ public static int[] MASK = new int[] { 1, 2, 4, 8, 16, 32, 64, 128};
+ @Override
+ public Set<String> generateCandidates(String originalString) {
+ Set<String> ret = new HashSet<>();
+ char[] str = originalString.toCharArray();
+ for(int i = 0;i < str.length;++i) {
+ char c = str[i];
+ for(int j : MASK) {
+ int maskedNum = (int)c ^ j;
+ char maskedChar = (char)maskedNum;
+ if((maskedNum >= 48 && maskedNum <= 57) || (maskedNum >= 97 && maskedNum <= 122) || maskedNum == 45) {
+ ret.add(pasteTogether(str, i, maskedChar));
+ }
+ }
+ }
+ return ret;
+ }
+
+ @Override
+ public String name() {
+ return "Bitsquatting";
+ }
+
+ private static String pasteTogether(char[] str, int replacementPoint, char maskedChar) {
+ String ret = "";
+ for(int i = 0;i < replacementPoint;++i) {
+ ret += str[i];
+ }
+ ret += maskedChar;
+ for(int i = replacementPoint+1;i < str.length;++i) {
+ ret += str[i];
+ }
+ return ret;
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/0996b734/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/HomoglyphStrategy.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/HomoglyphStrategy.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/HomoglyphStrategy.java
new file mode 100644
index 0000000..19479de
--- /dev/null
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/HomoglyphStrategy.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.common.typosquat;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.invoke.MethodHandles;
+import java.net.IDN;
+import java.util.*;
+
+/**
+ * Substituting characters for ascii or unicode analogues which are visually similar (e.g. latlmes.com for latimes.com)
+ *
+ */
+public class HomoglyphStrategy implements TyposquattingStrategy{
+
+ private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ public static final Map<Character, List<String>> glyphs = new HashMap<Character, List<String>>()
+ {{
+ put('a', ImmutableList.of("à", "á", "â", "ã", "ä", "å", "ɑ", "а", "ạ", "ǎ", "ă", "ȧ", "ӓ"));
+ put('b', ImmutableList.of("d", "lb", "ib", "ʙ", "Ь", "b̔", "ɓ", "Б"));
+ put('c', ImmutableList.of("ϲ", "с", "ƈ", "ċ", "ć", "ç"));
+ put('d', ImmutableList.of("b", "cl", "dl", "di", "ԁ", "ժ", "ɗ", "đ"));
+ put('e', ImmutableList.of("é", "ê", "ë", "ē", "ĕ", "ě", "ė", "е", "ẹ", "ę", "є", "ϵ", "ҽ"));
+ put('f', ImmutableList.of("Ϝ", "ƒ", "Ғ"));
+ put('g', ImmutableList.of("q", "ɢ", "ɡ", "Ԍ", "Ԍ", "ġ", "ğ", "ց", "ǵ", "ģ"));
+ put('h', ImmutableList.of("lh", "ih", "һ", "հ", "Ꮒ", "н"));
+ put('i', ImmutableList.of("1", "l", "Ꭵ", "í", "ï", "ı", "ɩ", "ι", "ꙇ", "ǐ", "ĭ"));
+ put('j', ImmutableList.of("ј", "ʝ", "ϳ", "ɉ"));
+ put('k', ImmutableList.of("lk", "ik", "lc", "κ", "ⲕ", "κ"));
+ put('l', ImmutableList.of("1", "i", "ɫ", "ł"));
+ put('m', ImmutableList.of("n", "nn", "rn", "rr", "ṃ", "ᴍ", "м", "ɱ"));
+ put('n', ImmutableList.of("m", "r", "ń"));
+ put('o', ImmutableList.of("0", "Ο", "ο", "О", "о", "Օ", "ȯ", "ọ", "ỏ", "ơ", "ó", "ö", "ӧ"));
+ put('p', ImmutableList.of("ρ", "р", "ƿ", "Ϸ", "Þ"));
+ put('q', ImmutableList.of("g", "զ", "ԛ", "գ", "ʠ"));
+ put('r', ImmutableList.of("ʀ", "Г", "ᴦ", "ɼ", "ɽ"));
+ put('s', ImmutableList.of("Ⴝ", "Ꮪ", "ʂ", "ś", "ѕ"));
+ put('t', ImmutableList.of("τ", "т", "ţ"));
+ put('u', ImmutableList.of("μ", "υ", "Ս", "ս", "ц", "ᴜ", "ǔ", "ŭ"));
+ put('v', ImmutableList.of("ѵ", "ν", "v̇"));
+ put('w', ImmutableList.of("vv", "ѡ", "ա", "ԝ"));
+ put('x', ImmutableList.of("х", "ҳ", "ẋ"));
+ put('y', ImmutableList.of("ʏ", "γ", "у", "Ү", "ý"));
+ put('z', ImmutableList.of("ʐ", "ż", "ź", "ʐ", "ᴢ"));
+ }};
+
+ @Override
+ public Set<String> generateCandidates(String originalString) {
+ Set<String> result = new HashSet<>();
+ String domain = originalString;
+ if(StringUtils.isEmpty(domain)) {
+ return result;
+ }
+ if(isAce(domain)) {
+ //this is an ace domain.
+ domain = IDN.toUnicode(domain);
+ }
+ for(int ws = 0;ws < domain.length();ws++) {
+ for(int i = 0;i < domain.length() - ws + 1;++i) {
+ String win = domain.substring(i, i+ws);
+ for(int j = 0;j < ws;j++) {
+ char c = win.charAt(j);
+ if( glyphs.containsKey(c)) {
+ for( String g : glyphs.get(c)) {
+ String winNew = win.replaceAll("" + c, g);
+ String d = domain.substring(0, i) + winNew + domain.substring(i + ws);
+ result.add(d);
+ if(!isAce(d)) {
+ try {
+ String dAscii = IDN.toASCII(d, IDN.ALLOW_UNASSIGNED);
+ if (!d.equals(dAscii)) {
+ result.add(dAscii);
+ }
+ }
+ catch(IllegalArgumentException iae) {
+ LOG.debug("Unable to parse " + d + ": " + iae.getMessage(), iae);
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ return result;
+ }
+
+ public static boolean isAce(String domainRaw) {
+ String domain = domainRaw.toLowerCase();
+ return domain.startsWith("xn--") || domain.contains(".xn--");
+ }
+
+ @Override
+ public String name() {
+ return "Homoglyph";
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/0996b734/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/HyphenationStrategy.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/HyphenationStrategy.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/HyphenationStrategy.java
new file mode 100644
index 0000000..ca6ce8f
--- /dev/null
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/HyphenationStrategy.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.common.typosquat;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * Typos based on random hyphenation (e.g. am-azon.com vs amazon.com)
+ */
+public class HyphenationStrategy implements TyposquattingStrategy {
+ @Override
+ public Set<String> generateCandidates(String originalString) {
+ Set<String> ret = new HashSet<>();
+ for(int i = 1;i < originalString.length();++i) {
+ ret.add(originalString.substring(0, i) + "-" + originalString.substring(i));
+ }
+ return ret;
+ }
+
+ @Override
+ public String name() {
+ return "Hyphenation";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/0996b734/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/InsertionStrategy.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/InsertionStrategy.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/InsertionStrategy.java
new file mode 100644
index 0000000..b601a35
--- /dev/null
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/InsertionStrategy.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.common.typosquat;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * Typo strategy based on random insertion of common typos based on keyboard layout proximity.
+ */
+public class InsertionStrategy implements TyposquattingStrategy {
+ @Override
+ public Set<String> generateCandidates(String domain) {
+ Set<String> ret = new HashSet<>();
+ for(int i = 1;i < domain.length() - 1;++i) {
+ for(Keyboards keyboard : Keyboards.values()) {
+ String mapping = keyboard.getMapping().get(domain.charAt(i));
+ if(mapping != null) {
+ for(Character c : mapping.toCharArray()) {
+ ret.add(domain.substring(0, i)
+ + c
+ + domain.charAt(i)
+ + domain.substring(i+1, domain.length())
+ );
+ ret.add(domain.substring(0, i)
+ + domain.charAt(i)
+ + c
+ + domain.substring(i+1, domain.length())
+ );
+ }
+ }
+ }
+ }
+ return ret;
+ }
+
+ @Override
+ public String name() {
+ return "Insertion";
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/0996b734/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/Keyboards.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/Keyboards.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/Keyboards.java
new file mode 100644
index 0000000..6fbd5ec
--- /dev/null
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/Keyboards.java
@@ -0,0 +1,151 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.common.typosquat;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * This provides a mapping of nearby keys for a variety of keyboard layouts. This is useful in determining likely
+ * typos.
+ */
+public enum Keyboards {
+ QWERTY(new HashMap<Character, String>()
+ {{
+ put('j', "ikmnhu");
+ put('w', "3esaq2");
+ put('v', "cfgb");
+ put('l', "kop");
+ put('y', "7uhgt6");
+ put('x', "zsdc");
+ put('r', "5tfde4");
+ put('u', "8ijhy7");
+ put('a', "qwsz");
+ put('q', "12wa");
+ put('c', "xdfv");
+ put('b', "vghn");
+ put('e', "4rdsw3");
+ put('d', "rfcxse");
+ put('g', "yhbvft");
+ put('p', "lo0");
+ put('i', "9okju8");
+ put('h', "ujnbgy");
+ put('k', "olmji");
+ put('f', "tgvcdr");
+ put('m', "njk");
+ put('s', "edxzaw");
+ put('o', "0plki9");
+ put('n', "bhjm");
+ put('1', "2q");
+ put('0', "po9");
+ put('3', "4ew2");
+ put('2', "3wq1");
+ put('5', "6tr4");
+ put('4', "5re3");
+ put('7', "8uy6");
+ put('6', "7yt5");
+ put('9', "0oi8");
+ put('8', "9iu7");
+ put('z', "asx");
+ put('t', "6ygfr5");
+ }}),
+ QWERTZ(new HashMap<Character, String>() {{
+ put('j', "ikmnhu");
+ put('w', "3esaq2");
+ put('v', "cfgb");
+ put('l', "kop");
+ put('y', "asx");
+ put('x', "ysdc");
+ put('r', "5tfde4");
+ put('u', "8ijhz7");
+ put('a', "qwsy");
+ put('q', "12wa");
+ put('c', "xdfv");
+ put('b', "vghn");
+ put('e', "4rdsw3");
+ put('d', "rfcxse");
+ put('g', "zhbvft");
+ put('p', "lo0");
+ put('i', "9okju8");
+ put('h', "ujnbgz");
+ put('k', "olmji");
+ put('f', "tgvcdr");
+ put('m', "njk");
+ put('s', "edxyaw");
+ put('o', "0plki9");
+ put('n', "bhjm");
+ put('1', "2q");
+ put('0', "po9");
+ put('3', "4ew2");
+ put('2', "3wq1");
+ put('5', "6tr4");
+ put('4', "5re3");
+ put('7', "8uz6");
+ put('6', "7zt5");
+ put('9', "0oi8");
+ put('8', "9iu7");
+ put('z', "7uhgt6");
+ put('t', "6zgfr5");
+ }}),
+ AZERTY(new HashMap<Character, String>() {{
+ put('j', "iknhu");
+ put('w', "sxq");
+ put('v', "cfgb");
+ put('l', "kopm");
+ put('y', "7uhgt6");
+ put('x', "zsdc");
+ put('r', "5tfde4");
+ put('u', "8ijhy7");
+ put('a', "2zq1");
+ put('q', "zswa");
+ put('c', "xdfv");
+ put('b', "vghn");
+ put('e', "4rdsz3");
+ put('d', "rfcxse");
+ put('g', "yhbvft");
+ put('p', "lo0m");
+ put('i', "9okju8");
+ put('h', "ujnbgy");
+ put('k', "olji");
+ put('f', "tgvcdr");
+ put('m', "lp");
+ put('s', "edxwqz");
+ put('o', "0plki9");
+ put('n', "bhj");
+ put('1', "2a");
+ put('0', "po9");
+ put('3', "4ez2");
+ put('2', "3za1");
+ put('5', "6tr4");
+ put('4', "5re3");
+ put('7', "8uy6");
+ put('6', "7yt5");
+ put('9', "0oi8");
+ put('8', "9iu7");
+ put('z', "3esqa2");
+ put('t', "6ygfr5");
+ }})
+ ;
+ private Map<Character, String> mapping;
+ Keyboards(Map<Character, String> mapping) {
+ this.mapping = mapping;
+ }
+ public Map<Character, String> getMapping() {
+ return mapping;
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/0996b734/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/OmissionStrategy.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/OmissionStrategy.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/OmissionStrategy.java
new file mode 100644
index 0000000..c49dbb0
--- /dev/null
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/OmissionStrategy.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.common.typosquat;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * A typo strategy based on omitting characters.
+ */
+public class OmissionStrategy implements TyposquattingStrategy {
+
+ @Override
+ public Set<String> generateCandidates(String domain) {
+ HashSet<String> ret = new HashSet<>();
+ for(int i = 0;i < domain.length();++i) {
+ ret.add(domain.substring(0, i) + domain.substring(i+1, domain.length()));
+ }
+ return ret;
+ }
+
+ @Override
+ public String name() {
+ return "Omission";
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/0996b734/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/RepetitionStrategy.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/RepetitionStrategy.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/RepetitionStrategy.java
new file mode 100644
index 0000000..fa1b0aa
--- /dev/null
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/RepetitionStrategy.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.common.typosquat;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * Typo strategy around repeating existing characters.
+ */
+public class RepetitionStrategy implements TyposquattingStrategy {
+ @Override
+ public Set<String> generateCandidates(String domain) {
+ Set<String> ret = new HashSet<>();
+ for(int i = 0;i < domain.length();++i) {
+ Character c = domain.charAt(i);
+ if(Character.isLetter(c)) {
+ ret.add( domain.substring(0, i)
+ + c
+ + c
+ + domain.substring(i+1, domain.length())
+ );
+ }
+ }
+ return ret;
+ }
+
+ @Override
+ public String name() {
+ return "Repetition";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/0996b734/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/ReplacementStrategy.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/ReplacementStrategy.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/ReplacementStrategy.java
new file mode 100644
index 0000000..78254d9
--- /dev/null
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/ReplacementStrategy.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.common.typosquat;
+
+import java.util.HashSet;
+import java.util.Set;
+
+public class ReplacementStrategy implements TyposquattingStrategy {
+ @Override
+ public Set<String> generateCandidates(String domain) {
+ Set<String> ret = new HashSet<>();
+ for(int i = 0;i < domain.length();++i) {
+ for(Keyboards keyboard : Keyboards.values()) {
+ String mapping = keyboard.getMapping().get(domain.charAt(i));
+ if(mapping != null) {
+ for(Character c : mapping.toCharArray()) {
+ ret.add( domain.substring(0, i)
+ + c
+ + domain.substring(i+1, domain.length())
+ );
+ }
+ }
+ }
+ }
+ return ret;
+ }
+
+ @Override
+ public String name() {
+ return "Replacement";
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/0996b734/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/SubdomainStrategy.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/SubdomainStrategy.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/SubdomainStrategy.java
new file mode 100644
index 0000000..2dc06e4
--- /dev/null
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/SubdomainStrategy.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.common.typosquat;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * A typo strategy around random subdomains (e.g. am.azon.com vs amazon.com)
+ */
+public class SubdomainStrategy implements TyposquattingStrategy{
+
+ @Override
+ public Set<String> generateCandidates(String domain) {
+ Set<String> ret = new HashSet<>();
+ for(int i = 1;i < domain.length();++i) {
+ Character c = domain.charAt(i);
+ Character prevC = domain.charAt(i-1);
+ if(c != '-' && c != '.'
+ && prevC != '-' && prevC != '.'
+ )
+ {
+ ret.add(domain.substring(0, i) + "." + domain.substring(i, domain.length()));
+ }
+ }
+ return ret;
+ }
+
+ @Override
+ public String name() {
+ return "Subdomain";
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/0996b734/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/TranspositionStrategy.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/TranspositionStrategy.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/TranspositionStrategy.java
new file mode 100644
index 0000000..7962553
--- /dev/null
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/TranspositionStrategy.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.common.typosquat;
+
+import java.util.HashSet;
+import java.util.Set;
+
+public class TranspositionStrategy implements TyposquattingStrategy {
+ @Override
+ public Set<String> generateCandidates(String domain) {
+
+ Set<String> ret = new HashSet<>();
+ for(int i = 0; i < domain.length()-1;++i) {
+ char nextC = domain.charAt(i+1);
+ char c = domain.charAt(i);
+ if(nextC != c) {
+ ret.add( domain.substring(0, i)
+ + nextC
+ + c
+ + domain.substring(i+2)
+ );
+ }
+ }
+ return ret;
+ }
+
+ @Override
+ public String name() {
+ return "Transposition";
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/0996b734/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/TyposquattingStrategies.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/TyposquattingStrategies.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/TyposquattingStrategies.java
new file mode 100644
index 0000000..a9f9a41
--- /dev/null
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/TyposquattingStrategies.java
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.common.typosquat;
+
+import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.dsl.ParseException;
+import org.apache.metron.stellar.dsl.Stellar;
+import org.apache.metron.stellar.dsl.StellarFunction;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * This is a set of strategies for generating likely typos from domains.
+ */
+public enum TyposquattingStrategies implements TyposquattingStrategy {
+ ADDITION(new AdditionStrategy()),
+ BITSQUATTING(new BitsquattingStrategy()),
+ HOMOGLYPH(new HomoglyphStrategy()),
+ HYPHENATION(new HyphenationStrategy()),
+ INSERTION(new InsertionStrategy()),
+ OMISSION(new OmissionStrategy()),
+ REPETITION(new RepetitionStrategy()),
+ REPLACEMENT(new ReplacementStrategy()),
+ SUBDOMAIN(new SubdomainStrategy()),
+ TRANSPOSITION(new TranspositionStrategy()),
+ VOWELSWAP(new VowelSwapStrategy()),
+ ;
+ TyposquattingStrategy strategy;
+ TyposquattingStrategies(TyposquattingStrategy strategy) {
+ this.strategy = strategy;
+ }
+
+ @Override
+ public Set<String> generateCandidates(String originalString) {
+ Set<String> candidates = strategy.generateCandidates(originalString);
+ candidates.remove(originalString);
+ return candidates;
+ }
+
+ public static Set<String> generateAllCandidates(String originalString) {
+ Set<String> ret = new HashSet<>();
+ for(TyposquattingStrategy s : values() ) {
+ ret.addAll(s.generateCandidates(originalString));
+ }
+ return ret;
+ }
+
+ public static TyposquattingStrategies byName(String name) {
+ for(TyposquattingStrategies s : values()) {
+ if(s.strategy.name().equals(name)) {
+ return s;
+ }
+ }
+ return null;
+ }
+
+ @Stellar(namespace="DOMAIN"
+ ,name="TYPOSQUAT"
+ ,description="Generate potential typosquatted domain from a passed domain. Strategies largely match https://github.com/elceef/dnstwist"
+ ,params = {
+ "domain - Domain (without subdomains or TLD) to generate typosquatted domains for."
+ }
+ ,returns = "A set of potential typosquatted domains."
+ )
+ public static class Generate implements StellarFunction{
+
+ @Override
+ public Object apply(List<Object> args, Context context) throws ParseException {
+ if(args.size() == 0) {
+ return null;
+ }
+ Object dnObj = args.get(0);
+ if(dnObj == null) {
+ return null;
+ }
+ if(!(dnObj instanceof String)) {
+ throw new IllegalStateException("Expected a domain without subdomains or a TLD, but got " + dnObj);
+ }
+ return TyposquattingStrategies.generateAllCandidates((String)dnObj);
+ }
+
+ @Override
+ public void initialize(Context context) {
+
+ }
+
+ @Override
+ public boolean isInitialized() {
+ return true;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/0996b734/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/TyposquattingStrategy.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/TyposquattingStrategy.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/TyposquattingStrategy.java
new file mode 100644
index 0000000..37e3f16
--- /dev/null
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/TyposquattingStrategy.java
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.common.typosquat;
+
+import java.util.Set;
+
+public interface TyposquattingStrategy {
+ Set<String> generateCandidates(String domain);
+ String name();
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/0996b734/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/VowelSwapStrategy.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/VowelSwapStrategy.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/VowelSwapStrategy.java
new file mode 100644
index 0000000..0a3388d
--- /dev/null
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/VowelSwapStrategy.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.common.typosquat;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * A typo strategy around swapping vowels (e.g. omazon.com vs amazon.com)
+ */
+public class VowelSwapStrategy implements TyposquattingStrategy {
+ private static Set<Character> VOWELS = new HashSet<Character>() {{
+ add('a');
+ add('e');
+ add('i');
+ add('o');
+ add('u');
+ }};
+
+ @Override
+ public Set<String> generateCandidates(String domain) {
+
+ HashSet<String> ret = new HashSet<>();
+ for(int i = 0;i < domain.length();++i) {
+ char c = domain.charAt(i);
+ for(char vowel : VOWELS) {
+ if(VOWELS.contains(c)) {
+ ret.add( domain.substring(0, i)
+ + vowel
+ + domain.substring(i+1)
+ );
+ }
+ }
+ }
+ return ret;
+ }
+
+ @Override
+ public String name() {
+ return "Vowel-swap";
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/0996b734/metron-platform/metron-common/src/test/java/org/apache/metron/common/typosquat/TyposquattingStrategiesTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/typosquat/TyposquattingStrategiesTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/typosquat/TyposquattingStrategiesTest.java
new file mode 100644
index 0000000..d7f99f0
--- /dev/null
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/typosquat/TyposquattingStrategiesTest.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.common.typosquat;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+import org.apache.metron.stellar.common.utils.StellarProcessorUtils;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.*;
+
+@RunWith(Parameterized.class)
+public class TyposquattingStrategiesTest {
+
+ /*
+ Skipping the miscellaneous typosquatted domains as they rely on the TLD.
+ */
+ static Set<String> typesToSkip = new HashSet<String>() {{
+ add("Various");
+ add("Original*");
+ }};
+
+ //These are the output from DNS Twist ( https://github.com/elceef/dnstwist ). We want to ensure we match their output at minimum
+ static Map<String, EnumMap<TyposquattingStrategies, Set<String>>> expected = new HashMap<String, EnumMap<TyposquattingStrategies, Set<String>>>()
+ {{
+ put("amazon", new EnumMap<>(TyposquattingStrategies.class));
+ put("github", new EnumMap<>(TyposquattingStrategies.class));
+ }};
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ for(Map.Entry<String, EnumMap<TyposquattingStrategies, Set<String>>> kv : expected.entrySet()) {
+ try(BufferedReader br = new BufferedReader(new FileReader( "src/test/resources/typosquat/" + kv.getKey() + ".csv") ) )
+ {
+ for(String line = null;(line = br.readLine()) != null;) {
+ if(line.startsWith("#")) {
+ continue;
+ }
+ Iterable<String> tokens = Splitter.on(",").split(line);
+ String name = Iterables.get(tokens, 0);
+ String domain = Iterables.get(tokens, 1);
+ domain = domain.replaceAll(".com", "");
+ EnumMap<TyposquattingStrategies, Set<String>> expectedValues = kv.getValue();
+ if(typesToSkip.contains(name)) {
+ continue;
+ }
+ TyposquattingStrategies strategy = TyposquattingStrategies.byName(name);
+ Assert.assertNotNull("Couldn't find " + name, strategy);
+ Set<String> s = expectedValues.get(strategy);
+ if(s == null) {
+ s = new HashSet<>();
+ expectedValues.put(strategy, s);
+ }
+ s.add(domain);
+ }
+ }
+ }
+ }
+
+ @Parameterized.Parameters
+ public static Collection<Object[]> strategies() {
+ List<Object[]> ret = new ArrayList<>();
+ for(TyposquattingStrategies strategy : TyposquattingStrategies.values()) {
+ ret.add(new Object[] { strategy });
+ }
+ return ret;
+ }
+
+ TyposquattingStrategies strategy;
+ public TyposquattingStrategiesTest(TyposquattingStrategies strategy) {
+ this.strategy = strategy;
+ }
+
+ public void assertExpected(String domain, TyposquattingStrategies strategy) {
+ Set<String> expectedValues = expected.get(domain).get(strategy);
+ Set<String> actualValues = strategy.generateCandidates(domain);
+ Assert.assertFalse(actualValues.contains(domain));
+ {
+ Sets.SetView<String> vals = Sets.difference(expectedValues, actualValues);
+ String diff = Joiner.on(",").join(vals);
+ Assert.assertTrue(strategy.name() + ": Found values expected but not generated: " + diff, vals.isEmpty());
+ }
+ }
+
+ @Test
+ public void test() {
+ for(String domain : expected.keySet()) {
+ assertExpected(domain, strategy);
+ }
+ }
+
+ @Test
+ public void testStellar() {
+ for(String domain : expected.keySet()) {
+ Set<String> expectedAll = TyposquattingStrategies.generateAllCandidates(domain);
+ Set<String> generatedAll = (Set<String>) StellarProcessorUtils.run("DOMAIN_TYPOSQUAT(domain)", ImmutableMap.of("domain", domain));
+ Assert.assertTrue(Sets.symmetricDifference(expectedAll, generatedAll).isEmpty());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/0996b734/metron-platform/metron-common/src/test/resources/typosquat/amazon.csv
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/resources/typosquat/amazon.csv b/metron-platform/metron-common/src/test/resources/typosquat/amazon.csv
new file mode 100644
index 0000000..ceb4e93
--- /dev/null
+++ b/metron-platform/metron-common/src/test/resources/typosquat/amazon.csv
@@ -0,0 +1,259 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#fuzzer,domain-name,dns-a,dns-aaaa,dns-mx,dns-ns,geoip-country,whois-created,whois-updated,ssdeep-score
+Original*,amazon.com,176.32.103.205,,amazon-smtp.amazon.com,ns1.p31.dynect.net,,,,
+Addition,amazona.com,198.50.233.235,,,ns1.domainmx.com,,,,
+Addition,amazonb.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Addition,amazonc.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Addition,amazond.com,72.52.10.14,,,ns1.markmonitor.com,,,,
+Addition,amazone.com,87.238.81.156,,,,,,,
+Addition,amazonf.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Addition,amazong.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Addition,amazonh.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Addition,amazoni.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Addition,amazonj.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Addition,amazonk.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Addition,amazonl.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Addition,amazonm.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Addition,amazonn.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Addition,amazono.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Addition,amazonp.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Addition,amazonq.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Addition,amazonr.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Addition,amazons.com,54.197.246.244,,,ns-1019.awsdns-63.net,,,,
+Addition,amazont.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Addition,amazonu.com,,,,,,,,
+Addition,amazonv.com,207.171.166.22,,,,,,,
+Addition,amazonw.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Addition,amazonx.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Addition,amazony.com,207.171.166.22,,,,,,,
+Addition,amazonz.com,72.52.10.14,,,ns1.markmonitor.com,,,,
+Bitsquatting,cmazon.com,103.224.182.251,,mx92.m1bp.com,ns1.above.com,,,,
+Bitsquatting,emazon.com,72.52.10.14,,,ns1.markmonitor.com,,,,
+Bitsquatting,imazon.com,185.53.178.6,,mail.h-email.net,ns1.parkingcrew.net,,,,
+Bitsquatting,qmazon.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Bitsquatting,alazon.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Bitsquatting,aoazon.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Bitsquatting,aiazon.com,50.63.202.55,,mailstore1.secureserver.net,ns77.domaincontrol.com,,,,
+Bitsquatting,aeazon.com,,,,,,,,
+Bitsquatting,a-azon.com,50.63.202.4,,,ns07.domaincontrol.com,,,,
+Bitsquatting,amczon.com,207.171.166.22,,,,,,,
+Bitsquatting,amezon.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Bitsquatting,amizon.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Bitsquatting,amqzon.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Bitsquatting,amaxon.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Bitsquatting,amaron.com,72.143.24.146,,mx05-1.mycloudmailbox.com,ns61.worldnic.com,,,,
+Bitsquatting,amajon.com,185.53.177.30,,,ns1.parkingspa.com,,,,
+Bitsquatting,amaznn.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Bitsquatting,amazmn.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Bitsquatting,amazkn.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Bitsquatting,amazgn.com,185.53.178.7,,mail.h-email.net,ns1.parkingcrew.net,,,,
+Bitsquatting,amazoo.com,209.236.72.34,,amazoo.com,ns1.westservers.net,,,,
+Bitsquatting,amazol.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Bitsquatting,amazoj.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Bitsquatting,amazof.com,,,,ns-1.amazon.com,,,,
+Homoglyph,xn--mazon-qwa.com,,,,,,,,
+Homoglyph,xn--amzon-ucc.com,104.27.182.15,2400:cb00:2048:1::681b:b60f,,cass.ns.cloudflare.com,,,,
+Homoglyph,xn--mazon-zjc.com,,,,ns1.markmonitor.com,,,,
+Homoglyph,xn--mzon-pzbb.com,,,,,,,,
+Homoglyph,xn--mzon-9nab.com,,,,,,,,
+Homoglyph,amaz0n.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Homoglyph,xn--mzon-4nab.com,,,,ns1.markmonitor.com,,,,
+Homoglyph,xn--amzon-1jc.com,54.163.185.105,,,ns01.domaincontrol.com,,,,
+Homoglyph,xn--amzon-k11b.com,104.18.48.131,2400:cb00:2048:1::6812:3083,,kara.ns.cloudflare.com,,,,
+Homoglyph,annazon.com,207.171.166.22,,,,,,,
+Homoglyph,xn--mazon-i11b.com,,,,,,,,
+Homoglyph,xn--mzon-koab.com,,,,,,,,
+Homoglyph,xn--amzon-nra.com,,,,,,,,
+Homoglyph,arrazon.com,213.186.33.2,,mx1.ovh.net,dns.ovh.net,,,,
+Homoglyph,xn--amaon-x59a.com,,,,ns1gmz.name.com,,,,
+Homoglyph,xn--amazn-i91b.com,184.168.221.36,,mailstore1.secureserver.net,pdns09.domaincontrol.com,,,,
+Homoglyph,xn--amzon-sqa.com,198.105.244.111,2620:118:7002::1111,,a1-144.akam.net,,,,
+Homoglyph,xn--amazn-uce.com,,,,ns1.markmonitor.com,,,,
+Homoglyph,xn--mzon-znab.com,,,,ns1.markmonitor.com,,,,
+Homoglyph,xn--aazon-6xe.com,,,,ns1.markmonitor.com,,,,
+Homoglyph,xn--amaon-vuc.com,107.180.1.108,,mailstore1.secureserver.net,pdns09.domaincontrol.com,,,,
+Homoglyph,xn--mzon-poab.com,,,,,,,,
+Homoglyph,xn--amzon-swa.com,,,,,,,,
+Homoglyph,anazon.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Homoglyph,xn--mzon-fseb.com,198.105.244.228,,,,,,,
+Homoglyph,xn--amzon-yqa.com,69.64.147.242,,,ns1bdg.name.com,,,,
+Homoglyph,xn--aazon-919a.com,198.105.244.228,,,,,,,
+Homoglyph,xn--amzon-4qa.com,198.105.244.228,,,,,,,
+Homoglyph,xn--mzon-4q5ab.com,,,,,,,,
+Homoglyph,xn--amzon-hra.com,,,,,,,,
+Homoglyph,xn--mzon-foab.com,198.105.244.228,,,,,,,
+Homoglyph,xn--mzon-p5bb.com,,,,ns1.markmonitor.com,,,,
+Homoglyph,xn--amazn-xob.com,50.63.202.43,,mailstore1.secureserver.net,pdns09.domaincontrol.com,,,,
+Homoglyph,xn--amazn-581b.com,184.168.221.57,,mailstore1.secureserver.net,pdns09.domaincontrol.com,,,,
+Homoglyph,xn--mazon-3ve.com,,,,ns1.markmonitor.com,,,,
+Homoglyph,xn--mazon-jwb.com,,,,,,,,
+Homoglyph,xn--amaon-kib.com,,,,ns-1417.awsdns-49.org,,,,
+Homoglyph,xn--amazn-okg.com,,,,,,,,
+Homoglyph,xn--amazn-mye.com,,,,,,,,
+Homoglyph,xn--amazn-lsf.com,,,,,,,,
+Homoglyph,xn--mzon-zsab.com,198.105.244.228,,,,,,,
+Homoglyph,xn--amzon-bra.com,50.63.202.58,,,ns33.domaincontrol.com,,,,
+Homoglyph,xn--mzon-zmbb.com,198.105.244.228,,,,,,,
+Homoglyph,amazor.com,68.178.213.61,,,ns1.namefind.com,,,,
+Homoglyph,xn--mazon-1of.com,,,,,,,,
+Homoglyph,amazom.com,174.129.240.167,,,ns-1.amazon.com,,,,
+Homoglyph,xn--mazon-wqa.com,66.96.149.32,,mx.xn--mazon-wqa.com,ns1.dotster.com,,,,
+Homoglyph,xn--amazn-3ta.com,,,,ns1.markmonitor.com,,,,
+Homoglyph,xn--amazn-uce.com,,,,ns1.markmonitor.com,,,,
+Homoglyph,xn--mazon-fra.com,,,,ns1.markmonitor.com,,,,
+Homoglyph,arnazon.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Homoglyph,xn--amazn-mua.com,184.168.221.32,,mailstore1.secureserver.net,ns19.domaincontrol.com,,,,
+Homoglyph,xn--aazon-ipc.com,,,,,,,,
+Homoglyph,xn--mazon-8qa.com,184.168.221.32,,mailstore1.secureserver.net,ns05.domaincontrol.com,,,,
+Homoglyph,xn--amaon-7hb.com,207.244.67.138,,,ns1.wombatdns.com,,,,
+Homoglyph,xn--aazon-fl1b.com,91.215.153.210,,mailstore1.secureserver.net,ns23.domaincontrol.com,,,,
+Homoglyph,xn--amazn-mye.com,,,,,,,,
+Homoglyph,xn--mazon-2qa.com,103.224.212.222,,mx92.m1bp.com,ns1.above.com,,,,
+Homoglyph,xn--mazon-scc.com,,,,,,,,
+Homoglyph,xn--mazon-qqa.com,199.59.242.150,,mx76.m2bp.com,ns1.bodis.com,,,,
+Homoglyph,xn--mazon-lra.com,,,,ns1.markmonitor.com,,,,
+Homoglyph,xn--amzon-lwb.com,,,,,,,,
+Homoglyph,xn--amzon-5ve.com,,,,ns1.markmonitor.com,,,,
+Homoglyph,xn--amazn-9dc.com,184.168.221.46,,mailstore1.secureserver.net,ns37.domaincontrol.com,,,,
+Homoglyph,xn--amazo-07a.com,,,,,,,,
+Homoglyph,xn--mzon-43db.com,,,,ns1.markmonitor.com,,,,
+Homoglyph,xn--amzon-3of.com,,,,,,,,
+Hyphenation,a-mazon.com,72.52.10.14,,,ns1.markmonitor.com,,,,
+Hyphenation,am-azon.com,,,,,,,,
+Hyphenation,ama-zon.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Hyphenation,amaz-on.com,72.52.10.14,,,ns1.markmonitor.com,,,,
+Hyphenation,amazo-n.com,,,,ns-1.amazon.com,,,,
+Insertion,amsazon.com,72.52.10.14,,,ns1.markmonitor.com,,,,
+Insertion,amyazon.com,207.171.166.22,,,,,,,
+Insertion,amazqon.com,103.224.182.214,,mx92.m1bp.com,ns1.above.com,,,,
+Insertion,amjazon.com,,,,ns-1.amazon.com,,,,
+Insertion,anmazon.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Insertion,ama3zon.com,198.105.244.228,,,,,,,
+Insertion,ajmazon.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Insertion,amayzon.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Insertion,apmazon.com,103.224.182.207,,mx92.m1bp.com,ns1.above.com,,,,
+Insertion,amazxon.com,,,,,,,,
+Insertion,am1azon.com,,,,,,,,
+Insertion,ama2zon.com,,,,,,,,
+Insertion,amwazon.com,72.52.10.14,,,ns1.markmonitor.com,,,,
+Insertion,amazokn.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Insertion,ama6zon.com,,,,,,,,
+Insertion,amazuon.com,72.52.10.14,,,ns1.markmonitor.com,,,,
+Insertion,amnazon.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Insertion,amazpon.com,,,,,,,,
+Insertion,amaz6on.com,72.52.10.14,,,ns1.markmonitor.com,,,,
+Insertion,amazton.com,,,,ns-1.amazon.com,,,,
+Insertion,amaz9on.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Insertion,am2azon.com,,,,,,,,
+Insertion,amazson.com,54.235.212.68,,mail.amazson.com,ns1.pql.net,,,,
+Insertion,ampazon.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Insertion,ama1zon.com,,,,,,,,
+Insertion,amlazon.com,,,,,,,,
+Insertion,amawzon.com,184.168.221.15,,mailstore1.secureserver.net,ns11.domaincontrol.com,,,,
+Insertion,amaz7on.com,,,,,,,,
+Insertion,amazo9n.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Insertion,almazon.com,208.91.197.128,,p.webcom.ctmail.com,ns45.worldnic.com,,,,
+Insertion,amazlon.com,,,,,,,,
+Insertion,amaszon.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Insertion,amatzon.com,,,,ns-1.amazon.com,,,,
+Insertion,amazopn.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Insertion,amahzon.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Insertion,amazgon.com,72.52.10.14,,,ns1.markmonitor.com,,,,
+Insertion,amaz0on.com,207.171.166.22,,,,,,,
+Insertion,amazoin.com,207.171.166.22,,,,,,,
+Insertion,amazaon.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Insertion,amaezon.com,207.171.166.22,,,,,,,
+Insertion,amkazon.com,207.171.166.22,,,,,,,
+Insertion,amqazon.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Insertion,amaxzon.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Insertion,amauzon.com,,,,,,,,
+Insertion,amazzon.com,,,,ns-1.amazon.com,,,,
+Insertion,akmazon.com,,,,ns-1.amazon.com,,,,
+Insertion,amazhon.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Insertion,amazion.com,,,,ns-1.amazon.com,,,,
+Insertion,amaazon.com,,,,ns-1.amazon.com,,,,
+Insertion,amazoln.com,207.171.166.22,,,,,,,
+Insertion,amagzon.com,185.53.178.7,,mail.h-email.net,ns1.parkingcrew.net,,,,
+Insertion,amazeon.com,208.73.210.202,,,ns1.dsredirection.com,,,,
+Insertion,amaz3on.com,198.105.244.228,,,,,,,
+Insertion,amzazon.com,,,,,,,,
+Insertion,amazo0n.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Insertion,amaqzon.com,207.171.166.22,,,,,,,
+Insertion,amaz2on.com,,,,,,,,
+Insertion,ama7zon.com,,,,,,,,
+Insertion,amazkon.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Omission,amaon.com,207.171.166.22,,,,,,,
+Omission,mazon.com,184.168.221.23,,mazon-com.mail.protection.outlook.com,ns05.domaincontrol.com,,,,
+Omission,amzon.com,207.171.166.22,,,,,,,
+Omission,aazon.com,144.76.0.242,,,ns1.ndsplitter.com,,,,
+Omission,amazo.com,144.76.0.242,,,ns1.ndsplitter.com,,,,
+Omission,amazn.com,207.171.166.22,,,pdns1.ultradns.net,,,,
+Repetition,aamazon.com,,,,,,,,
+Repetition,amazoon.com,,,,,,,,
+Repetition,ammazon.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Replacement,amazln.com,207.171.166.22,,,,,,,
+Replacement,amwzon.com,,,,pdns1.ultradns.net,,,,
+Replacement,ajazon.com,,,,,,,,
+Replacement,am2zon.com,198.105.244.228,,,,,,,
+Replacement,amaton.com,208.91.197.26,,,,,,,
+Replacement,amzzon.com,,,,,,,,
+Replacement,ama3on.com,184.168.221.43,,mailstore1.secureserver.net,ns03.domaincontrol.com,,,,
+Replacement,amaaon.com,72.52.4.122,,localhost,ns1.sedoparking.com,,,,
+Replacement,ama2on.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Replacement,ama7on.com,72.52.10.14,,,ns1.markmonitor.com,,,,
+Replacement,amahon.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Replacement,ymazon.com,72.52.10.14,,,ns1.markmonitor.com,,,,
+Replacement,amagon.com,208.91.197.39,,,sk.s5.ans1.ns121.ztomy.com,,,,
+Replacement,wmazon.com,,,,ns-1.amazon.com,,,,
+Replacement,amazin.com,,,,,,,,
+Replacement,amaeon.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Replacement,apazon.com,72.52.10.14,,,ns1.markmonitor.com,,,,
+Replacement,amazpn.com,,,,ns-1.amazon.com,,,,
+Replacement,am1zon.com,216.239.32.21,2001:4860:4802:32::15,,ns-cloud-b1.googledomains.com,,,,
+Replacement,amaz9n.com,,,,ns-1.amazon.com,,,,
+Replacement,amazoh.com,,,,,,,,
+Replacement,ama6on.com,72.52.10.14,,,ns1.markmonitor.com,,,,
+Replacement,amyzon.com,72.52.10.14,,,ns1.markmonitor.com,,,,
+Replacement,amszon.com,,,,pdns1.ultradns.net,,,,
+Replacement,amazob.com,207.171.166.22,,,,,,,
+Replacement,1mazon.com,172.217.8.211,2607:f8b0:4009:811::2013,,,,,,
+Replacement,amason.com,212.47.223.137,,smx1.web-hosting.com,dns1.namecheaphosting.com,,,,
+Replacement,amauon.com,207.171.166.22,,,,,,,
+Replacement,smazon.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Replacement,zmazon.com,,,,,,,,
+Replacement,amaqon.com,,,,,,,,
+Replacement,2mazon.com,198.105.244.228,,,,,,,
+Replacement,akazon.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Subdomain,a.mazon.com,,,,,,,,
+Subdomain,am.azon.com,199.16.189.254,,,,,,,
+Subdomain,ama.zon.com,184.168.221.104,,,ns1.afternic.com,,,,
+Subdomain,amaz.on.com,208.78.94.160,,,,,,,
+Subdomain,amazo.n.com,,,,,,,,
+Transposition,maazon.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Transposition,aamzon.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Transposition,amzaon.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Transposition,amaozn.com,207.171.166.22,,,,,,,
+Transposition,amazno.com,,,,ns-1.amazon.com,,,,
+Vowel-swap,omazon.com,208.91.196.152,,,sk.s7.ans1.ns147.klczy.com,,,,
+Vowel-swap,amuzon.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Vowel-swap,amazan.com,54.255.202.167,,,ns1.meganameservers.eu,,,,
+Vowel-swap,amazen.com,199.59.242.150,,mx76.m2bp.com,ns1.bodis.com,,,,
+Vowel-swap,amozon.com,207.171.166.22,,,ns-1.amazon.com,,,,
+Vowel-swap,amazun.com,216.21.239.197,,mail2.siteamerica.com,dns143.b.register.com,,,,
+Vowel-swap,umazon.com,52.71.185.125,,,ns1.namebrightdns.com,,,,
+Various,amazoncom.com,208.73.210.202,,,ns1.dsredirection.com,,,,
http://git-wip-us.apache.org/repos/asf/metron/blob/0996b734/metron-platform/metron-common/src/test/resources/typosquat/github.csv
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/resources/typosquat/github.csv b/metron-platform/metron-common/src/test/resources/typosquat/github.csv
new file mode 100644
index 0000000..1965bf2
--- /dev/null
+++ b/metron-platform/metron-common/src/test/resources/typosquat/github.csv
@@ -0,0 +1,227 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#fuzzer,domain-name,dns-a,dns-aaaa,dns-mx,dns-ns,geoip-country,whois-created,whois-updated,ssdeep-score
+Original*,github.com,192.30.253.112,,ALT1.ASPMX.L.GOOGLE.com,ns-1283.awsdns-32.org,,,,
+Addition,githuba.com,184.168.221.50,,mailstore1.secureserver.net,ns37.domaincontrol.com,,,,
+Addition,githubb.com,103.16.130.124,,mail.ext.io,ns1.afraid.org,,,,
+Addition,githubc.com,103.224.212.222,,mx92.m1bp.com,ns1.above.com,,,,
+Addition,githubd.com,198.105.244.228,,,,,,,
+Addition,githube.com,81.171.22.6,,,ns1.weaponizedcow.com,,,,
+Addition,githubf.com,198.105.244.228,,,,,,,
+Addition,githubg.com,198.105.244.228,,,,,,,
+Addition,githubh.com,198.105.244.228,,,,,,,
+Addition,githubi.com,184.168.221.50,,mailstore1.secureserver.net,ns37.domaincontrol.com,,,,
+Addition,githubj.com,198.105.244.228,,,,,,,
+Addition,githubk.com,198.105.244.228,,,,,,,
+Addition,githubl.com,198.105.244.228,,,,,,,
+Addition,githubm.com,198.105.244.228,,,,,,,
+Addition,githubn.com,198.105.244.228,,,,,,,
+Addition,githubo.com,119.28.71.213,,,f1g1ns1.dnspod.net,,,,
+Addition,githubp.com,198.105.244.228,,,,,,,
+Addition,githubq.com,54.172.109.248,,eforward1.registrar-servers.com,dns1.registrar-servers.com,,,,
+Addition,githubr.com,74.208.236.31,2607:f1c0:100f:f000::252,mx00.1and1.com,ns1087.ui-dns.biz,,,,
+Addition,githubs.com,162.255.119.247,,eforward1.registrar-servers.com,dns1.registrar-servers.com,,,,
+Addition,githubt.com,67.227.226.240,,,ns1.parklogic.com,,,,
+Addition,githubu.com,106.187.91.218,,,ns1cnb.name.com,,,,
+Addition,githubv.com,,,,a.dnspod.com,,,,
+Addition,githubw.com,198.105.244.228,,,,,,,
+Addition,githubx.com,,,,,,,,
+Addition,githuby.com,198.105.244.228,,,,,,,
+Addition,githubz.com,198.105.244.228,,,,,,,
+Bitsquatting,fithub.com,4.35.164.141,,alt1.aspmx.l.google.com,ns1.monikerdns.net,,,,
+Bitsquatting,eithub.com,204.11.56.48,,,ns1626.ztomy.com,,,,
+Bitsquatting,cithub.com,,,,,,,,
+Bitsquatting,oithub.com,185.53.177.31,,,ns1.protectdelegation.ca,,,,
+Bitsquatting,withub.com,23.234.4.120,,,ns1.eedns.com,,,,
+Bitsquatting,ghthub.com,185.53.177.31,,,ns1.protectdelegation.ca,,,,
+Bitsquatting,gkthub.com,104.18.62.97,2400:cb00:2048:1::6812:3e61,mx.yandex.ru,sue.ns.cloudflare.com,,,,
+Bitsquatting,gmthub.com,49.236.200.154,,bkpmail.localdns.com,ns14.localdns.com,,,,
+Bitsquatting,gathub.com,216.250.120.46,2607:f1c0:1000:70f6:8ab4:44d2:262a:e822,mx00.1and1.com,ns1028.ui-dns.biz,,,,
+Bitsquatting,gythub.com,185.53.177.31,,,ns1.protectdelegation.ca,,,,
+Bitsquatting,giuhub.com,185.53.177.31,,,ns1.protectdelegation.ca,,,,
+Bitsquatting,givhub.com,185.53.178.7,,mail.h-email.net,ns1.parkingcrew.net,,,,
+Bitsquatting,giphub.com,47.90.201.75,,smtp.giphub.com,dns7.hichina.com,,,,
+Bitsquatting,gidhub.com,67.55.92.184,,mx7.gidhub.com,ns1.liverealdeals.com,,,,
+Bitsquatting,gi4hub.com,185.53.177.31,,,ns1.protectdelegation.ca,,,,
+Bitsquatting,gitiub.com,185.53.177.31,,,ns1.protectdelegation.ca,,,,
+Bitsquatting,gitjub.com,91.134.139.14,,,ns65.domaincontrol.com,,,,
+Bitsquatting,gitlub.com,166.78.103.6,,,,,,,
+Bitsquatting,gitxub.com,185.53.177.31,,,ns1.protectdelegation.ca,,,,
+Bitsquatting,githtb.com,185.53.177.31,,,ns1.protectdelegation.ca,,,,
+Bitsquatting,githwb.com,108.59.81.6,,,,,,,
+Bitsquatting,githqb.com,185.53.177.31,,,ns1.protectdelegation.ca,,,,
+Bitsquatting,githeb.com,185.53.177.31,,,ns1.protectdelegation.ca,,,,
+Bitsquatting,gith5b.com,185.53.177.31,,,ns1.protectdelegation.ca,,,,
+Bitsquatting,githuc.com,172.246.29.53,,,juming.dnsdun.com,,,,
+Bitsquatting,githuf.com,185.53.177.31,,,ns1.protectdelegation.ca,,,,
+Bitsquatting,githuj.com,104.31.90.172,2400:cb00:2048:1::681f:5aac,,coco.ns.cloudflare.com,,,,
+Bitsquatting,githur.com,104.27.140.182,2400:cb00:2048:1::681b:8cb6,dc-0831986e3235.githur.com,coco.ns.cloudflare.com,,,,
+Homoglyph,xn--githb-oxb.com,198.105.244.228,,,,,,,
+Homoglyph,xn--ithub-v1a.com,,,,,,,,
+Homoglyph,xn--gitub-fye.com,198.105.244.228,,,,,,,
+Homoglyph,xn--gitub-1gg.com,198.105.244.228,,,,,,,
+Homoglyph,xn--ithub-72b.com,198.105.244.228,,,,,,,
+Homoglyph,xn--gthub-zsa.com,,,,ns-cloud-c1.googledomains.com,,,,
+Homoglyph,xn--gitub-0kf.com,198.105.244.228,,,,,,,
+Homoglyph,xn--githb-bjg.com,198.105.244.228,,,,,,,
+Homoglyph,xn--githu-fwe.com,198.105.244.228,,,,,,,
+Homoglyph,g1thub.com,198.105.244.228,,,,,,,
+Homoglyph,gitlhub.com,198.105.244.228,,,,,,,
+Homoglyph,xn--githu-hwc.com,198.105.244.228,,,,,,,
+Homoglyph,githuib.com,141.22.27.142,,,ns1.domaindiscount24.net,,,,
+Homoglyph,xn--gitub-ify.com,198.105.244.228,,,,,,,
+Homoglyph,xn--gihub-8ye.com,198.105.244.228,,,,,,,
+Homoglyph,gitihub.com,173.239.5.6,,mx7.gitihub.com,ns1.expiereddnsmanager.com,,,,
+Homoglyph,xn--githu-10e.com,198.105.244.228,,,,,,,
+Homoglyph,xn--gthub-h9x.com,198.105.244.228,,,,,,,
+Homoglyph,qithub.com,184.168.221.34,,,ns13.domaincontrol.com,,,,
+Homoglyph,xn--githb-vde.com,198.105.244.228,,,,,,,
+Homoglyph,xn--ithub-zyf.com,198.105.244.228,,,,,,,
+Homoglyph,glthub.com,52.71.185.125,,,ns1.namebrightdns.com,,,,
+Homoglyph,xn--gthub-n4a.com,,,ec2-52-42-4-3.us-west-2.compute.amazonaws.com,dns1.registrar-servers.com,,,,
+Homoglyph,xn--gthub-wwb.com,198.105.244.228,,,,,,,
+Homoglyph,xn--ithub-qmc.com,50.63.202.41,,mailstore1.secureserver.net,ns25.domaincontrol.com,,,,
+Homoglyph,xn--gthub-y3a.com,198.105.244.228,,,,,,,
+Homoglyph,xn--gthub-cta.com,162.255.119.248,,eforward1.registrar-servers.com,dns1.registrar-servers.com,,,,
+Homoglyph,xn--gihub-8db.com,198.105.244.228,,,,,,,
+Homoglyph,xn--gihub-nde.com,198.105.244.228,,,,,,,
+Homoglyph,xn--githb-bjg.com,198.105.244.228,,,,,,,
+Homoglyph,xn--ithub-71a.com,198.105.244.228,,,,,,,
+Homoglyph,xn--ithub-vjg.com,198.105.244.228,,,,,,,
+Homoglyph,xn--gthub-4nc.com,198.105.244.228,,,,,,,
+Homoglyph,githulb.com,198.105.244.228,,,,,,,
+Homoglyph,githud.com,198.105.244.228,,,,,,,
+Homoglyph,xn--githb-x49a.com,69.64.147.242,,,ns1cny.name.com,,,,
+Homoglyph,xn--ithub-j1a.com,198.105.244.228,,,,,,,
+Homoglyph,xn--githu-hkc.com,198.105.244.228,,,,,,,
+Homoglyph,xn--ithub-wmc.com,69.64.147.10,,,dns1.name-services.com,,,,
+Homoglyph,xn--gthub-qbe.com,198.105.244.228,,,,,,,
+Homoglyph,xn--gthub-k41s.com,198.105.244.228,,,,,,,
+Homoglyph,xn--githb-cce.com,198.105.244.228,,,,,,,
+Homoglyph,xn--githb-0fb.com,198.105.244.228,,,,,,,
+Homoglyph,xn--github-g1d.com,198.105.244.228,,,,,,,
+Homoglyph,xn--githb-zze.com,,,,,,,,
+Hyphenation,g-ithub.com,198.105.244.228,,,,,,,
+Hyphenation,gi-thub.com,198.105.244.228,,,,,,,
+Hyphenation,git-hub.com,192.30.253.167,,,ns1.p16.dynect.net,,,,
+Hyphenation,gith-ub.com,198.105.244.228,,,,,,,
+Hyphenation,githu-b.com,198.105.244.228,,,,,,,
+Insertion,girthub.com,209.99.40.222,,,dns10.parkpage.foundationapi.com,,,,
+Insertion,githiub.com,67.227.226.240,,mx156.hostedmxserver.com,ns1.parklogic.com,,,,
+Insertion,g8ithub.com,198.105.244.228,,,,,,,
+Insertion,githu8b.com,198.105.244.228,,,,,,,
+Insertion,giythub.com,198.105.244.228,,,,,,,
+Insertion,gitbhub.com,67.55.92.182,,mx7.gitbhub.com,ns3.renewyourexpiereddomain.com,,,,
+Insertion,gith8ub.com,198.105.244.228,,,,,,,
+Insertion,githuyb.com,198.105.244.228,,,,,,,
+Insertion,giothub.com,198.105.244.228,,,,,,,
+Insertion,githyub.com,198.105.244.228,,,,,,,
+Insertion,gi6thub.com,198.105.244.228,,,,,,,
+Insertion,gitzhub.com,207.244.67.214,,,ns1.dnsnuts.com,,,,
+Insertion,gityhub.com,,,,,,,,
+Insertion,gi5thub.com,198.105.244.228,,,,,,,
+Insertion,gituhub.com,69.162.80.57,,,ns1.hastydns.com,,,,
+Insertion,goithub.com,,,,,,,,
+Insertion,gitnhub.com,198.105.244.228,,,,,,,
+Insertion,githgub.com,69.64.147.47,,,dns1.name-services.com,,,,
+Insertion,githu7b.com,198.105.244.228,,,,,,,
+Insertion,githnub.com,198.105.244.228,,,,,,,
+Insertion,git6hub.com,198.105.244.228,,,,,,,
+Insertion,githhub.com,173.239.22.42,,,ns1.mnsdnsmart.com,,,,
+Insertion,guithub.com,78.41.204.28,,,ns1.torresdns.com,,,,
+Insertion,githjub.com,198.105.244.228,,,,,,,
+Insertion,gkithub.com,198.105.244.228,,,,,,,
+Insertion,gigthub.com,198.105.244.228,,,,,,,
+Insertion,githuzb.com,198.105.244.228,,,,,,,
+Insertion,gi9thub.com,198.105.244.228,,,,,,,
+Insertion,githuub.com,104.27.184.65,2400:cb00:2048:1::681b:b841,,april.ns.cloudflare.com,,,,
+Insertion,githzub.com,198.105.244.228,,,,,,,
+Insertion,gitjhub.com,198.105.244.228,,,,,,,
+Insertion,gikthub.com,198.105.244.228,,,,,,,
+Insertion,githujb.com,198.105.244.228,,,,,,,
+Insertion,gitghub.com,185.151.28.157,2a07:7800::157,mx.stackmail.com,ns1.eco-dns.co.uk,,,,
+Insertion,githbub.com,103.224.212.222,,mx92.m1bp.com,ns1.above.com,,,,
+Insertion,gitrhub.com,141.8.224.93,,,ns7.rookdns.com,,,,
+Insertion,gjithub.com,198.105.244.228,,,,,,,
+Insertion,gizthub.com,198.105.244.228,,,,,,,
+Insertion,githuhb.com,198.105.244.228,,,,,,,
+Insertion,gi8thub.com,198.105.244.228,,,,,,,
+Insertion,giuthub.com,95.211.219.67,,,ns1.hastydns.com,,,,
+Insertion,gitfhub.com,198.105.244.228,,,,,,,
+Insertion,git5hub.com,198.105.244.228,,,,,,,
+Insertion,g9ithub.com,198.105.244.228,,,,,,,
+Insertion,gifthub.com,176.74.176.187,,,ns1.uniregistrymarket.link,,,,
+Insertion,gith7ub.com,198.105.244.228,,,,,,,
+Insertion,gijthub.com,198.105.244.228,,,,,,,
+Omission,gihub.com,208.73.210.202,,,ns1.dsredirection.com,,,,
+Omission,githu.com,69.172.201.153,,,ns1.uniregistrymarket.link,,,,
+Omission,ithub.com,,,,,,,,
+Omission,gitub.com,68.178.213.61,,,ns1.namefind.com,,,,
+Omission,gthub.com,72.52.179.174,,mx156.hostedmxserver.com,ns1.parklogic.com,,,,
+Omission,githb.com,109.201.135.44,,,ns1.kirklanddc.com,,,,
+Repetition,ggithub.com,52.69.166.231,,,,,,,
+Repetition,giithub.com,199.59.242.150,,mx76.m2bp.com,ns1.bodis.com,,,,
+Repetition,gitthub.com,54.71.255.210,,gitthub.com,ns3.ifund.com,,,,
+Replacement,gitgub.com,,,,,,,,
+Replacement,githyb.com,185.53.179.7,,mail.h-email.net,ns1.parkingcrew.net,,,,
+Replacement,yithub.com,121.42.226.133,,,dns10.hichina.com,,,,
+Replacement,tithub.com,176.74.176.187,,,ns1.uniregistrymarket.link,,,,
+Replacement,girhub.com,173.239.22.42,,,ns1.mnsdnsmart.com,,,,
+Replacement,gighub.com,,,,,,,,
+Replacement,zithub.com,,,,,,,,
+Replacement,gith8b.com,198.105.244.228,,,,,,,
+Replacement,gityub.com,50.63.202.50,,mailstore1.secureserver.net,ns01.domaincontrol.com,,,,
+Replacement,guthub.com,192.30.253.167,,,ns1.p16.dynect.net,,,,
+Replacement,hithub.com,50.63.202.32,,mailstore1.secureserver.net,ns23.domaincontrol.com,,,,
+Replacement,gith7b.com,107.161.23.204,,,ns1.dnsowl.com,,,,
+Replacement,githjb.com,34.233.88.36,,,ns09.domaincontrol.com,,,,
+Replacement,vithub.com,78.46.90.242,,mail.vithub.com,ns1.first-ns.de,,,,
+Replacement,g8thub.com,107.161.23.204,,,ns1.dnsowl.com,,,,
+Replacement,gifhub.com,184.168.221.23,,mailstore1.secureserver.net,ns45.domaincontrol.com,,,,
+Replacement,gitnub.com,,,,ns1.dnsimple.com,,,,
+Replacement,gjthub.com,98.124.245.24,,mail.b-io.co,ns1.fabulous.com,,,,
+Replacement,githzb.com,198.105.244.228,,,,,,,
+Replacement,githuv.com,103.224.182.239,,mx92.m1bp.com,ns1.above.com,,,,
+Replacement,giyhub.com,103.224.182.239,,mx92.m1bp.com,ns1.above.com,,,,
+Replacement,gothub.com,50.63.202.25,,mailstore1.secureserver.net,ns05.domaincontrol.com,,,,
+Replacement,g9thub.com,198.105.244.228,,,,,,,
+Replacement,gi5hub.com,107.161.23.204,,,ns1.dnsowl.com,,,,
+Replacement,gizhub.com,52.216.132.18,,alt1.aspmx.l.google.com,ns-1107.awsdns-10.org,,,,
+Replacement,githhb.com,198.105.244.228,,,,,,,
+Replacement,githug.com,74.124.210.249,,githug.com,ns.inmotionhosting.com,,,,
+Replacement,githuh.com,97.107.140.81,,,dns1.registrar-servers.com,,,,
+Replacement,githun.com,185.53.179.6,,mail.h-email.net,ns1.parkingcrew.net,,,,
+Replacement,bithub.com,,,,,,,,
+Replacement,githib.com,,,,,,,,
+Replacement,gi6hub.com,198.105.244.228,,,,,,,
+Replacement,gituub.com,184.168.221.44,,mailstore1.secureserver.net,ns57.domaincontrol.com,,,,
+Replacement,gitbub.com,199.59.242.150,,mx76.m2bp.com,ns1.bodis.com,,,,
+Replacement,gitzub.com,198.105.244.228,,,,,,,
+Subdomain,g.ithub.com,,,,,,,,
+Subdomain,gi.thub.com,,,,,,,,
+Subdomain,git.hub.com,,,,,,,,
+Subdomain,gith.ub.com,13.228.110.155,,,,,,,
+Subdomain,githu.b.com,,,,,,,,
+Transposition,igthub.com,,,smtp1.phocine.net,ns1djs.name.com,,,,
+Transposition,gtihub.com,199.59.242.150,,mx76.m2bp.com,ns1.bodis.com,,,,
+Transposition,gihtub.com,199.59.242.150,,mx76.m2bp.com,ns1.bodis.com,,,,
+Transposition,gituhb.com,103.224.182.241,,mx92.m1bp.com,ns1.above.com,,,,
+Transposition,githbu.com,167.114.142.2,,,ns1.chookdns.com,,,,
+Vowel-swap,githob.com,173.239.5.6,,mx7.githob.com,ns1.expiereddnsmanager.com,,,,
+Vowel-swap,githab.com,72.52.4.120,,localhost,ns1.sedoparking.com,,,,
+Vowel-swap,gethub.com,,,,ns.ktnet.co.kr,,,,
+Various,githubcom.com,185.53.179.7,,mail.h-email.net,ns1.parkingcrew.net,,,,
http://git-wip-us.apache.org/repos/asf/metron/blob/0996b734/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/FunctionalFunctions.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/FunctionalFunctions.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/FunctionalFunctions.java
index d45ffcb..1e30dae 100644
--- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/FunctionalFunctions.java
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/FunctionalFunctions.java
@@ -41,7 +41,7 @@ public class FunctionalFunctions {
@Override
public Object apply(List<Object> args) {
- List<Object> input = (List<Object>) args.get(0);
+ Iterable<Object> input = (Iterable<Object>) args.get(0);
LambdaExpression expression = (LambdaExpression)args.get(1);
if(input == null || expression == null) {
return input;
@@ -66,7 +66,7 @@ public class FunctionalFunctions {
@Override
public Object apply(List<Object> args) {
- List<Object> input = (List<Object>) args.get(0);
+ Iterable<Object> input = (Iterable<Object>) args.get(0);
LambdaExpression expression = (LambdaExpression) args.get(1);
if(input == null || expression == null) {
return input;
@@ -95,7 +95,7 @@ public class FunctionalFunctions {
@Override
public Object apply(List<Object> args) {
- List<Object> input = (List<Object>) args.get(0);
+ Iterable<Object> input = (Iterable<Object>) args.get(0);
if(input == null || args.size() < 3) {
return null;
}
@@ -105,8 +105,7 @@ public class FunctionalFunctions {
if(expression == null || runningResult == null) {
return null;
}
- for(int i = 0;i < input.size();++i) {
- Object rhs = input.get(i);
+ for(Object rhs : input) {
runningResult = expression.apply(listOf(runningResult, rhs));
}
return runningResult;