You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@metron.apache.org by GitBox <gi...@apache.org> on 2019/06/03 18:20:14 UTC
[GitHub] [metron] mmiklavc commented on issue #1436: METRON-2149: Shaded jar
classifier is not consistent
mmiklavc commented on issue #1436: METRON-2149: Shaded jar classifier is not consistent
URL: https://github.com/apache/metron/pull/1436#issuecomment-498369511
I did a crude grep through our Stellar function classes looking for non-java, non-metron dependencies, and this is what I came up with. This is a hit-list of functions that are probably worth a test of some sort or another to verify we haven't broken anything. It might be worth adding a more robust automated test at some point that we can spin up as a topology and include in release validation.
```
for file in $(grep -lR --include \*.java "@Stellar" .)
do
if grep "^import" $file | grep -v " java\| org.apache.metron";
then
echo " $file matches"
echo "====================="
fi
done > /tmp/matchlist.txt
import org.apache.commons.validator.routines.LongValidator;
./metron-platform/metron-common/src/main/java/org/apache/metron/common/field/validation/primitive/IntegerValidation.java matches
=====================
import org.apache.commons.validator.routines.EmailValidator;
./metron-platform/metron-common/src/main/java/org/apache/metron/common/field/validation/network/EmailValidation.java matches
=====================
import org.apache.commons.validator.routines.DomainValidator;
./metron-platform/metron-common/src/main/java/org/apache/metron/common/field/validation/network/DomainValidation.java matches
=====================
import org.apache.commons.validator.routines.InetAddressValidator;
./metron-platform/metron-common/src/main/java/org/apache/metron/common/field/validation/network/IPValidation.java matches
=====================
import org.apache.commons.validator.routines.UrlValidator;
./metron-platform/metron-common/src/main/java/org/apache/metron/common/field/validation/network/URLValidation.java matches
=====================
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
./metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/GeoEnrichmentFunctions.java matches
=====================
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
./metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/AsnEnrichmentFunctions.java matches
=====================
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
./metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/ObjectGet.java matches
=====================
import ch.hsr.geohash.WGS84Point;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
./metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/GeoHashFunctions.java matches
=====================
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
./metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/SimpleHBaseEnrichmentFunctions.java matches
=====================
import com.fasterxml.jackson.core.JsonProcessingException;
import org.apache.curator.framework.CuratorFramework;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
./metron-platform/metron-management/src/main/java/org/apache/metron/management/ConfigurationFunctions.java matches
=====================
import com.jakewharton.fliptables.FlipTable;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.slf4j.LoggerFactory;
./metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java matches
=====================
import com.jakewharton.fliptables.FlipTable;
import oi.thekraken.grok.api.Grok;
import oi.thekraken.grok.api.Match;
import oi.thekraken.grok.api.exception.GrokException;
import org.slf4j.LoggerFactory;
./metron-platform/metron-management/src/main/java/org/apache/metron/management/GrokFunctions.java matches
=====================
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.curator.framework.CuratorFramework;
./metron-platform/metron-management/src/main/java/org/apache/metron/management/ParserFunctions.java matches
=====================
import com.fasterxml.jackson.core.JsonProcessingException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
./metron-platform/metron-management/src/main/java/org/apache/metron/management/IndexingConfigFunctions.java matches
=====================
import com.fasterxml.jackson.core.JsonProcessingException;
import com.jakewharton.fliptables.FlipTable;
import org.apache.commons.collections4.ListUtils;
import org.apache.commons.lang3.ClassUtils;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
./metron-platform/metron-management/src/main/java/org/apache/metron/management/ThreatTriageFunctions.java matches
=====================
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ClassUtils;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.RecordMetadata;
import org.apache.kafka.common.PartitionInfo;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.apache.kafka.common.serialization.StringSerializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
./metron-platform/metron-management/src/main/java/org/apache/metron/management/KafkaFunctions.java matches
=====================
import com.fasterxml.jackson.core.JsonProcessingException;
import com.jakewharton.fliptables.FlipTable;
import org.slf4j.LoggerFactory;
./metron-platform/metron-management/src/main/java/org/apache/metron/management/ParserConfigFunctions.java matches
=====================
import com.fasterxml.jackson.core.JsonProcessingException;
import com.jakewharton.fliptables.FlipTable;
import org.slf4j.LoggerFactory;
./metron-platform/metron-management/src/main/java/org/apache/metron/management/EnrichmentConfigFunctions.java matches
=====================
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
./metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/WindowLookback.java matches
=====================
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
./metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/GetProfile.java matches
=====================
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
./metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/VerboseProfile.java matches
=====================
import org.apache.commons.lang3.ClassUtils;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.slf4j.LoggerFactory;
./metron-analytics/metron-profiler-repl/src/main/java/org/apache/metron/profiler/repl/ProfilerFunctions.java matches
=====================
import com.google.common.collect.ImmutableList;
./metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/StellarStatisticsFunctions.java matches
=====================
import com.codahale.metrics.Reservoir;
./metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/sampling/SamplingOpsFunctions.java matches
=====================
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import org.apache.curator.framework.CuratorFramework;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
./metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java matches
=====================
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.junit.Assert.assertThat;
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import org.apache.commons.lang3.StringUtils;
import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
./metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java matches
=====================
import static org.hamcrest.CoreMatchers.equalTo;
import static org.junit.Assert.assertThat;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
./metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/resolver/BaseFunctionResolverTest.java matches
=====================
import com.google.common.collect.Lists;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
./metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/resolver/SimpleFunctionResolverTest.java matches
=====================
import com.google.common.collect.Iterables;
./metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DataStructureFunctions.java matches
=====================
import com.google.common.collect.Lists;
./metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/FunctionalFunctions.java matches
=====================
import org.apache.commons.lang.StringUtils;
./metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/EncodingFunctions.java matches
=====================
import org.apache.commons.lang3.StringUtils;
./metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RegExFunctions.java matches
=====================
import com.github.benmanes.caffeine.cache.CacheLoader;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.LoadingCache;
./metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DateFunctions.java matches
=====================
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
./metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RestFunctions.java matches
=====================
import com.google.common.collect.Iterables;
./metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/SetFunctions.java matches
=====================
import com.google.common.collect.Iterables;
./metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/MapFunctions.java matches
=====================
import org.apache.commons.codec.EncoderException;
./metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/HashFunctions.java matches
=====================
import com.jakewharton.fliptables.FlipTable;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.text.WordUtils;
import org.jboss.aesh.console.Console;
import org.slf4j.LoggerFactory;
./metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/ShellFunctions.java matches
=====================
import com.google.common.collect.Iterables;
./metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctions.java matches
=====================
import com.google.common.collect.ImmutableList;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.text.similarity.FuzzyScore;
./metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/TextFunctions.java matches
=====================
import com.fasterxml.jackson.core.JsonProcessingException;
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.collect.Iterables;
import org.apache.commons.lang3.StringUtils;
./metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/StringFunctions.java matches
=====================
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.collect.Iterables;
import com.google.common.net.InternetDomainName;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.net.util.SubnetUtils;
./metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/NetworkFunctions.java matches
=====================
```
Just the files without the import context (there are currently 43 that I've found)
```
./metron-platform/metron-common/src/main/java/org/apache/metron/common/field/validation/primitive/IntegerValidation.java
./metron-platform/metron-common/src/main/java/org/apache/metron/common/field/validation/network/EmailValidation.java
./metron-platform/metron-common/src/main/java/org/apache/metron/common/field/validation/network/DomainValidation.java
./metron-platform/metron-common/src/main/java/org/apache/metron/common/field/validation/network/IPValidation.java
./metron-platform/metron-common/src/main/java/org/apache/metron/common/field/validation/network/URLValidation.java
./metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/GeoEnrichmentFunctions.java
./metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/AsnEnrichmentFunctions.java
./metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/ObjectGet.java
./metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/GeoHashFunctions.java
./metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/SimpleHBaseEnrichmentFunctions.java
./metron-platform/metron-management/src/main/java/org/apache/metron/management/ConfigurationFunctions.java
./metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java
./metron-platform/metron-management/src/main/java/org/apache/metron/management/GrokFunctions.java
./metron-platform/metron-management/src/main/java/org/apache/metron/management/ParserFunctions.java
./metron-platform/metron-management/src/main/java/org/apache/metron/management/IndexingConfigFunctions.java
./metron-platform/metron-management/src/main/java/org/apache/metron/management/ThreatTriageFunctions.java
./metron-platform/metron-management/src/main/java/org/apache/metron/management/KafkaFunctions.java
./metron-platform/metron-management/src/main/java/org/apache/metron/management/ParserConfigFunctions.java
./metron-platform/metron-management/src/main/java/org/apache/metron/management/EnrichmentConfigFunctions.java
./metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/WindowLookback.java
./metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/GetProfile.java
./metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/VerboseProfile.java
./metron-analytics/metron-profiler-repl/src/main/java/org/apache/metron/profiler/repl/ProfilerFunctions.java
./metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/StellarStatisticsFunctions.java
./metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/sampling/SamplingOpsFunctions.java
./metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java
./metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
./metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/resolver/BaseFunctionResolverTest.java
./metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/resolver/SimpleFunctionResolverTest.java
./metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DataStructureFunctions.java
./metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/FunctionalFunctions.java
./metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/EncodingFunctions.java
./metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RegExFunctions.java
./metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DateFunctions.java
./metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RestFunctions.java
./metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/SetFunctions.java
./metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/MapFunctions.java
./metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/HashFunctions.java
./metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/ShellFunctions.java
./metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctions.java
./metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/TextFunctions.java
./metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/StringFunctions.java
./metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/NetworkFunctions.java
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services