You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by le...@apache.org on 2017/12/19 14:13:10 UTC
[09/17] incubator-sdap-mudrod git commit: SDAP-7 Change all package
namespaces to org.apache.sdap
http://git-wip-us.apache.org/repos/asf/incubator-sdap-mudrod/blob/39379fa9/core/src/main/java/org/apache/sdap/mudrod/main/MudrodEngine.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sdap/mudrod/main/MudrodEngine.java b/core/src/main/java/org/apache/sdap/mudrod/main/MudrodEngine.java
new file mode 100644
index 0000000..341d5fc
--- /dev/null
+++ b/core/src/main/java/org/apache/sdap/mudrod/main/MudrodEngine.java
@@ -0,0 +1,456 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sdap.mudrod.main;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.sdap.mudrod.discoveryengine.DiscoveryEngineAbstract;
+import org.apache.sdap.mudrod.discoveryengine.MetadataDiscoveryEngine;
+import org.apache.sdap.mudrod.discoveryengine.OntologyDiscoveryEngine;
+import org.apache.sdap.mudrod.discoveryengine.RecommendEngine;
+import org.apache.sdap.mudrod.discoveryengine.WeblogDiscoveryEngine;
+import org.apache.sdap.mudrod.driver.ESDriver;
+import org.apache.sdap.mudrod.driver.SparkDriver;
+import org.apache.sdap.mudrod.integration.LinkageIntegration;
+import org.jdom2.Document;
+import org.jdom2.Element;
+import org.jdom2.JDOMException;
+import org.jdom2.input.SAXBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.sdap.mudrod.main.MudrodConstants.DATA_DIR;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.nio.file.Files;
+import java.util.List;
+import java.util.Properties;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+/**
+ * Main entry point for Running the Mudrod system. Invocation of this class is
+ * tightly linked to the primary Mudrod configuration which can be located at
+ * <a href=
+ * "https://github.com/mudrod/mudrod/blob/master/core/src/main/resources/config.xml">config.xml</a>.
+ */
+public class MudrodEngine {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MudrodEngine.class);
+ private Properties props = new Properties();
+ private ESDriver es = null;
+ private SparkDriver spark = null;
+ private static final String LOG_INGEST = "logIngest";
+ private static final String META_INGEST = "metaIngest";
+ private static final String FULL_INGEST = "fullIngest";
+ private static final String PROCESSING = "processingWithPreResults";
+ private static final String ES_HOST = "esHost";
+ private static final String ES_TCP_PORT = "esTCPPort";
+ private static final String ES_HTTP_PORT = "esPort";
+
+ /**
+ * Public constructor for this class.
+ */
+ public MudrodEngine() {
+ // default constructor
+ }
+
+ /**
+ * Start the {@link ESDriver}. Should only be called after call to
+ * {@link MudrodEngine#loadConfig()}
+ *
+ * @return fully provisioned {@link ESDriver}
+ */
+ public ESDriver startESDriver() {
+ return new ESDriver(props);
+ }
+
+ /**
+ * Start the {@link SparkDriver}. Should only be called after call to
+ * {@link MudrodEngine#loadConfig()}
+ *
+ * @return fully provisioned {@link SparkDriver}
+ */
+ public SparkDriver startSparkDriver() {
+ return new SparkDriver(props);
+ }
+
+ /**
+ * Retreive the Mudrod configuration as a Properties Map containing K, V of
+ * type String.
+ *
+ * @return a {@link java.util.Properties} object
+ */
+ public Properties getConfig() {
+ return props;
+ }
+
+ /**
+ * Retreive the Mudrod {@link ESDriver}
+ *
+ * @return the {@link ESDriver} instance.
+ */
+ public ESDriver getESDriver() {
+ return this.es;
+ }
+
+ /**
+ * Set the Elasticsearch driver for MUDROD
+ *
+ * @param es
+ * an ES driver instance
+ */
+ public void setESDriver(ESDriver es) {
+ this.es = es;
+ }
+
+ private InputStream locateConfig() {
+
+ String configLocation = System.getenv(MudrodConstants.MUDROD_CONFIG) == null ? "" : System.getenv(MudrodConstants.MUDROD_CONFIG);
+ File configFile = new File(configLocation);
+
+ try {
+ InputStream configStream = new FileInputStream(configFile);
+ LOG.info("Loaded config file from " + configFile.getAbsolutePath());
+ return configStream;
+ } catch (IOException e) {
+ LOG.info("File specified by environment variable " + MudrodConstants.MUDROD_CONFIG + "=\'" + configLocation + "\' could not be loaded. " + e.getMessage());
+ }
+
+ InputStream configStream = MudrodEngine.class.getClassLoader().getResourceAsStream("config.xml");
+
+ if (configStream != null) {
+ LOG.info("Loaded config file from {}", MudrodEngine.class.getClassLoader().getResource("config.xml").getPath());
+ }
+
+ return configStream;
+ }
+
+ /**
+ * Load the configuration provided at <a href=
+ * "https://github.com/mudrod/mudrod/blob/master/core/src/main/resources/config.xml">config.xml</a>.
+ *
+ * @return a populated {@link java.util.Properties} object.
+ */
+ public Properties loadConfig() {
+ SAXBuilder saxBuilder = new SAXBuilder();
+
+ InputStream configStream = locateConfig();
+
+ Document document;
+ try {
+ document = saxBuilder.build(configStream);
+ Element rootNode = document.getRootElement();
+ List<Element> paraList = rootNode.getChildren("para");
+
+ for (int i = 0; i < paraList.size(); i++) {
+ Element paraNode = paraList.get(i);
+ String attributeName = paraNode.getAttributeValue("name");
+ if (MudrodConstants.SVM_SGD_MODEL.equals(attributeName)) {
+ props.put(attributeName, decompressSVMWithSGDModel(paraNode.getTextTrim()));
+ } else {
+ props.put(attributeName, paraNode.getTextTrim());
+ }
+ }
+ } catch (JDOMException | IOException e) {
+ LOG.error("Exception whilst retrieving or processing XML contained within 'config.xml'!", e);
+ }
+ return getConfig();
+
+ }
+
+ private String decompressSVMWithSGDModel(String archiveName) throws IOException {
+
+ URL scmArchive = getClass().getClassLoader().getResource(archiveName);
+ if (scmArchive == null) {
+ throw new IOException("Unable to locate " + archiveName + " as a classpath resource.");
+ }
+ File tempDir = Files.createTempDirectory("mudrod").toFile();
+ assert tempDir.setWritable(true);
+ File archiveFile = new File(tempDir, archiveName);
+ FileUtils.copyURLToFile(scmArchive, archiveFile);
+
+ // Decompress archive
+ int BUFFER_SIZE = 512000;
+ ZipInputStream zipIn = new ZipInputStream(new FileInputStream(archiveFile));
+ ZipEntry entry;
+ while ((entry = zipIn.getNextEntry()) != null) {
+ File f = new File(tempDir, entry.getName());
+ // If the entry is a directory, create the directory.
+ if (entry.isDirectory() && !f.exists()) {
+ boolean created = f.mkdirs();
+ if (!created) {
+ LOG.error("Unable to create directory '{}', during extraction of archive contents.", f.getAbsolutePath());
+ }
+ } else if (!entry.isDirectory()) {
+ boolean created = f.getParentFile().mkdirs();
+ if (!created && !f.getParentFile().exists()) {
+ LOG.error("Unable to create directory '{}', during extraction of archive contents.", f.getParentFile().getAbsolutePath());
+ }
+ int count;
+ byte data[] = new byte[BUFFER_SIZE];
+ FileOutputStream fos = new FileOutputStream(new File(tempDir, entry.getName()), false);
+ try (BufferedOutputStream dest = new BufferedOutputStream(fos, BUFFER_SIZE)) {
+ while ((count = zipIn.read(data, 0, BUFFER_SIZE)) != -1) {
+ dest.write(data, 0, count);
+ }
+ }
+ }
+ }
+
+ return new File(tempDir, StringUtils.removeEnd(archiveName, ".zip")).toURI().toString();
+ }
+
+ /**
+ * Preprocess and process logs {@link DiscoveryEngineAbstract} implementations
+ * for weblog
+ */
+ public void startLogIngest() {
+ DiscoveryEngineAbstract wd = new WeblogDiscoveryEngine(props, es, spark);
+ wd.preprocess();
+ wd.process();
+ LOG.info("*****************logs have been ingested successfully******************");
+ }
+
+ /**
+ * updating and analysing metadata to metadata similarity results
+ */
+ public void startMetaIngest() {
+ DiscoveryEngineAbstract md = new MetadataDiscoveryEngine(props, es, spark);
+ md.preprocess();
+ md.process();
+
+ DiscoveryEngineAbstract recom = new RecommendEngine(props, es, spark);
+ recom.preprocess();
+ recom.process();
+ LOG.info("Metadata has been ingested successfully.");
+ }
+
+ public void startFullIngest() {
+ DiscoveryEngineAbstract wd = new WeblogDiscoveryEngine(props, es, spark);
+ wd.preprocess();
+ wd.process();
+
+ DiscoveryEngineAbstract md = new MetadataDiscoveryEngine(props, es, spark);
+ md.preprocess();
+ md.process();
+
+ DiscoveryEngineAbstract recom = new RecommendEngine(props, es, spark);
+ recom.preprocess();
+ recom.process();
+ LOG.info("Full ingest has finished successfully.");
+ }
+
+ /**
+ * Only preprocess various {@link DiscoveryEngineAbstract} implementations for
+ * weblog, ontology and metadata, linkage discovery and integration.
+ */
+ public void startProcessing() {
+ DiscoveryEngineAbstract wd = new WeblogDiscoveryEngine(props, es, spark);
+ wd.process();
+
+ DiscoveryEngineAbstract od = new OntologyDiscoveryEngine(props, es, spark);
+ od.preprocess();
+ od.process();
+
+ DiscoveryEngineAbstract md = new MetadataDiscoveryEngine(props, es, spark);
+ md.preprocess();
+ md.process();
+
+ LinkageIntegration li = new LinkageIntegration(props, es, spark);
+ li.execute();
+
+ DiscoveryEngineAbstract recom = new RecommendEngine(props, es, spark);
+ recom.process();
+ }
+
+ /**
+ * Close the connection to the {@link ESDriver} instance.
+ */
+ public void end() {
+ if (es != null) {
+ es.close();
+ }
+ }
+
+ /**
+ * Main program invocation. Accepts one argument denoting location (on disk)
+ * to a log file which is to be ingested. Help will be provided if invoked
+ * with incorrect parameters.
+ *
+ * @param args
+ * {@link java.lang.String} array contaning correct parameters.
+ */
+ public static void main(String[] args) {
+ // boolean options
+ Option helpOpt = new Option("h", "help", false, "show this help message");
+
+ // log ingest (preprocessing + processing)
+ Option logIngestOpt = new Option("l", LOG_INGEST, false, "begin log ingest");
+ // metadata ingest (preprocessing + processing)
+ Option metaIngestOpt = new Option("m", META_INGEST, false, "begin metadata ingest");
+ // ingest both log and metadata
+ Option fullIngestOpt = new Option("f", FULL_INGEST, false, "begin full ingest Mudrod workflow");
+ // processing only, assuming that preprocessing results is in dataDir
+ Option processingOpt = new Option("p", PROCESSING, false, "begin processing with preprocessing results");
+
+ // argument options
+ Option dataDirOpt = OptionBuilder.hasArg(true).withArgName("/path/to/data/directory").hasArgs(1).withDescription("the data directory to be processed by Mudrod").withLongOpt("dataDirectory")
+ .isRequired().create(DATA_DIR);
+
+ Option esHostOpt = OptionBuilder.hasArg(true).withArgName("host_name").hasArgs(1).withDescription("elasticsearch cluster unicast host").withLongOpt("elasticSearchHost").isRequired(false)
+ .create(ES_HOST);
+
+ Option esTCPPortOpt = OptionBuilder.hasArg(true).withArgName("port_num").hasArgs(1).withDescription("elasticsearch transport TCP port").withLongOpt("elasticSearchTransportTCPPort")
+ .isRequired(false).create(ES_TCP_PORT);
+
+ Option esPortOpt = OptionBuilder.hasArg(true).withArgName("port_num").hasArgs(1).withDescription("elasticsearch HTTP/REST port").withLongOpt("elasticSearchHTTPPort").isRequired(false)
+ .create(ES_HTTP_PORT);
+
+ // create the options
+ Options options = new Options();
+ options.addOption(helpOpt);
+ options.addOption(logIngestOpt);
+ options.addOption(metaIngestOpt);
+ options.addOption(fullIngestOpt);
+ options.addOption(processingOpt);
+ options.addOption(dataDirOpt);
+ options.addOption(esHostOpt);
+ options.addOption(esTCPPortOpt);
+ options.addOption(esPortOpt);
+
+ CommandLineParser parser = new GnuParser();
+ try {
+ CommandLine line = parser.parse(options, args);
+ String processingType = null;
+
+ if (line.hasOption(LOG_INGEST)) {
+ processingType = LOG_INGEST;
+ } else if (line.hasOption(PROCESSING)) {
+ processingType = PROCESSING;
+ } else if (line.hasOption(META_INGEST)) {
+ processingType = META_INGEST;
+ } else if (line.hasOption(FULL_INGEST)) {
+ processingType = FULL_INGEST;
+ }
+
+ String dataDir = line.getOptionValue(DATA_DIR).replace("\\", "/");
+ if (!dataDir.endsWith("/")) {
+ dataDir += "/";
+ }
+
+ MudrodEngine me = new MudrodEngine();
+ me.loadConfig();
+ me.props.put(DATA_DIR, dataDir);
+
+ if (line.hasOption(ES_HOST)) {
+ String esHost = line.getOptionValue(ES_HOST);
+ me.props.put(MudrodConstants.ES_UNICAST_HOSTS, esHost);
+ }
+
+ if (line.hasOption(ES_TCP_PORT)) {
+ String esTcpPort = line.getOptionValue(ES_TCP_PORT);
+ me.props.put(MudrodConstants.ES_TRANSPORT_TCP_PORT, esTcpPort);
+ }
+
+ if (line.hasOption(ES_HTTP_PORT)) {
+ String esHttpPort = line.getOptionValue(ES_HTTP_PORT);
+ me.props.put(MudrodConstants.ES_HTTP_PORT, esHttpPort);
+ }
+
+ me.es = new ESDriver(me.getConfig());
+ me.spark = new SparkDriver(me.getConfig());
+ loadFullConfig(me, dataDir);
+ if (processingType != null) {
+ switch (processingType) {
+ case PROCESSING:
+ me.startProcessing();
+ break;
+ case LOG_INGEST:
+ me.startLogIngest();
+ break;
+ case META_INGEST:
+ me.startMetaIngest();
+ break;
+ case FULL_INGEST:
+ me.startFullIngest();
+ break;
+ default:
+ break;
+ }
+ }
+ me.end();
+ } catch (Exception e) {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp("MudrodEngine: 'dataDir' argument is mandatory. " + "User must also provide an ingest method.", options, true);
+ LOG.error("Error whilst parsing command line.", e);
+ }
+ }
+
+ private static void loadFullConfig(MudrodEngine me, String dataDir) {
+ //TODO all of the properties defined below, which are determined are
+ //runtime need to be added to MudrodConstants.java and referenced
+ //accordingly and consistently from Properties.getProperty(MudrodConstant...);
+ me.props.put("ontologyInputDir", dataDir + "SWEET_ocean/");
+ me.props.put("oceanTriples", dataDir + "Ocean_triples.csv");
+ me.props.put("userHistoryMatrix", dataDir + "UserHistoryMatrix.csv");
+ me.props.put("clickstreamMatrix", dataDir + "ClickstreamMatrix.csv");
+ me.props.put("metadataMatrix", dataDir + "MetadataMatrix.csv");
+ me.props.put("clickstreamSVDMatrix_tmp", dataDir + "clickstreamSVDMatrix_tmp.csv");
+ me.props.put("metadataSVDMatrix_tmp", dataDir + "metadataSVDMatrix_tmp.csv");
+ me.props.put("raw_metadataPath", dataDir + me.props.getProperty(MudrodConstants.RAW_METADATA_TYPE));
+
+ me.props.put("jtopia", dataDir + "jtopiaModel");
+ me.props.put("metadata_term_tfidf_matrix", dataDir + "metadata_term_tfidf.csv");
+ me.props.put("metadata_word_tfidf_matrix", dataDir + "metadata_word_tfidf.csv");
+ me.props.put("session_metadata_Matrix", dataDir + "metadata_session_coocurrence_matrix.csv");
+
+ me.props.put("metadataOBCode", dataDir + "MetadataOHCode");
+ me.props.put("metadata_topic", dataDir + "metadata_topic");
+ me.props.put("metadata_topic_matrix", dataDir + "metadata_topic_matrix.csv");
+ }
+
+ /**
+ * Obtain the spark implementation.
+ *
+ * @return the {@link SparkDriver}
+ */
+ public SparkDriver getSparkDriver() {
+ return this.spark;
+ }
+
+ /**
+ * Set the {@link SparkDriver}
+ *
+ * @param sparkDriver
+ * a configured {@link SparkDriver}
+ */
+ public void setSparkDriver(SparkDriver sparkDriver) {
+ this.spark = sparkDriver;
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-mudrod/blob/39379fa9/core/src/main/java/org/apache/sdap/mudrod/main/package-info.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sdap/mudrod/main/package-info.java b/core/src/main/java/org/apache/sdap/mudrod/main/package-info.java
new file mode 100644
index 0000000..8a3af67
--- /dev/null
+++ b/core/src/main/java/org/apache/sdap/mudrod/main/package-info.java
@@ -0,0 +1,17 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * This package includes Main entry point for Running the Mudrod system.
+ */
+package org.apache.sdap.mudrod.main;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-sdap-mudrod/blob/39379fa9/core/src/main/java/org/apache/sdap/mudrod/metadata/package-info.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sdap/mudrod/metadata/package-info.java b/core/src/main/java/org/apache/sdap/mudrod/metadata/package-info.java
new file mode 100644
index 0000000..b7b6258
--- /dev/null
+++ b/core/src/main/java/org/apache/sdap/mudrod/metadata/package-info.java
@@ -0,0 +1,18 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * This package includes metadata pre-processing, processing, and data structure
+ * classes.
+ */
+package org.apache.sdap.mudrod.metadata;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-sdap-mudrod/blob/39379fa9/core/src/main/java/org/apache/sdap/mudrod/metadata/pre/ApiHarvester.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sdap/mudrod/metadata/pre/ApiHarvester.java b/core/src/main/java/org/apache/sdap/mudrod/metadata/pre/ApiHarvester.java
new file mode 100644
index 0000000..7b8b5c1
--- /dev/null
+++ b/core/src/main/java/org/apache/sdap/mudrod/metadata/pre/ApiHarvester.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sdap.mudrod.metadata.pre;
+
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.sdap.mudrod.discoveryengine.DiscoveryStepAbstract;
+import org.apache.sdap.mudrod.driver.ESDriver;
+import org.apache.sdap.mudrod.driver.SparkDriver;
+import org.apache.sdap.mudrod.main.MudrodConstants;
+import org.apache.sdap.mudrod.utils.HttpRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
+import java.util.Properties;
+
+/**
+ * ClassName: ApiHarvester Function: Harvest metadata from PO.DAACweb service.
+ */
+public class ApiHarvester extends DiscoveryStepAbstract {
+
+ private static final long serialVersionUID = 1L;
+ private static final Logger LOG = LoggerFactory.getLogger(ApiHarvester.class);
+
+ /**
+ * Creates a new instance of ApiHarvester.
+ *
+ * @param props the Mudrod configuration
+ * @param es the Elasticsearch drive
+ * @param spark the spark driver
+ */
+ public ApiHarvester(Properties props, ESDriver es, SparkDriver spark) {
+ super(props, es, spark);
+ }
+
+ @Override
+ public Object execute() {
+ LOG.info("Starting Metadata harvesting.");
+ startTime = System.currentTimeMillis();
+ //remove old metadata from ES
+ es.deleteType(props.getProperty(MudrodConstants.ES_INDEX_NAME), props.getProperty(MudrodConstants.RAW_METADATA_TYPE));
+ //harvest new metadata using PO.DAAC web services
+ harvestMetadatafromWeb();
+ es.createBulkProcessor();
+ addMetadataMapping();
+ importToES();
+ es.destroyBulkProcessor();
+ endTime = System.currentTimeMillis();
+ es.refreshIndex();
+ LOG.info("Metadata harvesting completed. Time elapsed: {}", (endTime - startTime) / 1000);
+ return null;
+ }
+
+ /**
+ * addMetadataMapping: Add mapping to index metadata in Elasticsearch. Please
+ * invoke this method before import metadata to Elasticsearch.
+ */
+ public void addMetadataMapping() {
+ String mappingJson = "{\r\n \"dynamic_templates\": " + "[\r\n " + "{\r\n \"strings\": " + "{\r\n \"match_mapping_type\": \"string\","
+ + "\r\n \"mapping\": {\r\n \"type\": \"text\"," + "\r\n \"fielddata\": true," + "\r\n \"analyzer\": \"english\","
+ + "\r\n \"fields\": {\r\n \"raw\": {" + "\r\n \"type\": \"string\"," + "\r\n \"index\": \"not_analyzed\"" + "\r\n }"
+ + "\r\n }\r\n " + "\r\n }" + "\r\n }\r\n }\r\n ]\r\n}";
+
+ es.getClient().admin().indices().preparePutMapping(props.getProperty(MudrodConstants.ES_INDEX_NAME)).setType(props.getProperty(MudrodConstants.RAW_METADATA_TYPE)).setSource(mappingJson).execute()
+ .actionGet();
+ }
+
+ /**
+ * importToES: Index metadata into elasticsearch from local file directory.
+ * Please make sure metadata have been harvest from web service before
+ * invoking this method.
+ */
+ private void importToES() {
+ File directory = new File(props.getProperty(MudrodConstants.RAW_METADATA_PATH));
+ if(!directory.exists())
+ directory.mkdir();
+ File[] fList = directory.listFiles();
+ for (File file : fList) {
+ InputStream is;
+ try {
+ is = new FileInputStream(file);
+ importSingleFileToES(is);
+ } catch (FileNotFoundException e) {
+ LOG.error("Error finding file!", e);
+ }
+
+ }
+ }
+
+ private void importSingleFileToES(InputStream is) {
+ try {
+ String jsonTxt = IOUtils.toString(is);
+ JsonParser parser = new JsonParser();
+ JsonElement item = parser.parse(jsonTxt);
+ IndexRequest ir = new IndexRequest(props.getProperty(MudrodConstants.ES_INDEX_NAME), props.getProperty(MudrodConstants.RAW_METADATA_TYPE)).source(item.toString());
+ es.getBulkProcessor().add(ir);
+ } catch (IOException e) {
+ LOG.error("Error indexing metadata record!", e);
+ }
+ }
+
+ /**
+ * harvestMetadatafromWeb: Harvest metadata from PO.DAAC web service.
+ */
+ private void harvestMetadatafromWeb() {
+ LOG.info("Metadata download started.");
+ int startIndex = 0;
+ int doc_length = 0;
+ JsonParser parser = new JsonParser();
+ do {
+ String searchAPI = "https://podaac.jpl.nasa.gov/api/dataset?startIndex=" + Integer.toString(startIndex) + "&entries=10&sortField=Dataset-AllTimePopularity&sortOrder=asc&id=&value=&search=";
+ HttpRequest http = new HttpRequest();
+ String response = http.getRequest(searchAPI);
+
+ JsonElement json = parser.parse(response);
+ JsonObject responseObject = json.getAsJsonObject();
+ JsonArray docs = responseObject.getAsJsonObject("response").getAsJsonArray("docs");
+
+ doc_length = docs.size();
+
+ File file = new File(props.getProperty(MudrodConstants.RAW_METADATA_PATH));
+ if (!file.exists()) {
+ if (file.mkdir()) {
+ LOG.info("Directory is created!");
+ } else {
+ LOG.error("Failed to create directory!");
+ }
+ }
+ for (int i = 0; i < doc_length; i++) {
+ JsonElement item = docs.get(i);
+ int docId = startIndex + i;
+ File itemfile = new File(props.getProperty(MudrodConstants.RAW_METADATA_PATH) + "/" + docId + ".json");
+
+ try (FileWriter fw = new FileWriter(itemfile.getAbsoluteFile()); BufferedWriter bw = new BufferedWriter(fw);) {
+ itemfile.createNewFile();
+ bw.write(item.toString());
+ } catch (IOException e) {
+ LOG.error("Error writing metadata to local file!", e);
+ }
+ }
+
+ startIndex += 10;
+
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ LOG.error("Error entering Elasticsearch Mappings!", e);
+ Thread.currentThread().interrupt();
+ }
+
+ } while (doc_length != 0);
+
+ LOG.info("Metadata downloading finished");
+ }
+
+ @Override
+ public Object execute(Object o) {
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-mudrod/blob/39379fa9/core/src/main/java/org/apache/sdap/mudrod/metadata/pre/MatrixGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sdap/mudrod/metadata/pre/MatrixGenerator.java b/core/src/main/java/org/apache/sdap/mudrod/metadata/pre/MatrixGenerator.java
new file mode 100644
index 0000000..63565b2
--- /dev/null
+++ b/core/src/main/java/org/apache/sdap/mudrod/metadata/pre/MatrixGenerator.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sdap.mudrod.metadata.pre;
+
+import org.apache.sdap.mudrod.discoveryengine.DiscoveryStepAbstract;
+import org.apache.sdap.mudrod.driver.ESDriver;
+import org.apache.sdap.mudrod.driver.SparkDriver;
+import org.apache.sdap.mudrod.main.MudrodConstants;
+import org.apache.sdap.mudrod.metadata.structure.MetadataExtractor;
+import org.apache.sdap.mudrod.utils.LabeledRowMatrix;
+import org.apache.sdap.mudrod.utils.MatrixUtil;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.Properties;
+
+/**
+ * Generate term-metadata matrix from original metadata. Each row in
+ * the matrix is corresponding to a term, and each column is a metadata.
+ */
+public class MatrixGenerator extends DiscoveryStepAbstract {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+ private static final Logger LOG = LoggerFactory.getLogger(MatrixGenerator.class);
+
+ /**
+ * Creates a new instance of MatrixGenerator.
+ *
+ * @param props the Mudrod configuration
+ * @param es the Elasticsearch drive
+ * @param spark the spark drive
+ */
+ public MatrixGenerator(Properties props, ESDriver es, SparkDriver spark) {
+ super(props, es, spark);
+ }
+
+ /**
+ * Generate a csv which is a term-metadata matrix genetrated from original
+ * metadata.
+ *
+ * @see DiscoveryStepAbstract#execute()
+ */
+ @Override
+ public Object execute() {
+ LOG.info("Metadata matrix started");
+ startTime = System.currentTimeMillis();
+
+ String metadataMatrixFile = props.getProperty("metadataMatrix");
+ try {
+ MetadataExtractor extractor = new MetadataExtractor();
+ JavaPairRDD<String, List<String>> metadataTermsRDD = extractor.loadMetadata(this.es, this.spark.sc, props.getProperty(MudrodConstants.ES_INDEX_NAME), props.getProperty(MudrodConstants.RAW_METADATA_TYPE));
+ LabeledRowMatrix wordDocMatrix = MatrixUtil.createWordDocMatrix(metadataTermsRDD);
+ MatrixUtil.exportToCSV(wordDocMatrix.rowMatrix, wordDocMatrix.rowkeys, wordDocMatrix.colkeys, metadataMatrixFile);
+
+ } catch (Exception e) {
+ LOG.error("Error during Metadata matrix generaion: {}", e);
+ }
+
+ endTime = System.currentTimeMillis();
+ LOG.info("Metadata matrix finished time elapsed: {}s", (endTime - startTime) / 1000);
+ return null;
+ }
+
+ @Override
+ public Object execute(Object o) {
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-mudrod/blob/39379fa9/core/src/main/java/org/apache/sdap/mudrod/metadata/pre/package-info.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sdap/mudrod/metadata/pre/package-info.java b/core/src/main/java/org/apache/sdap/mudrod/metadata/pre/package-info.java
new file mode 100644
index 0000000..ffecbc8
--- /dev/null
+++ b/core/src/main/java/org/apache/sdap/mudrod/metadata/pre/package-info.java
@@ -0,0 +1,17 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * This package includes metadata pre-processing functions.
+ */
+package org.apache.sdap.mudrod.metadata.pre;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-sdap-mudrod/blob/39379fa9/core/src/main/java/org/apache/sdap/mudrod/metadata/process/MetadataAnalyzer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sdap/mudrod/metadata/process/MetadataAnalyzer.java b/core/src/main/java/org/apache/sdap/mudrod/metadata/process/MetadataAnalyzer.java
new file mode 100644
index 0000000..80e23c1
--- /dev/null
+++ b/core/src/main/java/org/apache/sdap/mudrod/metadata/process/MetadataAnalyzer.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sdap.mudrod.metadata.process;
+
+import org.apache.sdap.mudrod.discoveryengine.DiscoveryStepAbstract;
+import org.apache.sdap.mudrod.driver.ESDriver;
+import org.apache.sdap.mudrod.driver.SparkDriver;
+import org.apache.sdap.mudrod.semantics.SVDAnalyzer;
+import org.apache.sdap.mudrod.utils.LinkageTriple;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Properties;
+
+/**
+ * ClassName: MetadataAnalyzer
+ * Function: Calculate semantic relationship of vocabularies extracted from
+ * metadata.
+ */
+public class MetadataAnalyzer extends DiscoveryStepAbstract implements Serializable {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+ private static final Logger LOG = LoggerFactory.getLogger(MetadataAnalyzer.class);
+
+ /**
+ * Creates a new instance of MetadataAnalyzer.
+ *
+ * @param props the Mudrod configuration
+ * @param es the Elasticsearch drive
+ * @param spark the spark drive
+ */
+ public MetadataAnalyzer(Properties props, ESDriver es, SparkDriver spark) {
+ super(props, es, spark);
+ }
+
+ @Override
+ public Object execute(Object o) {
+ return null;
+ }
+
+ /**
+ * Calculate semantic relationship of vocabularies from a csv file which is a
+ * term-metadata matrix.
+ *
+ * @see DiscoveryStepAbstract#execute()
+ */
+ @Override
+ public Object execute() {
+ try {
+ LOG.info("*****************Metadata Analyzer starts******************");
+ startTime = System.currentTimeMillis();
+
+ SVDAnalyzer analyzer = new SVDAnalyzer(props, es, spark);
+ int svdDimension = Integer.parseInt(props.getProperty("metadataSVDDimension"));
+ String metadataMatrixFile = props.getProperty("metadataMatrix");
+ String svdMatrixFileName = props.getProperty("metadataSVDMatrix_tmp");
+
+ analyzer.getSVDMatrix(metadataMatrixFile, svdDimension, svdMatrixFileName);
+ List<LinkageTriple> triples = analyzer.calTermSimfromMatrix(svdMatrixFileName);
+
+ analyzer.saveToES(triples, props.getProperty("indexName"), props.getProperty("metadataLinkageType"));
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ endTime = System.currentTimeMillis();
+ es.refreshIndex();
+ LOG.info("*****************Metadata Analyzer ends******************Took {}s", (endTime - startTime) / 1000);
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-mudrod/blob/39379fa9/core/src/main/java/org/apache/sdap/mudrod/metadata/process/package-info.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sdap/mudrod/metadata/process/package-info.java b/core/src/main/java/org/apache/sdap/mudrod/metadata/process/package-info.java
new file mode 100644
index 0000000..a0c0799
--- /dev/null
+++ b/core/src/main/java/org/apache/sdap/mudrod/metadata/process/package-info.java
@@ -0,0 +1,17 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * This package includes metadata processing classes.
+ */
+package org.apache.sdap.mudrod.metadata.process;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-sdap-mudrod/blob/39379fa9/core/src/main/java/org/apache/sdap/mudrod/metadata/structure/MetadataExtractor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sdap/mudrod/metadata/structure/MetadataExtractor.java b/core/src/main/java/org/apache/sdap/mudrod/metadata/structure/MetadataExtractor.java
new file mode 100644
index 0000000..379d5b9
--- /dev/null
+++ b/core/src/main/java/org/apache/sdap/mudrod/metadata/structure/MetadataExtractor.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sdap.mudrod.metadata.structure;
+
+import org.apache.sdap.mudrod.driver.ESDriver;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.JavaRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.api.java.function.Function2;
+import org.apache.spark.api.java.function.PairFunction;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.SearchHit;
+import scala.Tuple2;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+
+public class MetadataExtractor implements Serializable {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+ public MetadataExtractor() {
+ }
+
+ /**
+ * loadMetadata:Load all metadata from Elasticsearch and convert them to
+ * pairRDD Please make sure metadata has been already harvested from web
+ * service and stored in Elasticsearch.
+ *
+ * @param es an Elasticsearch client node instance
+ * @param sc spark context
+ * @param index index name of log processing application
+ * @param type metadata type name
+ * @return PairRDD, in each pair key is metadata short name and value is term
+ * list extracted from metadata variables.
+ */
+ public JavaPairRDD<String, List<String>> loadMetadata(ESDriver es, JavaSparkContext sc, String index, String type) {
+ List<PODAACMetadata> metadatas = this.loadMetadataFromES(es, index, type);
+ JavaPairRDD<String, List<String>> metadataTermsRDD = this.buildMetadataRDD(es, sc, index, metadatas);
+ return metadataTermsRDD;
+ }
+
+ /**
+ * loadMetadataFromES: Load all metadata from Elasticsearch.
+ *
+ * @param es an Elasticsearch client node instance
+ * @param index index name of log processing application
+ * @param type metadata type name
+ * @return metadata list
+ */
+ protected List<PODAACMetadata> loadMetadataFromES(ESDriver es, String index, String type) {
+
+ List<PODAACMetadata> metadatas = new ArrayList<PODAACMetadata>();
+ SearchResponse scrollResp = es.getClient().prepareSearch(index).setTypes(type).setQuery(QueryBuilders.matchAllQuery()).setScroll(new TimeValue(60000)).setSize(100).execute().actionGet();
+
+ while (true) {
+ for (SearchHit hit : scrollResp.getHits().getHits()) {
+ Map<String, Object> result = hit.getSource();
+ String shortname = (String) result.get("Dataset-ShortName");
+ List<String> topic = (List<String>) result.get("DatasetParameter-Topic");
+ List<String> term = (List<String>) result.get("DatasetParameter-Term");
+ List<String> keyword = (List<String>) result.get("Dataset-Metadata");
+ List<String> variable = (List<String>) result.get("DatasetParameter-Variable");
+ List<String> longname = (List<String>) result.get("DatasetProject-Project-LongName");
+
+ List<String> region = (List<String>) result.get("DatasetRegion-Region");
+
+ PODAACMetadata metadata = null;
+ try {
+ metadata = new PODAACMetadata(shortname, longname, es.customAnalyzing(index, topic), es.customAnalyzing(index, term), es.customAnalyzing(index, variable), es.customAnalyzing(index, keyword),
+ es.customAnalyzing(index, region));
+ } catch (InterruptedException | ExecutionException e) {
+ e.printStackTrace();
+
+ }
+ metadatas.add(metadata);
+ }
+ scrollResp = es.getClient().prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet();
+ if (scrollResp.getHits().getHits().length == 0) {
+ break;
+ }
+ }
+
+ return metadatas;
+ }
+
+ /**
+ * buildMetadataRDD: Convert metadata list to JavaPairRDD
+ *
+ * @param es an Elasticsearch client node instance
+ * @param sc spark context
+ * @param index index name of log processing application
+ * @param metadatas metadata list
+ * @return PairRDD, in each pair key is metadata short name and value is term
+ * list extracted from metadata variables.
+ */
+ protected JavaPairRDD<String, List<String>> buildMetadataRDD(ESDriver es, JavaSparkContext sc, String index, List<PODAACMetadata> metadatas) {
+ JavaRDD<PODAACMetadata> metadataRDD = sc.parallelize(metadatas);
+ JavaPairRDD<String, List<String>> metadataTermsRDD = metadataRDD.mapToPair(new PairFunction<PODAACMetadata, String, List<String>>() {
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Tuple2<String, List<String>> call(PODAACMetadata metadata) throws Exception {
+ return new Tuple2<String, List<String>>(metadata.getShortName(), metadata.getAllTermList());
+ }
+ }).reduceByKey(new Function2<List<String>, List<String>, List<String>>() {
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public List<String> call(List<String> v1, List<String> v2) throws Exception {
+ List<String> list = new ArrayList<String>();
+ list.addAll(v1);
+ list.addAll(v2);
+ return list;
+ }
+ });
+
+ return metadataTermsRDD;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-mudrod/blob/39379fa9/core/src/main/java/org/apache/sdap/mudrod/metadata/structure/PODAACMetadata.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sdap/mudrod/metadata/structure/PODAACMetadata.java b/core/src/main/java/org/apache/sdap/mudrod/metadata/structure/PODAACMetadata.java
new file mode 100644
index 0000000..de3edf7
--- /dev/null
+++ b/core/src/main/java/org/apache/sdap/mudrod/metadata/structure/PODAACMetadata.java
@@ -0,0 +1,337 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sdap.mudrod.metadata.structure;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * ClassName: PODAACMetadata Function: PODAACMetadata setter and getter methods
+ */
+public class PODAACMetadata implements Serializable {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+ // shortname: data set short name
+ private String shortname;
+ // abstractStr: data set abstract
+ private String abstractStr;
+ // isoTopic: data set topic
+ private String isoTopic;
+ // sensor: sensor
+ private String sensor;
+ // source: data source
+ private String source;
+ // project: data project
+ private String project;
+ // hasAbstarct: whether data set has abstract
+ boolean hasAbstarct;
+
+ // longnameList: data set long name list
+ private List<String> longnameList;
+ // keywordList:data set key word list
+ private List<String> keywordList;
+ // termList: data set term list
+ private List<String> termList;
+ // topicList: data set topic list
+ private List<String> topicList;
+ // variableList: data set variable list
+ private List<String> variableList;
+ // abstractList: data set abstract term list
+ private List<String> abstractList;
+ // isotopicList: data set iso topic list
+ private List<String> isotopicList;
+ // sensorList: data set sensor list
+ private List<String> sensorList;
+ // sourceList: data set source list
+ private List<String> sourceList;
+ // projectList: data set project list
+ private List<String> projectList;
+ // regionList: data set region list
+ private List<String> regionList;
+
+ public PODAACMetadata() {
+ // Default constructor
+ }
+
+ /**
+ * Creates a new instance of PODAACMetadata.
+ *
+ * @param shortname data set short name
+ * @param longname data set long name
+ * @param topics data set topics
+ * @param terms data set terms
+ * @param variables data set variables
+ * @param keywords data set keywords
+ * @param region list of regions
+ */
+ public PODAACMetadata(String shortname, List<String> longname, List<String> topics, List<String> terms, List<String> variables, List<String> keywords, List<String> region) {
+ this.shortname = shortname;
+ this.longnameList = longname;
+ this.keywordList = keywords;
+ this.termList = terms;
+ this.topicList = topics;
+ this.variableList = variables;
+ this.regionList = region;
+ }
+
+ /**
+ * setTerms: set term of data set
+ *
+ * @param termstr data set terms
+ */
+ public void setTerms(String termstr) {
+ this.splitString(termstr, this.termList);
+ }
+
+ /**
+ * setKeywords: set key word of data set
+ *
+ * @param keywords data set keywords
+ */
+ public void setKeywords(String keywords) {
+ this.splitString(keywords, this.keywordList);
+ }
+
+ /**
+ * setTopicList: set topic of data set
+ *
+ * @param topicStr data set topics
+ */
+ public void setTopicList(String topicStr) {
+ this.splitString(topicStr, this.topicList);
+ }
+
+ /**
+ * setVaraliableList: set varilable of data set
+ *
+ * @param varilableStr data set variables
+ */
+ public void setVaraliableList(String varilableStr) {
+ this.splitString(varilableStr, this.variableList);
+ }
+
+ /**
+ * setProjectList:set project of data set
+ *
+ * @param project data set projects
+ */
+ public void setProjectList(String project) {
+ this.splitString(project, this.projectList);
+ }
+
+ /**
+ * setSourceList: set source of data set
+ *
+ * @param source data set sources
+ */
+ public void setSourceList(String source) {
+ this.splitString(source, this.sourceList);
+ }
+
+ /**
+ * setSensorList: set sensor of data set
+ *
+ * @param sensor data set sensors
+ */
+ public void setSensorList(String sensor) {
+ this.splitString(sensor, this.sensorList);
+ }
+
+ /**
+ * setISOTopicList:set iso topic of data set
+ *
+ * @param isoTopic data set iso topics
+ */
+ public void setISOTopicList(String isoTopic) {
+ this.splitString(isoTopic, this.isotopicList);
+ }
+
+ /**
+ * getKeywordList: get key word of data set
+ *
+ * @return data set keyword list
+ */
+ public List<String> getKeywordList() {
+ return this.keywordList;
+ }
+
+ /**
+ * getTermList:get term list of data set
+ *
+ * @return data set term list
+ */
+ public List<String> getTermList() {
+ return this.termList;
+ }
+
+ /**
+ * getShortName:get short name of data set
+ *
+ * @return data set short name
+ */
+ public String getShortName() {
+ return this.shortname;
+ }
+
+ /**
+ * getKeyword:get key word of data set
+ *
+ * @return data set keyword string
+ */
+ public String getKeyword() {
+ return String.join(",", this.keywordList);
+ }
+
+ /**
+ * getTerm:get term of data set
+ *
+ * @return data set term string
+ */
+ public String getTerm() {
+ return String.join(",", this.termList);
+ }
+
+ /**
+ * getTopic:get topic of data set
+ *
+ * @return data set topic string
+ */
+ public String getTopic() {
+ return String.join(",", this.topicList);
+ }
+
+ /**
+ * getVariable:get variable of data set
+ *
+ * @return data set variable string
+ */
+ public String getVariable() {
+ return String.join(",", this.variableList);
+ }
+
+ /**
+ * getAbstract:get abstract of data set
+ *
+ * @return data set abstract
+ */
+ public String getAbstract() {
+ return this.abstractStr;
+ }
+
+ /**
+ * getProject:get project of data set
+ *
+ * @return data set project string
+ */
+ public String getProject() {
+ return this.project;
+ }
+
+ /**
+ * getSource:get source of data set
+ *
+ * @return data set source string
+ */
+ public String getSource() {
+ return this.source;
+ }
+
+ /**
+ * getSensor:get sensor of data set
+ *
+ * @return data set sensor string
+ */
+ public String getSensor() {
+ return this.sensor;
+ }
+
+ /**
+ * getISOTopic:get iso topic of data set
+ *
+ * @return data set ISO topic string
+ */
+ public String getISOTopic() {
+ return this.isoTopic;
+ }
+
+ /**
+ * getAllTermList: get all term list of data set
+ *
+ * @return data set term list
+ */
+ public List<String> getAllTermList() {
+ List<String> allterms = new ArrayList<>();
+
+ if (this.termList != null && !this.termList.isEmpty()) {
+ allterms.addAll(this.termList);
+ }
+
+ if (this.keywordList != null && !this.keywordList.isEmpty()) {
+ allterms.addAll(this.keywordList);
+ }
+
+ if (this.topicList != null && !this.topicList.isEmpty()) {
+ allterms.addAll(this.topicList);
+ }
+
+ if (this.variableList != null && !this.variableList.isEmpty()) {
+ allterms.addAll(this.variableList);
+ }
+
+ if (this.regionList != null && !this.regionList.isEmpty()) {
+ allterms.addAll(this.regionList);
+ }
+ return allterms;
+ }
+
+ /**
+ * splitString: split value of fields of data set
+ *
+ * @param oristr original string
+ * @param list result after splitting
+ */
+ private void splitString(String oristr, List<String> list) {
+ if (oristr == null) {
+ return;
+ }
+
+ if (oristr.startsWith("\"")) {
+ oristr = oristr.substring(1);
+ }
+ if (oristr.endsWith("\"")) {
+ oristr = oristr.substring(0, oristr.length() - 1);
+ }
+
+ String strs[] = oristr.trim().split(",");
+ if (strs != null) {
+ for (int i = 0; i < strs.length; i++) {
+ String str = strs[i].trim();
+ if (str.startsWith(",") || str.startsWith("\"")) {
+ str = str.substring(1);
+ }
+ if (str.endsWith(",") || str.endsWith("\"")) {
+ str = str.substring(0, str.length() - 1);
+ }
+ if (str == "") {
+ continue;
+ }
+ list.add(str);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-mudrod/blob/39379fa9/core/src/main/java/org/apache/sdap/mudrod/metadata/structure/package-info.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sdap/mudrod/metadata/structure/package-info.java b/core/src/main/java/org/apache/sdap/mudrod/metadata/structure/package-info.java
new file mode 100644
index 0000000..938b4ac
--- /dev/null
+++ b/core/src/main/java/org/apache/sdap/mudrod/metadata/structure/package-info.java
@@ -0,0 +1,17 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * This package includes classes needed for metadata analysis
+ */
+package org.apache.sdap.mudrod.metadata.structure;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-sdap-mudrod/blob/39379fa9/core/src/main/java/org/apache/sdap/mudrod/ontology/Ontology.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sdap/mudrod/ontology/Ontology.java b/core/src/main/java/org/apache/sdap/mudrod/ontology/Ontology.java
new file mode 100644
index 0000000..70116de
--- /dev/null
+++ b/core/src/main/java/org/apache/sdap/mudrod/ontology/Ontology.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sdap.mudrod.ontology;
+
+import java.util.Iterator;
+
+/**
+ * Base class for working with ontologies. Methods indicate ability
+ * to load, merge e.g. merge relevant ontology subgraphs into a new
+ * subgraph which can be used within Mudrod, subclass retreival,
+ * synonym expansion, etc.
+ *
+ * @author lewismc
+ */
+public interface Ontology {
+
+ /**
+ * Load an array URIs which resolve to ontology resources.
+ *
+ * @param urls a {@link java.lang.String} containing ontology URIs.
+ */
+ public void load(String[] urls);
+
+ /**
+ * Load a collection of default ontology resources.
+ */
+ public void load() ;
+
+ /**
+ * merge relevant ontology subgraphs into a new subgraph which can
+ * be used within Mudrod
+ *
+ * @param o an ontology to merge with the current ontology held
+ * within Mudrod.
+ */
+ public void merge(Ontology o);
+
+ /**
+ * Retreive all subclasses for a particular entity provided within the
+ * search term e.g.subclass-based query expansion.
+ *
+ * @param entitySearchTerm an input search term
+ * @return an {@link java.util.Iterator} object containing subClass entries.
+ */
+ public Iterator<String> subclasses(String entitySearchTerm);
+
+ /**
+ * Retreive all synonyms for a particular entity provided within the
+ * search term e.g.synonym-based query expansion.
+ *
+ * @param queryKeyPhrase a phrase to undertake synonym expansion on.
+ * @return an {@link java.util.Iterator} object containing synonym entries.
+ */
+ public Iterator<String> synonyms(String queryKeyPhrase);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-mudrod/blob/39379fa9/core/src/main/java/org/apache/sdap/mudrod/ontology/OntologyFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sdap/mudrod/ontology/OntologyFactory.java b/core/src/main/java/org/apache/sdap/mudrod/ontology/OntologyFactory.java
new file mode 100644
index 0000000..984607c
--- /dev/null
+++ b/core/src/main/java/org/apache/sdap/mudrod/ontology/OntologyFactory.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sdap.mudrod.ontology;
+
+import org.apache.sdap.mudrod.main.MudrodConstants;
+import org.apache.sdap.mudrod.ontology.process.EsipCOROntology;
+import org.apache.sdap.mudrod.ontology.process.EsipPortalOntology;
+import org.apache.sdap.mudrod.ontology.process.LocalOntology;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Properties;
+
+/**
+ * The mechanism for creating an {@link Ontology}
+ * implementation. The {@link Ontology} implementation
+ * should be specified in
+ * <a href="https://github.com/mudrod/mudrod/blob/master/core/src/main/resources/config.xml">
+ * config.xml</a> with configuration key
+ * <code>mudrod.ontology.implementation</code>.
+ * This property can also be accessed via
+ * {@link MudrodConstants#ONTOLOGY_IMPL}.
+ *
+ * @author lewismc
+ */
+public class OntologyFactory {
+
+ public static final Logger LOG = LoggerFactory.getLogger(OntologyFactory.class);
+
+ private Properties props;
+
+ /**
+ * The mechanism for creating an {@link Ontology}
+ * implementation.
+ *
+ * @param props a populated Mudrod {@link java.util.Properties} object.
+ */
+ public OntologyFactory(Properties props) {
+ this.props = props;
+ }
+
+ /**
+ * Obtain the {@link Ontology}
+ * implementation for use within Mudrod.
+ *
+ * @return Returns the ontology implementation specified
+ * in <a href="https://github.com/mudrod/mudrod/blob/master/core/src/main/resources/config.xml">
+ * config.xml</a> with configuration key
+ * <code>mudrod.ontology.implementation</code>. This property can also be accessed via
+ * {@link MudrodConstants#ONTOLOGY_IMPL}.
+ */
+ public Ontology getOntology() {
+
+ String ontologyImpl = this.props.getProperty(MudrodConstants.ONTOLOGY_IMPL, "Local");
+
+ LOG.info("Using ontology extension: {}", ontologyImpl);
+ Ontology ontImpl;
+ switch (ontologyImpl) {
+ case "EsipCOR":
+ ontImpl = new EsipCOROntology();
+ break;
+ case "EsipPortal":
+ ontImpl = new EsipPortalOntology();
+ break;
+ default:
+ ontImpl = new LocalOntology();
+ break;
+ }
+ return ontImpl;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-mudrod/blob/39379fa9/core/src/main/java/org/apache/sdap/mudrod/ontology/package-info.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sdap/mudrod/ontology/package-info.java b/core/src/main/java/org/apache/sdap/mudrod/ontology/package-info.java
new file mode 100644
index 0000000..44596e3
--- /dev/null
+++ b/core/src/main/java/org/apache/sdap/mudrod/ontology/package-info.java
@@ -0,0 +1,17 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * This package includes ontology pre-processing and processing classes.
+ */
+package org.apache.sdap.mudrod.ontology;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-sdap-mudrod/blob/39379fa9/core/src/main/java/org/apache/sdap/mudrod/ontology/pre/AggregateTriples.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sdap/mudrod/ontology/pre/AggregateTriples.java b/core/src/main/java/org/apache/sdap/mudrod/ontology/pre/AggregateTriples.java
new file mode 100644
index 0000000..e94d678
--- /dev/null
+++ b/core/src/main/java/org/apache/sdap/mudrod/ontology/pre/AggregateTriples.java
@@ -0,0 +1,225 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sdap.mudrod.ontology.pre;
+
+import org.apache.commons.io.FilenameUtils;
+import org.apache.sdap.mudrod.discoveryengine.DiscoveryStepAbstract;
+import org.apache.sdap.mudrod.driver.ESDriver;
+import org.apache.sdap.mudrod.driver.SparkDriver;
+import org.jdom2.Document;
+import org.jdom2.Element;
+import org.jdom2.JDOMException;
+import org.jdom2.Namespace;
+import org.jdom2.filter.ElementFilter;
+import org.jdom2.input.SAXBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Properties;
+
+/**
+ * Supports ability to extract triples (subclassOf, equivalent class) from OWL file
+ */
+public class AggregateTriples extends DiscoveryStepAbstract {
+ private static final long serialVersionUID = 1L;
+ private static final Logger LOG = LoggerFactory.getLogger(AggregateTriples.class);
+
+ public AggregateTriples(Properties props, ESDriver es, SparkDriver spark) {
+ super(props, es, spark);
+ }
+
+ /**
+ * Method of executing triple aggregation
+ */
+ @Override
+ public Object execute() {
+ File file = new File(this.props.getProperty("oceanTriples"));
+ if (file.exists()) {
+ file.delete();
+ }
+ try {
+ file.createNewFile();
+ } catch (IOException e2) {
+ e2.printStackTrace();
+ }
+
+ FileWriter fw;
+ try {
+ fw = new FileWriter(file.getAbsoluteFile());
+ bw = new BufferedWriter(fw);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ File[] files = new File(this.props.getProperty("ontologyInputDir")).listFiles();
+ for (File file_in : files) {
+ String ext = FilenameUtils.getExtension(file_in.getAbsolutePath());
+ if ("owl".equals(ext)) {
+ try {
+ loadxml(file_in.getAbsolutePath());
+ getAllClass();
+ } catch (JDOMException e1) {
+ e1.printStackTrace();
+ } catch (IOException e1) {
+ e1.printStackTrace();
+ }
+
+ }
+ }
+
+ try {
+ bw.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ return null;
+ }
+
+ public Document document;
+ public Element rootNode = null;
+ final static String owl_namespace = "http://www.w3.org/2002/07/owl#";
+ final static String rdf_namespace = "http://www.w3.org/1999/02/22-rdf-syntax-ns#";
+ final static String rdfs_namespace = "http://www.w3.org/2000/01/rdf-schema#";
+
+ BufferedWriter bw = null;
+
+ /**
+ * Load OWL file into memory
+ *
+ * @param filePathName local path of OWL file
+ * @throws JDOMException JDOMException
+ * @throws IOException IOException
+ */
+ public void loadxml(String filePathName) throws JDOMException, IOException {
+ SAXBuilder saxBuilder = new SAXBuilder();
+ File file = new File(filePathName);
+
+ document = saxBuilder.build(file);
+ rootNode = document.getRootElement();
+ }
+
+ /**
+ * Method of going through OWL structure
+ */
+ public void loopxml() {
+ Iterator<?> processDescendants = rootNode.getDescendants(new ElementFilter());
+ String text = "";
+
+ while (processDescendants.hasNext()) {
+ Element e = (Element) processDescendants.next();
+ String currentName = e.getName();
+ text = e.getTextTrim();
+ if ("".equals(text)) {
+ LOG.info(currentName);
+ } else {
+ LOG.info("{} : {}", currentName, text);
+ }
+ }
+ }
+
+ /**
+ * Method of identifying a specific child given a element name
+ *
+ * @param str element name
+ * @param ele parent element
+ * @return the element of child
+ */
+ public Element findChild(String str, Element ele) {
+ Iterator<?> processDescendants = ele.getDescendants(new ElementFilter());
+ String name = "";
+ Element result = null;
+
+ while (processDescendants.hasNext()) {
+ Element e = (Element) processDescendants.next();
+ name = e.getName();
+ if (name.equals(str)) {
+ result = e;
+ return result;
+ }
+ }
+ return result;
+
+ }
+
+ /**
+ * Method of extract triples (subclassOf, equivalent class) from OWL file
+ *
+ * @throws IOException IOException
+ */
+ public void getAllClass() throws IOException {
+ List<?> classElements = rootNode.getChildren("Class", Namespace.getNamespace("owl", owl_namespace));
+
+ for (int i = 0; i < classElements.size(); i++) {
+ Element classElement = (Element) classElements.get(i);
+ String className = classElement.getAttributeValue("about", Namespace.getNamespace("rdf", rdf_namespace));
+
+ if (className == null) {
+ className = classElement.getAttributeValue("ID", Namespace.getNamespace("rdf", rdf_namespace));
+ }
+
+ List<?> subclassElements = classElement.getChildren("subClassOf", Namespace.getNamespace("rdfs", rdfs_namespace));
+ for (int j = 0; j < subclassElements.size(); j++) {
+ Element subclassElement = (Element) subclassElements.get(j);
+ String subclassName = subclassElement.getAttributeValue("resource", Namespace.getNamespace("rdf", rdf_namespace));
+ if (subclassName == null) {
+ Element allValuesFromEle = findChild("allValuesFrom", subclassElement);
+ if (allValuesFromEle != null) {
+ subclassName = allValuesFromEle.getAttributeValue("resource", Namespace.getNamespace("rdf", rdf_namespace));
+ bw.write(cutString(className) + ",SubClassOf," + cutString(subclassName) + "\n");
+ }
+ } else {
+ bw.write(cutString(className) + ",SubClassOf," + cutString(subclassName) + "\n");
+ }
+
+ }
+
+ List equalClassElements = classElement.getChildren("equivalentClass", Namespace.getNamespace("owl", owl_namespace));
+ for (int k = 0; k < equalClassElements.size(); k++) {
+ Element equalClassElement = (Element) equalClassElements.get(k);
+ String equalClassElementName = equalClassElement.getAttributeValue("resource", Namespace.getNamespace("rdf", rdf_namespace));
+
+ if (equalClassElementName != null) {
+ bw.write(cutString(className) + ",equivalentClass," + cutString(equalClassElementName) + "\n");
+ }
+ }
+
+ }
+ }
+
+ /**
+ * Method of cleaning up a string
+ *
+ * @param str String needed to be processed
+ * @return the processed string
+ */
+ public String cutString(String str) {
+ str = str.substring(str.indexOf("#") + 1);
+ String[] strArray = str.split("(?=[A-Z])");
+ str = Arrays.toString(strArray);
+ return str.substring(1, str.length() - 1).replace(",", "");
+ }
+
+ @Override
+ public Object execute(Object o) {
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-mudrod/blob/39379fa9/core/src/main/java/org/apache/sdap/mudrod/ontology/pre/package-info.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sdap/mudrod/ontology/pre/package-info.java b/core/src/main/java/org/apache/sdap/mudrod/ontology/pre/package-info.java
new file mode 100644
index 0000000..3f7c87e
--- /dev/null
+++ b/core/src/main/java/org/apache/sdap/mudrod/ontology/pre/package-info.java
@@ -0,0 +1,17 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * This package includes ontology pre-processing classes.
+ */
+package org.apache.sdap.mudrod.ontology.pre;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-sdap-mudrod/blob/39379fa9/core/src/main/java/org/apache/sdap/mudrod/ontology/process/EsipCOROntology.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sdap/mudrod/ontology/process/EsipCOROntology.java b/core/src/main/java/org/apache/sdap/mudrod/ontology/process/EsipCOROntology.java
new file mode 100644
index 0000000..45d04a8
--- /dev/null
+++ b/core/src/main/java/org/apache/sdap/mudrod/ontology/process/EsipCOROntology.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sdap.mudrod.ontology.process;
+
+import java.util.Iterator;
+
+import org.apache.sdap.mudrod.ontology.Ontology;
+
+/**
+ * @author lewismc
+ */
+public class EsipCOROntology implements Ontology {
+
+ /**
+ *
+ */
+ public EsipCOROntology() {
+ //default constructor
+ }
+
+ @Override
+ public void load() {
+ // to be completed
+ }
+
+ /* (non-Javadoc)
+ * @see Ontology#load(java.lang.String[])
+ */
+ @Override
+ public void load(String[] urls) {
+ // to be completed
+ }
+
+ /* (non-Javadoc)
+ * @see Ontology#merge(Ontology)
+ */
+ @Override
+ public void merge(Ontology o) {
+ // to be completed
+ }
+
+ /* (non-Javadoc)
+ * @see Ontology#subclasses(java.lang.String)
+ */
+ @Override
+ public Iterator<String> subclasses(String entitySearchTerm) {
+ return null;
+ }
+
+ /* (non-Javadoc)
+ * @see Ontology#synonyms(java.lang.String)
+ */
+ @Override
+ public Iterator<String> synonyms(String queryKeyPhrase) {
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-mudrod/blob/39379fa9/core/src/main/java/org/apache/sdap/mudrod/ontology/process/EsipPortalOntology.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sdap/mudrod/ontology/process/EsipPortalOntology.java b/core/src/main/java/org/apache/sdap/mudrod/ontology/process/EsipPortalOntology.java
new file mode 100644
index 0000000..c989a29
--- /dev/null
+++ b/core/src/main/java/org/apache/sdap/mudrod/ontology/process/EsipPortalOntology.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sdap.mudrod.ontology.process;
+
+import java.util.Iterator;
+
+import org.apache.sdap.mudrod.ontology.Ontology;
+
+/**
+ * @author lewismc
+ */
+public class EsipPortalOntology implements Ontology {
+
+ /**
+ *
+ */
+ public EsipPortalOntology() {
+ //default constructor
+ }
+
+ /* (non-Javadoc)
+ * @see Ontology#load(java.lang.String[])
+ */
+ @Override
+ public void load(String[] urls) {
+ // to be completed
+ }
+
+ /* (non-Javadoc)
+ * @see Ontology#load()
+ */
+ @Override
+ public void load() {
+ // to be completed
+ }
+
+ /* (non-Javadoc)
+ * @see Ontology#merge(Ontology)
+ */
+ @Override
+ public void merge(Ontology o) {
+ // to be completed
+ }
+
+ /* (non-Javadoc)
+ * @see Ontology#subclasses(java.lang.String)
+ */
+ @Override
+ public Iterator<String> subclasses(String entitySearchTerm) {
+ return null;
+ }
+
+ /* (non-Javadoc)
+ * @see Ontology#synonyms(java.lang.String)
+ */
+ @Override
+ public Iterator<String> synonyms(String queryKeyPhrase) {
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-mudrod/blob/39379fa9/core/src/main/java/org/apache/sdap/mudrod/ontology/process/LocalOntology.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sdap/mudrod/ontology/process/LocalOntology.java b/core/src/main/java/org/apache/sdap/mudrod/ontology/process/LocalOntology.java
new file mode 100644
index 0000000..0380c07
--- /dev/null
+++ b/core/src/main/java/org/apache/sdap/mudrod/ontology/process/LocalOntology.java
@@ -0,0 +1,391 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sdap.mudrod.ontology.process;
+
+import org.apache.jena.ontology.Individual;
+import org.apache.jena.ontology.OntClass;
+import org.apache.jena.ontology.OntModel;
+import org.apache.jena.ontology.OntModelSpec;
+import org.apache.jena.ontology.OntResource;
+import org.apache.jena.ontology.Restriction;
+import org.apache.jena.rdf.model.AnonId;
+import org.apache.jena.rdf.model.Literal;
+import org.apache.jena.rdf.model.ModelFactory;
+import org.apache.jena.rdf.model.Resource;
+import org.apache.jena.shared.PrefixMapping;
+import org.apache.sdap.mudrod.ontology.Ontology;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.PrintStream;
+import java.net.MalformedURLException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * The LocalOntology implementation enables us to work with Ontology files
+ * whcih are cached locally and available on the runtime classpath e.g.
+ * in <code>src/main/resource/ontology/...</code>.
+ * From here we can test and iterate on how use of ontology can enhance search.
+ */
+public class LocalOntology implements Ontology {
+
+ public static final Logger LOG = LoggerFactory.getLogger(LocalOntology.class);
+
+ public static final String DELIMITER_SEARCHTERM = " ";
+
+ private Map<Object, Object> searchTerms = new HashMap<>();
+ private static OntologyParser parser;
+ private static OntModel ontologyModel;
+ private Ontology ontology;
+ private static Map<AnonId, String> mAnonIDs = new HashMap<>();
+ private static int mAnonCount = 0;
+ private List<String> ontArrayList;
+
+ public LocalOntology() {
+ //only initialize all the static variables
+ //if first time called to this ontology constructor
+ if (ontology == null) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Creating new ontology");
+ }
+ parser = new OwlParser();
+ ontology = this;
+ }
+ if (ontologyModel == null)
+ ontologyModel = ModelFactory.createOntologyModel(OntModelSpec.OWL_MEM, null);
+ load();
+ }
+
+ /**
+ * Static accessor for {@link LocalOntology}
+ * instance implementation defined within <code>config.xml</code>.
+ *
+ * @return a {@link LocalOntology}
+ */
+ public Ontology getInstance() {
+ if (ontology == null) {
+ ontology = new LocalOntology();
+ }
+ return ontology;
+ }
+
+ /**
+ * Load the default <i>sweetAll.owl</i> ontology
+ * from <a href="https://raw.githubusercontent.com/ESIPFed/sweet/master/2.4/sweetAll.owl">
+ * https://raw.githubusercontent.com/ESIPFed/sweet/master/2.4/sweetAll.owl</a>
+ */
+ @Override
+ public void load() {
+ URL ontURL = null;
+ try {
+ ontURL = new URL("https://raw.githubusercontent.com/ESIPFed/sweet/master/2.4/sweetAll.owl");
+ //ontURL = new URL("https://raw.githubusercontent.com/ESIPFed/sweet/master/2.4/reprDataProduct.owl");
+ } catch (MalformedURLException e) {
+ LOG.error("Error when attempting to create URL resource: ", e);
+ }
+ ontArrayList = new ArrayList<>();
+ try {
+ ontArrayList.add(ontURL.toURI().toString());
+ } catch (URISyntaxException e) {
+ LOG.error("Error in URL syntax, please check your Ontology resource: ", e);
+ }
+ if (!ontArrayList.isEmpty()) {
+ load(ontArrayList.stream().toArray(String[]::new));
+ }
+ }
+
+ /**
+ * Load a string array of local URIs which refernece .owl files.
+ */
+ @Override
+ public void load(String[] urls) {
+ for (int i = 0; i < urls.length; i++) {
+ String url = urls[i].trim();
+ if (!"".equals(url))
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Reading and processing {}", url);
+ }
+ load(ontologyModel, url);
+ }
+ parser.parse(ontology, ontologyModel);
+ }
+
+ private void load(Object m, String url) {
+ try {
+ ((OntModel) m).read(url, null, null);
+ LOG.info("Successfully processed {}", url);
+ } catch (Exception e) {
+ LOG.error("Failed whilst attempting to read ontology {}: Error: ", url, e);
+ }
+ }
+
+ /**
+ * Get the {@link org.apache.sdap.mudrod.ontology.process.OntologyParser}
+ * implementation being used to process the input ontology resources.
+ * @return an {@link org.apache.sdap.mudrod.ontology.process.OntologyParser} implementation
+ */
+ public OntologyParser getParser() {
+ if (parser == null) {
+ parser = new OwlParser();
+ }
+ return parser;
+ }
+
+ /**
+ * Return the {@link org.apache.jena.ontology.OntModel} instance
+ * which created from input ontology resources.
+ * @return a constructed {@link org.apache.jena.ontology.OntModel}
+ */
+ public static OntModel getModel() {
+ return ontologyModel;
+ }
+
+ /**
+ * Return the loaded Ontology resources.
+ * @return a {@link java.util.List} of resources.
+ */
+ public List<String> getLoadedOntologyResources() {
+ if (ontArrayList != null) {
+ return ontArrayList;
+ } else {
+ return new ArrayList<>();
+ }
+ }
+ /**
+ * Not yet implemented.
+ */
+ @Override
+ public void merge(Ontology o) {
+ // not yet implemented
+ }
+
+ /**
+ * Retrieve all subclasses of entity(ies) hashed to searchTerm
+ * @param entitySearchTerm a query (keywords) for which to obtain
+ * subclasses.
+ * @return an {@link java.util.Iterator} containing the subclass as Strings.
+ */
+ @Override
+ public Iterator<String> subclasses(String entitySearchTerm) {
+ Map<OntResource, String> classMap = retrieve(entitySearchTerm);
+ Map<String, String> subclasses = new HashMap<>();
+
+ Iterator<OntResource> iter = classMap.keySet().iterator();
+ while (iter.hasNext()) {
+ OntResource resource = iter.next();
+
+ if (resource instanceof OntClass) {
+ //get subclasses N.B. we only get direct sub-classes e.g. direct children
+ //it is possible for us to navigate the entire class tree if we wish, we simply
+ //need to pass the .listSubClasses(true) boolean parameter.
+ for (Iterator<?> i = ((OntClass) resource).listSubClasses(); i.hasNext();) {
+ OntResource subclass = (OntResource) i.next();
+ for (Iterator<?> j = subclass.listLabels(null); j.hasNext();) {
+ Literal l = (Literal) j.next();
+ subclasses.put(l.toString(), "1");
+ }
+ }
+ //get individuals
+ for (Iterator<?> i = ((OntClass) resource).listInstances(); i.hasNext(); ) {
+ OntResource subclass = (OntResource) i.next();
+ for (Iterator<?> j = subclass.listLabels(null); j.hasNext();) {
+ Literal l = (Literal) j.next();
+ subclasses.put(l.toString(), "1");
+ }
+ }
+ } else if (resource instanceof Individual) {
+ for (Iterator<?> i = resource.listSameAs(); i.hasNext();) {
+ OntResource subclass = (OntResource) i.next();
+ for (Iterator<?> j = subclass.listLabels(null); j.hasNext();) {
+ Literal l = (Literal) j.next();
+ subclasses.put(l.toString(), "1");
+ }
+ }
+ }
+ }
+ return subclasses.keySet().iterator();
+ }
+
+ /**
+ * Retreives synonyms for an given phrase if the phrase
+ * is present in the ontology
+ * @param queryKeyPhrase an input string representing a phrase
+ * for which we wish to obtain synonyms.
+ * @return an {@link java.util.Iterator} containing synonyms string tokens
+ * or an empty if no synonyms exist for the given queryKeyPhrase.
+ */
+ @Override
+ public Iterator synonyms(String queryKeyPhrase) {
+
+ Map<?, ?> classMap = retrieve(queryKeyPhrase);
+
+ Map<Object, Object> synonyms = new HashMap<>();
+
+ Iterator<?> iter = classMap.keySet().iterator();
+ while (iter.hasNext()) {
+ OntResource resource = (OntResource) iter.next();
+
+ //listLabels
+ for (Iterator<?> i = resource.listLabels(null); i.hasNext();) {
+ Literal l = (Literal) i.next();
+ synonyms.put(l.toString(), "1");
+ }
+
+ if (resource instanceof Individual) {
+ //get all individuals same as this one
+ for (Iterator<?> i = resource.listSameAs(); i.hasNext();) {
+ Individual individual = (Individual) i.next();
+ //add labels
+ for (Iterator<?> j = individual.listLabels(null); j.hasNext();) {
+ Literal l = (Literal) i.next();
+ synonyms.put(l.toString(), "1");
+ }
+ }
+ } else if (resource instanceof OntClass) {
+ //list equivalent classes
+ for (Iterator<?> i = ((OntClass) resource).listEquivalentClasses(); i.hasNext();) {
+ OntClass equivClass = (OntClass) i.next();
+ //add labels
+ for (Iterator<?> j = equivClass.listLabels(null); j.hasNext();) {
+ Literal l = (Literal) j.next();
+ synonyms.put(l.toString(), "1");
+ }
+ }
+ }
+ }
+
+ return synonyms.keySet().iterator();
+ }
+
+ public void addSearchTerm(String label, OntResource resource) {
+ Map<OntResource, String> m = retrieve(label);
+ if (m == null) {
+ m = new HashMap<>();
+ }
+ m.put(resource, "1");
+ searchTerms.put(label.toLowerCase(), m);
+ }
+
+ /**
+ * A basic lookup function for retrieving keys (phrases or tokens)
+ * from the ontology search terms map. Right now only exact lookups
+ * will retrieve a result... this could be improved by using some
+ * advanced parsing logic... such as Lucene query parser.
+ * @param label the label (phrases or tokens) to retrieve from the
+ * ontology search terms map.
+ * @return an {@link java.util.Map} if there are match(es)
+ * or an empty {@link java.util.HashMap} if there are no
+ * matches.
+ */
+ public Map<OntResource, String> retrieve(String label) {
+ @SuppressWarnings("unchecked")
+ Map<OntResource, String> m = (Map<OntResource, String>) searchTerms.get(label.toLowerCase());
+ if (m == null) {
+ m = new HashMap<>();
+ }
+ return m;
+ }
+
+ protected static void renderHierarchy(PrintStream out, OntClass cls, List<Object> occurs, int depth) {
+ renderClassDescription(out, cls, depth);
+ out.println();
+
+ // recurse to the next level down
+ if (cls.canAs(OntClass.class) && !occurs.contains(cls)) {
+ for (Iterator<?> i = cls.listSubClasses(true); i.hasNext(); ) {
+ OntClass sub = (OntClass) i.next();
+
+ // we push this expression on the occurs list before we recurse
+ occurs.add(cls);
+ renderHierarchy(out, sub, occurs, depth + 1);
+ occurs.remove(cls);
+ }
+ for (Iterator<?> i = cls.listInstances(); i.hasNext(); ) {
+ Individual individual = (Individual) i.next();
+ renderURI(out, individual.getModel(), individual.getURI());
+ out.print(" [");
+ for (Iterator<?> j = individual.listLabels(null); j.hasNext(); ) {
+ out.print(((Literal) j.next()).getString() + ", ");
+ }
+ out.print("] ");
+ out.println();
+ }
+ }
+ }
+
+ public static void renderClassDescription(PrintStream out, OntClass c, int depth) {
+ indent(out, depth);
+
+ if (c.isRestriction()) {
+ renderRestriction(out, (Restriction) c.as(Restriction.class));
+ } else {
+ if (!c.isAnon()) {
+ out.print("Class ");
+ renderURI(out, c.getModel(), c.getURI());
+
+ out.print(c.getLocalName());
+
+ out.print(" [");
+ for (Iterator<?> i = c.listLabels(null); i.hasNext(); ) {
+ out.print(((Literal) i.next()).getString() + ", ");
+ }
+ out.print("] ");
+ } else {
+ renderAnonymous(out, c, "class");
+ }
+ }
+ }
+
+ protected static void renderRestriction(PrintStream out, Restriction r) {
+ if (!r.isAnon()) {
+ out.print("Restriction ");
+ renderURI(out, r.getModel(), r.getURI());
+ } else {
+ renderAnonymous(out, r, "restriction");
+ }
+
+ out.print(" on property ");
+ renderURI(out, r.getModel(), r.getOnProperty().getURI());
+ }
+
+ protected static void renderURI(PrintStream out, PrefixMapping prefixes, String uri) {
+ out.print(prefixes.expandPrefix(uri));
+ }
+
+ protected static void renderAnonymous(PrintStream out, Resource anon, String name) {
+ String anonID = mAnonIDs.get(anon.getId());
+ if (anonID == null) {
+ anonID = "a-" + mAnonCount++;
+ mAnonIDs.put(anon.getId(), anonID);
+ }
+
+ out.print("Anonymous ");
+ out.print(name);
+ out.print(" with ID ");
+ out.print(anonID);
+ }
+
+ protected static void indent(PrintStream out, int depth) {
+ for (int i = 0; i < depth; i++) {
+ out.print(" ");
+ }
+ }
+
+}