You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oodt.apache.org by ma...@apache.org on 2014/08/10 09:33:38 UTC
svn commit: r1617057 [1/2] - in /oodt/trunk: ./
metadata/src/main/java/org/apache/oodt/cas/metadata/filenaming/
mvn/archetypes/radix/src/main/resources/archetype-resources/workflow/src/main/resources/bin/
pge/src/main/java/org/apache/oodt/cas/pge/ pge/...
Author: mattmann
Date: Sun Aug 10 07:33:37 2014
New Revision: 1617057
URL: http://svn.apache.org/r1617057
Log:
- fix for OODT-667 CAS-PGE no longer respects writers and file tags from earlier pgeConfig.xml files.
Added:
oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/config/RegExprOutputFiles.java
oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/ExternExtractorMetWriter.java
oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/FilenameExtractorWriter.java
oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/PcsMetFileWriter.java
oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/SciPgeConfigFileWriter.java
oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/metlist/
oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/metlist/MetadataListPcsMetFileWriter.java
oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/xslt/
oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/xslt/XslTransformWriter.java
Removed:
oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/XslTransformWriter.java
Modified:
oodt/trunk/CHANGES.txt
oodt/trunk/metadata/src/main/java/org/apache/oodt/cas/metadata/filenaming/PathUtilsNamingConvention.java
oodt/trunk/mvn/archetypes/radix/src/main/resources/archetype-resources/workflow/src/main/resources/bin/wmgr
oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/PGETaskInstance.java
oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/config/OutputDir.java
oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/config/PgeConfigMetKeys.java
oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/metadata/PgeMetadata.java
oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/metadata/PgeTaskMetKeys.java
oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/metadata/PgeTaskStatus.java
oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/GenericPgeObjectFactory.java
oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java
oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/CsvConfigFileWriter.java
oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/DynamicConfigFileWriter.java
oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/MetadataKeyReplacerTemplateWriter.java
oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/TextConfigFileWriter.java
oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/VelocityConfigFileWriter.java
oodt/trunk/pge/src/main/resources/examples/PgeConfigFiles/pge-config.xml
oodt/trunk/pge/src/test/java/org/apache/oodt/cas/pge/metadata/TestPgeMetadata.java
oodt/trunk/pge/src/test/java/org/apache/oodt/cas/pge/staging/TestFileStager.java
oodt/trunk/pge/src/test/java/org/apache/oodt/cas/pge/writers/MockDynamicConfigFileWriter.java
oodt/trunk/workflow/src/main/bin/wmgr
Modified: oodt/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/oodt/trunk/CHANGES.txt?rev=1617057&r1=1617056&r2=1617057&view=diff
==============================================================================
--- oodt/trunk/CHANGES.txt (original)
+++ oodt/trunk/CHANGES.txt Sun Aug 10 07:33:37 2014
@@ -4,6 +4,9 @@ Apache OODT Change Log
Release 0.7 - Current Development
--------------------------------------------
+* OODT-667 CAS-PGE no longer respects writers and file tags from
+ earlier pgeConfig.xml files (mattmann)
+
* OODT-741 Remove static modifier from elementMap, subToSuperMap and
productTypeElementMap fields in XMLValidationLayer (rlaidlaw)
Modified: oodt/trunk/metadata/src/main/java/org/apache/oodt/cas/metadata/filenaming/PathUtilsNamingConvention.java
URL: http://svn.apache.org/viewvc/oodt/trunk/metadata/src/main/java/org/apache/oodt/cas/metadata/filenaming/PathUtilsNamingConvention.java?rev=1617057&r1=1617056&r2=1617057&view=diff
==============================================================================
--- oodt/trunk/metadata/src/main/java/org/apache/oodt/cas/metadata/filenaming/PathUtilsNamingConvention.java (original)
+++ oodt/trunk/metadata/src/main/java/org/apache/oodt/cas/metadata/filenaming/PathUtilsNamingConvention.java Sun Aug 10 07:33:37 2014
@@ -20,6 +20,8 @@ package org.apache.oodt.cas.metadata.fil
import java.io.File;
import java.io.IOException;
+import java.util.List;
+
//Apache imports
import org.apache.commons.lang.Validate;
@@ -32,11 +34,18 @@ import org.apache.oodt.cas.metadata.util
* A {@link NamingConvention} which utilizes {@link PathUtils}.
*
* @author bfoster (Brian Foster)
+ * @author mattmann (Chris Mattmann)
*/
public class PathUtilsNamingConvention implements NamingConvention {
private String namingConv;
+ private Metadata tmpReplaceMet;
+
+ public PathUtilsNamingConvention(){
+ this.tmpReplaceMet = new Metadata();
+ }
+
public File rename(File file, Metadata metadata)
throws NamingConventionException {
try {
@@ -59,4 +68,12 @@ public class PathUtilsNamingConvention i
public void setNamingConv(String namingConv) {
this.namingConv = namingConv;
}
+
+ public void addTmpReplaceMet(String key, List<String> values) {
+ this.tmpReplaceMet.replaceMetadata(key, values);
+ }
+
+ public Metadata getTmpReplaceMet() {
+ return this.tmpReplaceMet;
+ }
}
Modified: oodt/trunk/mvn/archetypes/radix/src/main/resources/archetype-resources/workflow/src/main/resources/bin/wmgr
URL: http://svn.apache.org/viewvc/oodt/trunk/mvn/archetypes/radix/src/main/resources/archetype-resources/workflow/src/main/resources/bin/wmgr?rev=1617057&r1=1617056&r2=1617057&view=diff
==============================================================================
--- oodt/trunk/mvn/archetypes/radix/src/main/resources/archetype-resources/workflow/src/main/resources/bin/wmgr (original)
+++ oodt/trunk/mvn/archetypes/radix/src/main/resources/archetype-resources/workflow/src/main/resources/bin/wmgr Sun Aug 10 07:33:37 2014
@@ -84,6 +84,8 @@ if [ "$1" = "start" ]; then
-Djava.util.logging.config.file="$WORKFLOW_HOME"/etc/logging.properties \
-Dorg.apache.oodt.cas.workflow.properties="$WORKFLOW_HOME"/etc/workflow.properties \
-Djava.io.tmpdir="$OODT_TMPDIR" \
+ -Dorg.apache.oodt.cas.pge.task.metkeys.legacyMode="true" \
+ -Dorg.apache.oodt.cas.pge.task.status.legacyMode="true" \
org.apache.oodt.cas.workflow.system.XmlRpcWorkflowManager \
--portNum "$WORKFLOW_PORT" 2>&1 &
Modified: oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/PGETaskInstance.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/PGETaskInstance.java?rev=1617057&r1=1617056&r2=1617057&view=diff
==============================================================================
--- oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/PGETaskInstance.java (original)
+++ oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/PGETaskInstance.java Sun Aug 10 07:33:37 2014
@@ -27,6 +27,7 @@ import static org.apache.oodt.cas.pge.me
import static org.apache.oodt.cas.pge.metadata.PgeTaskMetKeys.INGEST_CLIENT_TRANSFER_SERVICE_FACTORY;
import static org.apache.oodt.cas.pge.metadata.PgeTaskMetKeys.INGEST_FILE_MANAGER_URL;
import static org.apache.oodt.cas.pge.metadata.PgeTaskMetKeys.LOG_FILENAME_PATTERN;
+import static org.apache.oodt.cas.pge.metadata.PgeTaskMetKeys.MET_FILE_EXT;
import static org.apache.oodt.cas.pge.metadata.PgeTaskMetKeys.NAME;
import static org.apache.oodt.cas.pge.metadata.PgeTaskMetKeys.MIME_EXTRACTOR_REPO;
import static org.apache.oodt.cas.pge.metadata.PgeTaskMetKeys.PGE_CONFIG_BUILDER;
@@ -37,14 +38,17 @@ import static org.apache.oodt.cas.pge.me
import static org.apache.oodt.cas.pge.metadata.PgeTaskStatus.CONF_FILE_BUILD;
import static org.apache.oodt.cas.pge.metadata.PgeTaskStatus.CRAWLING;
import static org.apache.oodt.cas.pge.metadata.PgeTaskStatus.RUNNING_PGE;
+import static org.apache.oodt.cas.pge.metadata.PgeTaskStatus.STAGING_INPUT;
import static org.apache.oodt.cas.pge.util.GenericPgeObjectFactory.createConfigFilePropertyAdder;
import static org.apache.oodt.cas.pge.util.GenericPgeObjectFactory.createFileStager;
import static org.apache.oodt.cas.pge.util.GenericPgeObjectFactory.createPgeConfigBuilder;
-import static org.apache.oodt.cas.pge.util.GenericPgeObjectFactory.createDynamicConfigFileWriter;
+import static org.apache.oodt.cas.pge.util.GenericPgeObjectFactory.createSciPgeConfigFileWriter;
//JDK imports
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.FileOutputStream;
+import java.io.IOException;
import java.net.URL;
import java.util.Date;
import java.util.LinkedList;
@@ -53,6 +57,7 @@ import java.util.logging.FileHandler;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.logging.SimpleFormatter;
+import java.util.regex.Pattern;
//Apache imports
import org.apache.commons.lang.Validate;
@@ -60,18 +65,22 @@ import org.apache.commons.lang.Validate;
//OODT imports
import org.apache.oodt.cas.crawl.AutoDetectProductCrawler;
import org.apache.oodt.cas.crawl.ProductCrawler;
+import org.apache.oodt.cas.crawl.StdProductCrawler;
import org.apache.oodt.cas.crawl.status.IngestStatus;
import org.apache.oodt.cas.metadata.Metadata;
import org.apache.oodt.cas.metadata.SerializableMetadata;
+import org.apache.oodt.cas.metadata.filenaming.PathUtilsNamingConvention;
import org.apache.oodt.cas.pge.config.DynamicConfigFile;
import org.apache.oodt.cas.pge.config.OutputDir;
import org.apache.oodt.cas.pge.config.PgeConfig;
+import org.apache.oodt.cas.pge.config.RegExprOutputFiles;
import org.apache.oodt.cas.pge.config.XmlFilePgeConfigBuilder;
import org.apache.oodt.cas.pge.metadata.PgeMetadata;
import org.apache.oodt.cas.pge.metadata.PgeTaskMetKeys;
import org.apache.oodt.cas.pge.staging.FileManagerFileStager;
import org.apache.oodt.cas.pge.staging.FileStager;
-import org.apache.oodt.cas.pge.writers.DynamicConfigFileWriter;
+import org.apache.oodt.cas.pge.writers.PcsMetFileWriter;
+import org.apache.oodt.cas.pge.writers.SciPgeConfigFileWriter;
import org.apache.oodt.cas.workflow.metadata.CoreMetKeys;
import org.apache.oodt.cas.workflow.structs.WorkflowTaskConfiguration;
import org.apache.oodt.cas.workflow.structs.WorkflowTaskInstance;
@@ -125,6 +134,7 @@ public class PGETaskInstance implements
createOuputDirsIfRequested();
updateStatus(CONF_FILE_BUILD.getWorkflowStatusName());
createDynamicConfigFiles();
+ updateStatus(STAGING_INPUT.getWorkflowStatusName());
stageFiles();
// Run the PGE.
@@ -335,39 +345,39 @@ public class PGETaskInstance implements
}
protected void createDynamicConfigFiles() throws Exception {
- logger.info("Starting creation of dynamic config files...");
+ logger.info("Starting creation of sci pge config files...");
for (DynamicConfigFile dynamicConfigFile : pgeConfig
.getDynamicConfigFiles()) {
createDynamicConfigFile(dynamicConfigFile);
}
- logger.info("Successfully wrote all dynamic config files!");
+ logger.info("Successfully wrote all sci pge config files!");
}
protected void createDynamicConfigFile(DynamicConfigFile dynamicConfigFile)
throws Exception {
Validate.notNull(dynamicConfigFile, "dynamicConfigFile cannot be null");
- logger.fine("Starting creation of dynamic config file ["
+ logger.fine("Starting creation of sci pge config file ["
+ dynamicConfigFile.getFilePath() + "]...");
// Create parent directory if it doesn't exist.
File parentDir = new File(dynamicConfigFile.getFilePath())
.getParentFile();
if (!(parentDir.exists() || parentDir.mkdirs())) {
- throw new Exception("Failed to create directory where dynamic config file ["
+ throw new Exception("Failed to create directory where sci pge config file ["
+ dynamicConfigFile.getFilePath() + "] was to be written");
}
// Load writer and write file.
- logger.fine("Loading writer class for dynamic config file ["
+ logger.fine("Loading writer class for sci pge config file ["
+ dynamicConfigFile.getFilePath() + "]...");
- DynamicConfigFileWriter writer = createDynamicConfigFileWriter(
+ SciPgeConfigFileWriter writer = createSciPgeConfigFileWriter(
dynamicConfigFile.getWriterClass(), logger);
logger.fine("Loaded writer [" + writer.getClass().getCanonicalName()
- + "] for dynamic config file [" + dynamicConfigFile.getFilePath()
+ + "] for sci pge config file [" + dynamicConfigFile.getFilePath()
+ "]...");
- logger.info("Writing dynamic config file [" + dynamicConfigFile.getFilePath()
+ logger.info("Writing sci pge config file [" + dynamicConfigFile.getFilePath()
+ "]...");
- File configFile = writer.generateFile(dynamicConfigFile.getFilePath(),
+ File configFile = writer.createConfigFile(dynamicConfigFile.getFilePath(),
pgeMetadata.asMetadata(), logger, dynamicConfigFile.getArgs());
if (!configFile.exists()) {
throw new Exception("Writer failed to create config file ["
@@ -435,11 +445,72 @@ public class PGETaskInstance implements
protected boolean wasPgeSuccessful(int returnCode) {
return returnCode == 0;
}
+
+ protected void processOutput() throws FileNotFoundException, IOException {
+ for (final OutputDir outputDir : this.pgeConfig.getOuputDirs()) {
+ File[] createdFiles = new File(outputDir.getPath()).listFiles();
+ for (File createdFile : createdFiles) {
+ Metadata outputMetadata = new Metadata();
+ for (RegExprOutputFiles regExprFiles : outputDir
+ .getRegExprOutputFiles()) {
+ if (Pattern.matches(regExprFiles.getRegExp(), createdFile
+ .getName())) {
+ try {
+ PcsMetFileWriter writer = (PcsMetFileWriter) Class
+ .forName(regExprFiles.getConverterClass())
+ .newInstance();
+ outputMetadata.replaceMetadata(this.getMetadataForFile(
+ (regExprFiles.getRenamingConv() != null)
+ ? createdFile = this.renameFile(createdFile, regExprFiles.getRenamingConv())
+ : createdFile, writer, regExprFiles.getArgs()));
+ } catch (Exception e) {
+ logger.severe(
+ "Failed to create metadata file for '"
+ + createdFile + "' : "
+ + e.getMessage());
+ }
+ }
+ }
+ if (outputMetadata.getAllKeys().size() > 0)
+ this.writeFromMetadata(outputMetadata, createdFile.getAbsolutePath()
+ + "." + this.pgeMetadata.getMetadata(MET_FILE_EXT));
+ }
+ }
+ }
+
+ protected File renameFile(File file, PathUtilsNamingConvention renamingConv)
+ throws Exception {
+ Metadata curMetadata = this.pgeMetadata.asMetadata();
+ curMetadata.replaceMetadata(renamingConv.getTmpReplaceMet());
+ return renamingConv.rename(file, curMetadata);
+ }
+
+ protected Metadata getMetadataForFile(File sciPgeCreatedDataFile,
+ PcsMetFileWriter writer, Object[] args) throws Exception {
+ return writer.getMetadataForFile(sciPgeCreatedDataFile,
+ this.pgeMetadata, args);
+ }
+
+ protected void writeFromMetadata(Metadata metadata, String toMetFilePath)
+ throws FileNotFoundException, IOException {
+ new SerializableMetadata(metadata, "UTF-8", false)
+ .writeMetadataToXmlStream(new FileOutputStream(toMetFilePath));
+ }
- protected ProductCrawler createProductCrawler() throws Exception {
+ protected ProductCrawler createProductCrawler() throws Exception {
+ /* create a ProductCrawler based on whether or not the output dir specifies a MIME_EXTRACTOR_REPO */
logger.info("Configuring ProductCrawler...");
- AutoDetectProductCrawler crawler = new AutoDetectProductCrawler();
- crawler.setMimeExtractorRepo(pgeMetadata.getMetadata(MIME_EXTRACTOR_REPO));
+ ProductCrawler crawler = null;
+ if (pgeMetadata.getMetadata(MIME_EXTRACTOR_REPO) != null &&
+ pgeMetadata.getMetadata(MIME_EXTRACTOR_REPO).equals("")){
+ crawler = new AutoDetectProductCrawler();
+ ((AutoDetectProductCrawler)crawler).
+ setMimeExtractorRepo(pgeMetadata.getMetadata(MIME_EXTRACTOR_REPO));
+ }
+ else{
+ crawler = new StdProductCrawler();
+ }
+
crawler.setClientTransferer(pgeMetadata
.getMetadata(INGEST_CLIENT_TRANSFER_SERVICE_FACTORY));
crawler.setFilemgrUrl(pgeMetadata.getMetadata(INGEST_FILE_MANAGER_URL));
@@ -466,7 +537,12 @@ public class PGETaskInstance implements
}
protected void runIngestCrawler(ProductCrawler crawler) throws Exception {
- // Determine directories to crawl.
+ // Determine if we need to create Metadata files
+ if (crawler instanceof StdProductCrawler){
+ this.processOutput();
+ }
+
+ // Determine directories to crawl.
List<File> crawlDirs = new LinkedList<File>();
for (OutputDir outputDir : pgeConfig.getOuputDirs()) {
crawlDirs.add(new File(outputDir.getPath()));
Modified: oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/config/OutputDir.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/config/OutputDir.java?rev=1617057&r1=1617056&r2=1617057&view=diff
==============================================================================
--- oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/config/OutputDir.java (original)
+++ oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/config/OutputDir.java Sun Aug 10 07:33:37 2014
@@ -16,6 +16,10 @@
*/
package org.apache.oodt.cas.pge.config;
+//JDK imports
+import java.util.List;
+import java.util.Vector;
+
//Apache imports
import org.apache.commons.lang.Validate;
@@ -29,23 +33,24 @@ public class OutputDir {
private String path;
private boolean createBeforeExe;
+ private List<RegExprOutputFiles> regExprOutputFilesList;
public OutputDir() {
- path = null;
- createBeforeExe = false;
+ this(null, false);
}
public OutputDir(String path, boolean createBeforeExe) {
setPath(path);
setCreateBeforeExe(createBeforeExe);
+ this.regExprOutputFilesList = new Vector<RegExprOutputFiles>();
+
}
public void setPath(String path) {
Validate.notNull(path, "path cannot be null");
-
this.path = path;
}
-
+
public String getPath() {
return path;
}
@@ -57,4 +62,13 @@ public class OutputDir {
public boolean isCreateBeforeExe() {
return createBeforeExe;
}
+
+ public void addRegExprOutputFiles(RegExprOutputFiles regExprOutputFiles) {
+ this.regExprOutputFilesList.add(regExprOutputFiles);
+ }
+
+ public List<RegExprOutputFiles> getRegExprOutputFiles() {
+ return this.regExprOutputFilesList;
+ }
+
}
Modified: oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/config/PgeConfigMetKeys.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/config/PgeConfigMetKeys.java?rev=1617057&r1=1617056&r2=1617057&view=diff
==============================================================================
--- oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/config/PgeConfigMetKeys.java (original)
+++ oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/config/PgeConfigMetKeys.java Sun Aug 10 07:33:37 2014
@@ -73,6 +73,18 @@ public interface PgeConfigMetKeys {
public static final String DIR_TAG = "dir";
public static final String CREATE_BEFORE_EXEC_ATTR = "createBeforeExe";
+
+ public static final String FILES_TAG = "files";
+
+ public static final String REGEX_ATTR = "regExp";
+
+ public static final String NAME_ATTR = "name";
+
+ public static final String MET_FILE_WRITER_CLASS_ATTR = "metFileWriterClass";
+
+ public static final String RENAMING_CONV_TAG = "renamingConv";
+
+ public static final String NAMING_EXPR_ATTR = "namingExpr";
public static final String FILE_STAGING_TAG = "fileStaging";
Added: oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/config/RegExprOutputFiles.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/config/RegExprOutputFiles.java?rev=1617057&view=auto
==============================================================================
--- oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/config/RegExprOutputFiles.java (added)
+++ oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/config/RegExprOutputFiles.java Sun Aug 10 07:33:37 2014
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.oodt.cas.pge.config;
+
+import org.apache.oodt.cas.metadata.filenaming.PathUtilsNamingConvention;
+
+/**
+ *
+ * @author bfoster
+ * @version $Revision$
+ *
+ * <p>
+ * A regular expression class to define what OutputFiles
+ * to identify after running a PGE
+ * </p>.
+ */
+public class RegExprOutputFiles {
+
+ private String regExp;
+
+ private String converterClass;
+
+ private Object[] args;
+
+ private PathUtilsNamingConvention renamingConv;
+
+ public RegExprOutputFiles(String regExp, String converterClass,
+ PathUtilsNamingConvention renamingConv, Object[] args) {
+ this.regExp = regExp;
+ this.converterClass = converterClass;
+ this.renamingConv = renamingConv;
+ this.args = args;
+ }
+
+ public PathUtilsNamingConvention getRenamingConv() {
+ return this.renamingConv;
+ }
+
+ public String getRegExp() {
+ return this.regExp;
+ }
+
+ public String getConverterClass() {
+ return this.converterClass;
+ }
+
+ public Object[] getArgs() {
+ return this.args;
+ }
+
+}
Modified: oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/metadata/PgeMetadata.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/metadata/PgeMetadata.java?rev=1617057&r1=1617056&r2=1617057&view=diff
==============================================================================
--- oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/metadata/PgeMetadata.java (original)
+++ oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/metadata/PgeMetadata.java Sun Aug 10 07:33:37 2014
@@ -22,12 +22,15 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.Vector;
+
//Apache imports
import org.apache.commons.lang.Validate;
//OODT imports
import org.apache.oodt.cas.metadata.Metadata;
+
//Google imports
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
@@ -43,14 +46,18 @@ import com.google.common.collect.Sets;
* key as dynamic and commit it.
*
* @author bfoster (Brian Foster)
+ * @author mattmann (Chris Mattmann)
*/
public class PgeMetadata {
public enum Type {
STATIC, DYNAMIC, LOCAL;
}
- public static final List<Type> DEFAULT_PRECENDENCE_HIERARCHY = Lists
- .newArrayList(Type.DYNAMIC, Type.LOCAL, Type.STATIC);
+ public static final List<Type> DEFAULT_COMBINE_ORDER = Lists
+ .newArrayList(Type.LOCAL, Type.DYNAMIC, Type.STATIC);
+
+ public static final List<Type> DEFAULT_QUERY_ORDER = Lists
+ .newArrayList(Type.STATIC, Type.DYNAMIC, Type.LOCAL);
private final Metadata staticMetadata;
private final Metadata dynamicMetadata;
@@ -382,22 +389,21 @@ public class PgeMetadata {
* following: pgeMetadata.asMetadata(LOCAL, STATIC) then only LOCAL and
* STATIC metadata will be combined and LOCAL metadata will trump STATIC
* metadata if they both contain the same key. If no arguments are specified
- * then DEFAULT_PRECENDENCE_HIERARCHY is used.
+ * then DEFAULT_COMBINE_ORDER is used.
*
* @param types
* The Type hierarchy you which to use when metadata is combined,
- * if no args then DEFAULT_PRECENDENCE_HIERARCHY is used.
+ * if no args then DEFAULT_COMBINE_ORDER is used.
* @return Combined metadata.
*/
public Metadata asMetadata(Type... types) {
- List<Type> conbineOrder = Lists.newArrayList(types);
- if (conbineOrder.isEmpty()) {
- conbineOrder.addAll(DEFAULT_PRECENDENCE_HIERARCHY);
+ List<Type> combineOrder = Lists.newArrayList(types);
+ if (combineOrder.isEmpty()) {
+ combineOrder.addAll(DEFAULT_COMBINE_ORDER);
}
- Collections.reverse(conbineOrder);
Metadata combinedMetadata = new Metadata();
- for (Type type : conbineOrder) {
+ for (Type type : combineOrder) {
switch (type) {
case DYNAMIC:
combinedMetadata.replaceMetadata(dynamicMetadata);
@@ -426,7 +432,7 @@ public class PgeMetadata {
/**
* Get metadata values for given key. If Types are specified then it provides
* the precedence order in which to search for the key. If no Type args are
- * specified then DEFAULT_PRECENDENCE_HIERARCHY will be used. For example if
+ * specified then DEFAULT_QUERY_ORDER will be used. For example if
* you pass in Type args: STATIC, LOCAL then STATIC metadata will first be
* checked for the key and if it contains it, then it will return the found
* value, otherwise it will then check LOCAL metadata for the key and if it
@@ -436,13 +442,13 @@ public class PgeMetadata {
* The key for whose metadata values should be returned.
* @param types
* The type hierarchy which should be used, if no Types specified
- * DEFAULT_PRECENDENCE_HIERARCHY will be used.
+ * DEFAULT_QUERY_ORDER will be used.
* @return Metadata values for given key.
*/
public List<String> getAllMetadata(String key, Type... types) {
List<Type> queryOrder = Lists.newArrayList(types);
if (queryOrder.isEmpty()) {
- queryOrder.addAll(DEFAULT_PRECENDENCE_HIERARCHY);
+ queryOrder.addAll(DEFAULT_QUERY_ORDER);
}
String useKey = resolveKey(key);
@@ -465,7 +471,7 @@ public class PgeMetadata {
break;
}
}
- return null;
+ return new Vector<String>();
}
public String getMetadata(PgeTaskMetKeys key, Type... types) {
@@ -478,6 +484,6 @@ public class PgeMetadata {
*/
public String getMetadata(String key, Type... types) {
List<String> values = getAllMetadata(key, types);
- return values != null ? values.get(0) : null;
+ return values != null && values.size() > 0 ? values.get(0) : null;
}
}
Modified: oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/metadata/PgeTaskMetKeys.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/metadata/PgeTaskMetKeys.java?rev=1617057&r1=1617056&r2=1617057&view=diff
==============================================================================
--- oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/metadata/PgeTaskMetKeys.java (original)
+++ oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/metadata/PgeTaskMetKeys.java Sun Aug 10 07:33:37 2014
@@ -43,13 +43,13 @@ public enum PgeTaskMetKeys {
*/
PGE_CONFIG_BUILDER(
"PGETask/PgeConfigBuilder",
- "PGETask/PgeConfigBuilder"),
+ "PGETask_PgeConfigBuilder"),
/**
* The java logger {@link FileHandler} pattern (only for filename though).
*/
LOG_FILENAME_PATTERN(
"PGETask/LogFilenamePattern",
- "PGETask/LogFilenamePattern"),
+ "PGETask_LogFilenamePattern"),
/**
* List of {@link ConfigFilePropertyAdder}s classpaths to be run.
*/
@@ -62,13 +62,13 @@ public enum PgeTaskMetKeys {
*/
FILE_STAGER(
"PGETask/FileStager",
- "PGETask/FileStager"),
+ "PGETask_FileStager"),
/**
* List of {@link ConfigFilePropertyAdder}s classpaths to be run.
*/
DUMP_METADATA(
"PGETask/DumpMetadata",
- "PGETask/DumpMetadata"),
+ "PGETask_DumpMetadata"),
/**
* Set by CAS-PGE to the number of milliseconds it took CAS-PGE to run.
*/
@@ -111,7 +111,7 @@ public enum PgeTaskMetKeys {
*/
CRAWLER_CONFIG_FILE(
"PGETask/Ingest/CrawlerConfigFile",
- "PGETask/Ingest/CrawlerConfigFile"),
+ "PCS_ActionRepoFile"),
/**
* The IDs of the {@link CrawlerAction}s in the {@link #ACTION_REPO_FILE} to run.
*/
@@ -136,7 +136,7 @@ public enum PgeTaskMetKeys {
*/
MIME_EXTRACTOR_REPO(
"PGETask/Ingest/MimeExtractorRepo",
- "PGETask/Ingest/MimeExtractorRepo"),
+ "PGETask_MimeExtractorRepo"),
/**
* List of metadata keys required for Product ingest.
*/
@@ -150,9 +150,21 @@ public enum PgeTaskMetKeys {
*/
ATTEMPT_INGEST_ALL(
"PGETask/Ingest/AttemptIngestAll",
- "PGETask_AttemptIngestAll");
+ "PGETask_AttemptIngestAll"),
+
+ /**
+ * Identifies the metadata file name extension to use when CAS-PGE
+ * is running in legacy mode and generating metadata files for the
+ * StdProductCrawler. If not set, will default in CAS-PGE to .met.
+ * Note, there is no new version of this property, it only exists in
+ * legacy mode, but an attempt is made to provide a new path style/grouped
+ * version of the key for forward compat.
+ */
+ MET_FILE_EXT("PGETask/Ingest/MetFileExtension",
+ "PCS_MetFileExtension");
+
- public static final String USE_LEGACY_PROPERTY = "org.apache.oodt.cas.pge.legacyMode";
+ public static final String USE_LEGACY_PROPERTY = "org.apache.oodt.cas.pge.task.metkeys.legacyMode";
@VisibleForTesting String name;
@VisibleForTesting String legacyName;
Modified: oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/metadata/PgeTaskStatus.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/metadata/PgeTaskStatus.java?rev=1617057&r1=1617056&r2=1617057&view=diff
==============================================================================
--- oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/metadata/PgeTaskStatus.java (original)
+++ oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/metadata/PgeTaskStatus.java Sun Aug 10 07:33:37 2014
@@ -16,25 +16,55 @@
*/
package org.apache.oodt.cas.pge.metadata;
+//Google imports
+import com.google.common.annotations.VisibleForTesting;
+
/**
* Workflow Statuses for {@link PGETaskInstance}.
*
* @author bfoster (Brian Foster)
+ * @author mattmann (Chris Mattmann)
*/
public enum PgeTaskStatus {
+
+ /**
+ * The PGE is staging its input to the job working directory.
+ */
+ STAGING_INPUT("PGETask_Staging_Input", "STAGING INPUT"),
+
+ /**
+ * The PGE is building its internal configuration file.
+ */
+ CONF_FILE_BUILD("PGETask_Building_Config_File", "BUILDING CONFIG FILE"),
+
+ /**
+ * The PGE is executing.
+ */
+ RUNNING_PGE("PGETask_Running", "PGE EXEC"),
+
+ /**
+ * The ingest crawler has been created, either a StdProductCrawler
+ * or if {@link PgeTaskMetKeys#MIME_EXTRACTOR_REPO} has been specified
+ * an AutoDetectProductCrawler.
+ */
+ CRAWLING("PGETask_Crawling", "CRAWLING");
- STAGING_INPUT("PGETask_Staging_Input"),
- CONF_FILE_BUILD("PGETask_Building_Config_File"),
- RUNNING_PGE("PGETask_Running"),
- CRAWLING("PGETask_Crawling");
-
- private String workflowStatusName;
+ public static final String USE_LEGACY_STATUS_PROPERTY = "org.apache.oodt.cas.pge.task.status.legacyMode";
+
+ @VisibleForTesting private String workflowStatusName;
+ @VisibleForTesting private String legacyName;
- PgeTaskStatus(String workflowStatusName) {
+ PgeTaskStatus(String workflowStatusName, String legacyName) {
this.workflowStatusName = workflowStatusName;
+ this.legacyName = legacyName;
}
public String getWorkflowStatusName() {
- return workflowStatusName;
+ return Boolean.getBoolean(USE_LEGACY_STATUS_PROPERTY) ? this.legacyName:this.workflowStatusName;
+ }
+
+ @Override
+ public String toString() {
+ return getWorkflowStatusName();
}
}
Modified: oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/GenericPgeObjectFactory.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/GenericPgeObjectFactory.java?rev=1617057&r1=1617056&r2=1617057&view=diff
==============================================================================
--- oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/GenericPgeObjectFactory.java (original)
+++ oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/GenericPgeObjectFactory.java Sun Aug 10 07:33:37 2014
@@ -20,12 +20,14 @@ package org.apache.oodt.cas.pge.util;
import java.util.logging.Level;
import java.util.logging.Logger;
+
//OODT imports
import org.apache.oodt.cas.pge.ConfigFilePropertyAdder;
import org.apache.oodt.cas.pge.PGETaskInstance;
import org.apache.oodt.cas.pge.config.PgeConfigBuilder;
import org.apache.oodt.cas.pge.staging.FileStager;
import org.apache.oodt.cas.pge.writers.DynamicConfigFileWriter;
+import org.apache.oodt.cas.pge.writers.SciPgeConfigFileWriter;
/**
* Factory for creating {@link Object}s.
@@ -80,12 +82,12 @@ public class GenericPgeObjectFactory {
}
}
- public static DynamicConfigFileWriter createDynamicConfigFileWriter(
+ public static SciPgeConfigFileWriter createSciPgeConfigFileWriter(
String clazz, Logger logger) {
try {
- return (DynamicConfigFileWriter) Class.forName(clazz).newInstance();
+ return (SciPgeConfigFileWriter) Class.forName(clazz).newInstance();
} catch (Exception e) {
- logger.log(Level.SEVERE, "Failed to create DynamicConfigFileWriter ["
+ logger.log(Level.SEVERE, "Failed to create SciPgeConfigFileWriter ["
+ clazz + "] : " + e.getMessage(), e);
return null;
}
Modified: oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java?rev=1617057&r1=1617056&r2=1617057&view=diff
==============================================================================
--- oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java (original)
+++ oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java Sun Aug 10 07:33:37 2014
@@ -30,6 +30,7 @@ import static org.apache.oodt.cas.pge.co
import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.FILE_ATTR;
import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.FILE_STAGING_TAG;
import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.FILE_TAG;
+import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.FILES_TAG;
import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.FORCE_ATTR;
import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.IMPORT_TAG;
import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.KEYREF_ATTR;
@@ -37,9 +38,14 @@ import static org.apache.oodt.cas.pge.co
import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.KEY_GEN_ATTR;
import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.METADATA_KEY_ATTR;
import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.METADATA_TAG;
+import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.MET_FILE_WRITER_CLASS_ATTR;
+import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.NAME_ATTR;
+import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.NAMING_EXPR_ATTR;
import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.NAMESPACE_ATTR;
import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.OUTPUT_TAG;
import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.PATH_ATTR;
+import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.RENAMING_CONV_TAG;
+import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.REGEX_ATTR;
import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.SHELL_TYPE_ATTR;
import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.SPLIT_ATTR;
import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.STAGE_FILES_TAG;
@@ -54,6 +60,7 @@ import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.net.MalformedURLException;
import java.net.URL;
+import java.util.Arrays;
import java.util.List;
//OODT imports
@@ -61,9 +68,11 @@ import org.apache.oodt.cas.filemgr.syste
import org.apache.oodt.cas.filemgr.util.QueryUtils;
import org.apache.oodt.cas.filemgr.util.SqlParser;
import org.apache.oodt.cas.metadata.Metadata;
+import org.apache.oodt.cas.metadata.filenaming.PathUtilsNamingConvention;
import org.apache.oodt.cas.metadata.util.PathUtils;
import org.apache.oodt.cas.pge.config.DynamicConfigFile;
import org.apache.oodt.cas.pge.config.OutputDir;
+import org.apache.oodt.cas.pge.config.RegExprOutputFiles;
import org.apache.oodt.commons.xml.XMLUtils;
//DOM imports
@@ -78,339 +87,418 @@ import com.google.common.collect.Lists;
/**
* Help class with method for parsing XMLFilePgeConfigBuilder config XML file.
- *
+ *
* @author bfoster (Brian Foster)
+ * @author mattmann (Chris Mattmann)
*/
public class XmlHelper {
- private XmlHelper() {}
+ private XmlHelper() {
+ }
- public static Element getRootElement(String xmlFilePath)
- throws FileNotFoundException {
- return XMLUtils.getDocumentRoot(
- new FileInputStream(new File(xmlFilePath))).getDocumentElement();
- }
-
- public static List<Pair<String, String>> getImports(Element elem, Metadata metadata) throws Exception {
- List<Pair<String, String>> imports = Lists.newArrayList();
- NodeList nodeList = elem.getElementsByTagName(IMPORT_TAG);
- for (int i = 0; i < nodeList.getLength(); i++) {
- Element importElem = (Element) nodeList.item(i);
- String namespace = getNamespace(importElem, metadata);
- String file = getFile(importElem, metadata);
- imports.add(new Pair<String, String>(namespace, file));
- }
- return imports;
- }
-
- public static String getNamespace(Element elem, Metadata metadata) throws Exception {
- String namespace = elem.getAttribute(NAMESPACE_ATTR);
- if (!Strings.isNullOrEmpty(namespace)) {
- return fillIn(namespace, metadata, false);
- } else {
- return null;
- }
- }
-
- public static String getFile(Element elem, Metadata metadata) throws Exception {
- String file = elem.getAttribute(FILE_ATTR);
- if (!Strings.isNullOrEmpty(file)) {
- return fillIn(file, metadata, false);
- } else {
- return null;
- }
- }
-
- public static Element getCustomMetadataElement(Element root) throws Exception {
- NodeList nodes = root.getElementsByTagName(CUSTOM_METADATA_TAG);
- if (nodes.getLength() == 0) {
- return null;
- } else if (nodes.getLength() == 1) {
- return (Element) nodes.item(0);
- } else {
- throw new Exception("Found more than one '" + CUSTOM_METADATA_TAG
- + "' element");
- }
- }
-
- public static List<Element> getMetadataElements(Element customMetadataElem) {
- NodeList metElemsNodeList = customMetadataElem
- .getElementsByTagName(METADATA_TAG);
- List<Element> metElems = Lists.newArrayList();
- for (int i = 0; i < metElemsNodeList.getLength(); i++) {
- metElems.add((Element) metElemsNodeList.item(i));
- }
- return metElems;
- }
-
- public static String getMetadataKey(Element metElem, Metadata metadata) throws Exception {
- String key = metElem.getAttribute(KEY_ATTR);
-
- // no key attr, so check for key_gen attr.
- if (Strings.isNullOrEmpty(key)) {
- key = fillIn(metElem.getAttribute(KEY_GEN_ATTR), metadata);
- }
-
- // if still no key value, then fail.
- if (Strings.isNullOrEmpty(key)) {
- throw new Exception("Must specify either metadata attr '" + KEY_ATTR
- + "' or '" + KEY_GEN_ATTR + "'");
- }
-
- // else success!
- return key;
- }
-
- public static boolean isEnvReplaceNoRecur(Element elem, Metadata metadata)
- throws Exception {
- String isEnvReplaceNoRecur = elem.getAttribute(ENV_REPLACE_NO_RECUR_ATTR);
- if (Strings.isNullOrEmpty(isEnvReplaceNoRecur)) {
- return false;
- } else {
- return isEnvReplaceNoRecur.trim().toLowerCase().equals("true");
- }
- }
-
- public static boolean isEnvReplace(Element elem, Metadata metadata)
- throws Exception {
- String isEnvReplace = elem.getAttribute(ENV_REPLACE_ATTR);
- if (Strings.isNullOrEmpty(isEnvReplace)) {
- return true;
- } else {
- return !isEnvReplace.trim().toLowerCase().equals("false");
- }
- }
-
- public static boolean isMultiValue(Element elem, Metadata metadata)
- throws Exception {
- return Boolean.parseBoolean(fillIn(elem.getAttribute(SPLIT_ATTR),
- metadata));
- }
-
- public static List<String> getMetadataValues(Element elem, Metadata metadata)
- throws Exception {
- List<String> values = Lists.newArrayList();
-
- // Read val attr.
- String value = elem.getAttribute(VAL_ATTR);
-
- // Check if val tag was not specified see if value was given as element
- // text.
- if (Strings.isNullOrEmpty(value)) {
- value = elem.getTextContent();
- }
-
- // If value was found.
- if (!Strings.isNullOrEmpty(value)) {
-
- // Is multi-value so split up value.
- if (isMultiValue(elem, metadata)) {
- for (String v : Splitter.on(",").split(value)) {
-
- // Check for envReplace and perform met replacement on value if set.
- if (isEnvReplaceNoRecur(elem, metadata)) {
- values.add(fillIn(v, metadata, false));
- } else if (isEnvReplace(elem, metadata)) {
- values.add(fillIn(v, metadata));
- }
- }
-
- // Is scalar
- } else {
-
- // Check for envReplace and perform met replacement on value if set.
- if (isEnvReplaceNoRecur(elem, metadata)) {
- value = fillIn(value, metadata, false);
- } else if (isEnvReplace(elem, metadata)) {
- value = fillIn(value, metadata);
- }
- values.add(value);
- }
-
- }
- return values;
- }
-
- public static String getMetadataKeyRef(Element elem, Metadata metadata)
- throws Exception {
- String keyRef = elem.getAttribute(KEYREF_ATTR);
- if (!Strings.isNullOrEmpty(keyRef)) {
-
- // Check for envReplace and perform met replacement on value if set.
- if (isEnvReplaceNoRecur(elem, metadata)) {
- keyRef = fillIn(keyRef, metadata, false);
- } else if (isEnvReplace(elem, metadata)) {
- keyRef = fillIn(keyRef, metadata);
- }
-
- return keyRef;
- } else {
- return null;
- }
- }
-
- public static boolean isWorkflowMetKey(Element elem, Metadata metadata) throws Exception {
- return Boolean.parseBoolean(fillIn(elem.getAttribute(WORKFLOW_MET_ATTR), metadata, false));
- }
-
- public static String getPath(Element elem, Metadata metadata) throws Exception {
- return fillIn(elem.getAttribute(PATH_ATTR), metadata, false);
- }
-
- public static String getWriter(Element elem, Metadata metadata) throws Exception {
- return fillIn(elem.getAttribute(WRITER_CLASS_ATTR), metadata, false);
- }
-
- public static List<String> getArgs(Element elem, Metadata metadata)
- throws Exception {
- List<String> args = Lists.newArrayList();
- for (String arg : Splitter.on(",").split(elem.getAttribute(ARGS_ATTR))) {
- args.add(fillIn(arg, metadata, false));
- }
- if (args.size() == 1 && Strings.isNullOrEmpty(args.get(0))) {
- return Lists.newArrayList();
- } else {
- return args;
- }
- }
-
- public static List<DynamicConfigFile> getDynamicConfigFiles(Element elem,
- Metadata metadata) throws Exception {
- List<DynamicConfigFile> dynamicConfigFiles = Lists.newArrayList();
- NodeList nodeList = elem.getElementsByTagName(DYN_INPUT_FILES_TAG);
-
- // Check if dynInput element exists.
- if (nodeList.getLength() > 0) {
- Element dynamicConfigFilesElem = (Element) nodeList.item(0);
- nodeList = dynamicConfigFilesElem.getElementsByTagName(FILE_TAG);
-
- // Load each dynamic input file information.
- for (int i = 0; i < nodeList.getLength(); i++) {
- Element fileElem = (Element) nodeList.item(i);
- String path = getPath(fileElem, metadata);
- String writer = getWriter(fileElem, metadata);
- List<String> args = getArgs(fileElem, metadata);
- dynamicConfigFiles.add(new DynamicConfigFile(path, writer, args
- .toArray(new String[0])));
- }
- }
- return dynamicConfigFiles;
- }
-
- public static Element getExe(Element elem) throws Exception {
- NodeList nodeList = elem.getElementsByTagName(EXE_TAG);
- if (nodeList.getLength() > 1) {
- throw new Exception("Can only specify '" + EXE_TAG + "' once!");
- } else if (nodeList.getLength() == 1) {
- return (Element) nodeList.item(0);
- } else {
- return null;
- }
- }
-
- public static String getDir(Element elem, Metadata metadata)
- throws MalformedURLException, Exception {
- return fillIn(elem.getAttribute(DIR_ATTR), metadata);
- }
-
- public static String getShellType(Element elem, Metadata metadata)
- throws MalformedURLException, Exception {
- return fillIn(elem.getAttribute(SHELL_TYPE_ATTR), metadata);
- }
-
- public static List<String> getExeCmds(Element elem, Metadata metadata)
- throws MalformedURLException, DOMException, Exception {
- List<String> exeCmds = Lists.newArrayList();
- NodeList nodeList = elem.getElementsByTagName(CMD_TAG);
- for (int i = 0; i < nodeList.getLength(); i++) {
- Element cmdElem = (Element) nodeList.item(i);
- String exeCmd = cmdElem.getTextContent();
- if (isEnvReplaceNoRecur(cmdElem, metadata)) {
- exeCmd = fillIn(exeCmd, metadata, false);
- } else if (isEnvReplace(cmdElem, metadata)) {
- exeCmd = fillIn(exeCmd, metadata);
- }
- exeCmds.add(exeCmd);
- }
- return exeCmds;
- }
-
- public static Element getFileStaging(Element elem) throws Exception {
- NodeList nodeList = elem.getElementsByTagName(FILE_STAGING_TAG);
- if (nodeList.getLength() > 1) {
- throw new Exception("Can only specify '" + FILE_STAGING_TAG + "' once!");
- } else if (nodeList.getLength() == 1) {
- return (Element) nodeList.item(0);
- } else {
- return null;
- }
- }
-
- public static boolean isForceStage(Element elem, Metadata metadata) throws Exception {
- return Boolean.parseBoolean(fillIn(elem.getAttribute(FORCE_ATTR), metadata));
- }
-
- public static String getFileStagingMetadataKey(Element elem, Metadata metadata) throws Exception {
- return fillIn(elem.getAttribute(METADATA_KEY_ATTR), metadata);
- }
-
- public static List<String> getStageFilesMetKeys(Element elem, Metadata metadata) throws Exception {
- List<String> metKeys = Lists.newArrayList();
- NodeList nodeList = elem.getElementsByTagName(STAGE_FILES_TAG);
- for (int i = 0; i < nodeList.getLength(); i++) {
- Element stageFilesElem = (Element) nodeList.item(i);
- metKeys.add(getFileStagingMetadataKey(stageFilesElem, metadata));
- }
- return metKeys;
- }
-
- public static Element getOutput(Element elem) throws Exception {
- NodeList nodeList = elem.getElementsByTagName(OUTPUT_TAG);
- if (nodeList.getLength() > 1) {
- throw new Exception("Can only specify '" + OUTPUT_TAG + "' once!");
- } else if (nodeList.getLength() == 1) {
- return (Element) nodeList.item(0);
- } else {
- return null;
- }
- }
-
- public static boolean isCreateBeforeExe(Element elem, Metadata metadata) throws Exception {
- return Boolean.parseBoolean(fillIn(elem.getAttribute(CREATE_BEFORE_EXEC_ATTR), metadata));
- }
-
- public static List<OutputDir> getOuputDirs(Element elem, Metadata metadata) throws Exception {
- List<OutputDir> outputDirs = Lists.newArrayList();
- NodeList nodeList = elem.getElementsByTagName(DIR_TAG);
- for (int i = 0; i < nodeList.getLength(); i++) {
- Element outputDirElem = (Element) nodeList.item(i);
- String path = getPath(outputDirElem, metadata);
- boolean createBeforeExe = isCreateBeforeExe(outputDirElem, metadata);
- outputDirs.add(new OutputDir(path, createBeforeExe));
- }
- return outputDirs;
- }
-
- public static String fillIn(String value, Metadata inputMetadata) throws Exception {
- return fillIn(value, inputMetadata, true);
- }
-
- public static String fillIn(String value, Metadata inputMetadata,
- boolean envReplaceRecur) throws Exception {
- try {
- while ((value = PathUtils.doDynamicReplacement(value, inputMetadata))
- .contains("[") && envReplaceRecur)
- ;
- if (value.toUpperCase().matches(
- "^\\s*SQL\\s*\\(.*\\)\\s*\\{.*\\}\\s*$"))
- value = QueryUtils
- .getQueryResultsAsString(new XmlRpcFileManagerClient(new URL(
- inputMetadata.getMetadata(QUERY_FILE_MANAGER_URL
- .getName()))).complexQuery(SqlParser
- .parseSqlQueryMethod(value)));
- return value;
- } catch (Exception e) {
- throw new Exception("Failed to parse value: " + value, e);
- }
- }
+ public static Element getRootElement(String xmlFilePath)
+ throws FileNotFoundException {
+ return XMLUtils.getDocumentRoot(
+ new FileInputStream(new File(xmlFilePath)))
+ .getDocumentElement();
+ }
+
+ public static List<Pair<String, String>> getImports(Element elem,
+ Metadata metadata) throws Exception {
+ List<Pair<String, String>> imports = Lists.newArrayList();
+ NodeList nodeList = elem.getElementsByTagName(IMPORT_TAG);
+ for (int i = 0; i < nodeList.getLength(); i++) {
+ Element importElem = (Element) nodeList.item(i);
+ String namespace = getNamespace(importElem, metadata);
+ String file = getFile(importElem, metadata);
+ imports.add(new Pair<String, String>(namespace, file));
+ }
+ return imports;
+ }
+
+ public static String getNamespace(Element elem, Metadata metadata)
+ throws Exception {
+ String namespace = elem.getAttribute(NAMESPACE_ATTR);
+ if (!Strings.isNullOrEmpty(namespace)) {
+ return fillIn(namespace, metadata, false);
+ } else {
+ return null;
+ }
+ }
+
+ public static String getFile(Element elem, Metadata metadata)
+ throws Exception {
+ String file = elem.getAttribute(FILE_ATTR);
+ if (!Strings.isNullOrEmpty(file)) {
+ return fillIn(file, metadata, false);
+ } else {
+ return null;
+ }
+ }
+
+ public static Element getCustomMetadataElement(Element root)
+ throws Exception {
+ NodeList nodes = root.getElementsByTagName(CUSTOM_METADATA_TAG);
+ if (nodes.getLength() == 0) {
+ return null;
+ } else if (nodes.getLength() == 1) {
+ return (Element) nodes.item(0);
+ } else {
+ throw new Exception("Found more than one '" + CUSTOM_METADATA_TAG
+ + "' element");
+ }
+ }
+
+ public static List<Element> getMetadataElements(Element customMetadataElem) {
+ NodeList metElemsNodeList = customMetadataElem
+ .getElementsByTagName(METADATA_TAG);
+ List<Element> metElems = Lists.newArrayList();
+ for (int i = 0; i < metElemsNodeList.getLength(); i++) {
+ metElems.add((Element) metElemsNodeList.item(i));
+ }
+ return metElems;
+ }
+
+ public static String getMetadataKey(Element metElem, Metadata metadata)
+ throws Exception {
+ String key = metElem.getAttribute(KEY_ATTR);
+
+ // no key attr, so check for key_gen attr.
+ if (Strings.isNullOrEmpty(key)) {
+ key = fillIn(metElem.getAttribute(KEY_GEN_ATTR), metadata);
+ }
+
+ // if still no key value, then fail.
+ if (Strings.isNullOrEmpty(key)) {
+ throw new Exception("Must specify either metadata attr '"
+ + KEY_ATTR + "' or '" + KEY_GEN_ATTR + "'");
+ }
+
+ // else success!
+ return key;
+ }
+
+ public static boolean isEnvReplaceNoRecur(Element elem, Metadata metadata)
+ throws Exception {
+ String isEnvReplaceNoRecur = elem
+ .getAttribute(ENV_REPLACE_NO_RECUR_ATTR);
+ if (Strings.isNullOrEmpty(isEnvReplaceNoRecur)) {
+ return false;
+ } else {
+ return isEnvReplaceNoRecur.trim().toLowerCase().equals("true");
+ }
+ }
+
+ public static boolean isEnvReplace(Element elem, Metadata metadata)
+ throws Exception {
+ String isEnvReplace = elem.getAttribute(ENV_REPLACE_ATTR);
+ if (Strings.isNullOrEmpty(isEnvReplace)) {
+ return true;
+ } else {
+ return !isEnvReplace.trim().toLowerCase().equals("false");
+ }
+ }
+
+ public static boolean isMultiValue(Element elem, Metadata metadata)
+ throws Exception {
+ return Boolean.parseBoolean(fillIn(elem.getAttribute(SPLIT_ATTR),
+ metadata));
+ }
+
+ public static List<String> getMetadataValues(Element elem, Metadata metadata)
+ throws Exception {
+ List<String> values = Lists.newArrayList();
+
+ // Read val attr.
+ String value = elem.getAttribute(VAL_ATTR);
+
+ // Check if val tag was not specified see if value was given as element
+ // text.
+ if (Strings.isNullOrEmpty(value)) {
+ value = elem.getTextContent();
+ }
+
+ // If value was found.
+ if (!Strings.isNullOrEmpty(value)) {
+
+ // Is multi-value so split up value.
+ if (isMultiValue(elem, metadata)) {
+ for (String v : Splitter.on(",").split(value)) {
+
+ // Check for envReplace and perform met replacement on value
+ // if set.
+ if (isEnvReplaceNoRecur(elem, metadata)) {
+ values.add(fillIn(v, metadata, false));
+ } else if (isEnvReplace(elem, metadata)) {
+ values.add(fillIn(v, metadata));
+ }
+ }
+
+ // Is scalar
+ } else {
+
+ // Check for envReplace and perform met replacement on value if
+ // set.
+ if (isEnvReplaceNoRecur(elem, metadata)) {
+ value = fillIn(value, metadata, false);
+ } else if (isEnvReplace(elem, metadata)) {
+ value = fillIn(value, metadata);
+ }
+ values.add(value);
+ }
+
+ }
+ return values;
+ }
+
+ public static String getMetadataKeyRef(Element elem, Metadata metadata)
+ throws Exception {
+ String keyRef = elem.getAttribute(KEYREF_ATTR);
+ if (!Strings.isNullOrEmpty(keyRef)) {
+
+ // Check for envReplace and perform met replacement on value if set.
+ if (isEnvReplaceNoRecur(elem, metadata)) {
+ keyRef = fillIn(keyRef, metadata, false);
+ } else if (isEnvReplace(elem, metadata)) {
+ keyRef = fillIn(keyRef, metadata);
+ }
+
+ return keyRef;
+ } else {
+ return null;
+ }
+ }
+
+ public static boolean isWorkflowMetKey(Element elem, Metadata metadata)
+ throws Exception {
+ return Boolean.parseBoolean(fillIn(
+ elem.getAttribute(WORKFLOW_MET_ATTR), metadata, false));
+ }
+
+ public static String getPath(Element elem, Metadata metadata)
+ throws Exception {
+ return fillIn(elem.getAttribute(PATH_ATTR), metadata, false);
+ }
+
+ public static String getWriter(Element elem, Metadata metadata)
+ throws Exception {
+ return fillIn(elem.getAttribute(WRITER_CLASS_ATTR), metadata, false);
+ }
+
+ public static List<String> getArgs(Element elem, Metadata metadata)
+ throws Exception {
+ List<String> args = Lists.newArrayList();
+ for (String arg : Splitter.on(",").split(elem.getAttribute(ARGS_ATTR))) {
+ args.add(fillIn(arg, metadata, false));
+ }
+ if (args.size() == 1 && Strings.isNullOrEmpty(args.get(0))) {
+ return Lists.newArrayList();
+ } else {
+ return args;
+ }
+ }
+
+ public static List<DynamicConfigFile> getDynamicConfigFiles(Element elem,
+ Metadata metadata) throws Exception {
+ List<DynamicConfigFile> dynamicConfigFiles = Lists.newArrayList();
+ NodeList nodeList = elem.getElementsByTagName(DYN_INPUT_FILES_TAG);
+
+ // Check if dynInput element exists.
+ if (nodeList.getLength() > 0) {
+ Element dynamicConfigFilesElem = (Element) nodeList.item(0);
+ nodeList = dynamicConfigFilesElem.getElementsByTagName(FILE_TAG);
+
+ // Load each dynamic input file information.
+ for (int i = 0; i < nodeList.getLength(); i++) {
+ Element fileElem = (Element) nodeList.item(i);
+ String path = getPath(fileElem, metadata);
+ String writer = getWriter(fileElem, metadata);
+ List<String> args = getArgs(fileElem, metadata);
+ dynamicConfigFiles.add(new DynamicConfigFile(path, writer, args
+ .toArray(new String[0])));
+ }
+ }
+ return dynamicConfigFiles;
+ }
+
+ public static Element getExe(Element elem) throws Exception {
+ NodeList nodeList = elem.getElementsByTagName(EXE_TAG);
+ if (nodeList.getLength() > 1) {
+ throw new Exception("Can only specify '" + EXE_TAG + "' once!");
+ } else if (nodeList.getLength() == 1) {
+ return (Element) nodeList.item(0);
+ } else {
+ return null;
+ }
+ }
+
+ public static String getDir(Element elem, Metadata metadata)
+ throws MalformedURLException, Exception {
+ return fillIn(elem.getAttribute(DIR_ATTR), metadata);
+ }
+
+ public static String getShellType(Element elem, Metadata metadata)
+ throws MalformedURLException, Exception {
+ return fillIn(elem.getAttribute(SHELL_TYPE_ATTR), metadata);
+ }
+
+ public static List<String> getExeCmds(Element elem, Metadata metadata)
+ throws MalformedURLException, DOMException, Exception {
+ List<String> exeCmds = Lists.newArrayList();
+ NodeList nodeList = elem.getElementsByTagName(CMD_TAG);
+ for (int i = 0; i < nodeList.getLength(); i++) {
+ Element cmdElem = (Element) nodeList.item(i);
+ String exeCmd = cmdElem.getTextContent();
+ if (isEnvReplaceNoRecur(cmdElem, metadata)) {
+ exeCmd = fillIn(exeCmd, metadata, false);
+ } else if (isEnvReplace(cmdElem, metadata)) {
+ exeCmd = fillIn(exeCmd, metadata);
+ }
+ exeCmds.add(exeCmd);
+ }
+ return exeCmds;
+ }
+
+ public static Element getFileStaging(Element elem) throws Exception {
+ NodeList nodeList = elem.getElementsByTagName(FILE_STAGING_TAG);
+ if (nodeList.getLength() > 1) {
+ throw new Exception("Can only specify '" + FILE_STAGING_TAG
+ + "' once!");
+ } else if (nodeList.getLength() == 1) {
+ return (Element) nodeList.item(0);
+ } else {
+ return null;
+ }
+ }
+
+ public static boolean isForceStage(Element elem, Metadata metadata)
+ throws Exception {
+ return Boolean.parseBoolean(fillIn(elem.getAttribute(FORCE_ATTR),
+ metadata));
+ }
+
+ public static String getFileStagingMetadataKey(Element elem,
+ Metadata metadata) throws Exception {
+ return fillIn(elem.getAttribute(METADATA_KEY_ATTR), metadata);
+ }
+
+ public static List<String> getStageFilesMetKeys(Element elem,
+ Metadata metadata) throws Exception {
+ List<String> metKeys = Lists.newArrayList();
+ NodeList nodeList = elem.getElementsByTagName(STAGE_FILES_TAG);
+ for (int i = 0; i < nodeList.getLength(); i++) {
+ Element stageFilesElem = (Element) nodeList.item(i);
+ metKeys.add(getFileStagingMetadataKey(stageFilesElem, metadata));
+ }
+ return metKeys;
+ }
+
+ public static Element getOutput(Element elem) throws Exception {
+ NodeList nodeList = elem.getElementsByTagName(OUTPUT_TAG);
+ if (nodeList.getLength() > 1) {
+ throw new Exception("Can only specify '" + OUTPUT_TAG + "' once!");
+ } else if (nodeList.getLength() == 1) {
+ return (Element) nodeList.item(0);
+ } else {
+ return null;
+ }
+ }
+
+ public static boolean isCreateBeforeExe(Element elem, Metadata metadata)
+ throws Exception {
+ return Boolean.parseBoolean(fillIn(
+ elem.getAttribute(CREATE_BEFORE_EXEC_ATTR), metadata));
+ }
+
+ public static List<OutputDir> getOuputDirs(Element elem, Metadata metadata)
+ throws Exception {
+ List<OutputDir> outputDirs = Lists.newArrayList();
+ NodeList nodeList = elem.getElementsByTagName(DIR_TAG);
+ for (int i = 0; i < nodeList.getLength(); i++) {
+ Element outputDirElem = (Element) nodeList.item(i);
+ String path = getPath(outputDirElem, metadata);
+ boolean createBeforeExe = isCreateBeforeExe(outputDirElem, metadata);
+ OutputDir outputDir = new OutputDir(path, createBeforeExe);
+ getRegExpOutputFiles(outputDirElem, metadata, outputDir);
+ outputDirs.add(outputDir);
+ }
+ return outputDirs;
+
+ }
+
+ public static void getRegExpOutputFiles(Element elem, Metadata metadata,
+ OutputDir outputDir) throws Exception {
+ NodeList fileList = elem.getElementsByTagName(FILES_TAG);
+ for (int j = 0; j < fileList.getLength(); j++) {
+ Element fileElement = (Element) fileList.item(j);
+ String outputFile = fileElement.getAttribute(REGEX_ATTR);
+ if (outputFile.equals("")){
+ outputFile = fillIn(fileElement.getAttribute(NAME_ATTR),
+ metadata);
+ }
+ PathUtilsNamingConvention renamingConvention = null;
+ NodeList renamingConvNodes = fileElement
+ .getElementsByTagName(RENAMING_CONV_TAG);
+
+ if (renamingConvNodes.getLength() > 0) {
+ Element renamingElement = (Element) renamingConvNodes.item(0);
+ String namingExpr = renamingElement
+ .getAttribute(NAMING_EXPR_ATTR);
+ if (renamingElement.getAttribute(ENV_REPLACE_ATTR)
+ .toLowerCase().equals("true")){
+ namingExpr = fillIn(namingExpr, metadata, false);
+ }
+ else if (!renamingElement.getAttribute(ENV_REPLACE_ATTR)
+ .toLowerCase().equals("false")){
+ namingExpr = fillIn(namingExpr, metadata);
+ }
+ renamingConvention = new PathUtilsNamingConvention();
+ renamingConvention.setNamingConv(namingExpr);
+ NodeList metadataNodes = renamingElement
+ .getElementsByTagName(METADATA_TAG);
+ for (int k = 0; k < metadataNodes.getLength(); k++) {
+ renamingConvention.addTmpReplaceMet(((Element) metadataNodes
+ .item(k)).getAttribute(KEY_ATTR), Arrays
+ .asList(((Element) metadataNodes.item(k))
+ .getAttribute(VAL_ATTR).split(",")));
+ }
+ }
+
+ outputDir
+ .addRegExprOutputFiles(new RegExprOutputFiles(outputFile,
+ fillIn(fileElement
+ .getAttribute(MET_FILE_WRITER_CLASS_ATTR),
+ metadata), renamingConvention, (Object[]) fillIn(
+ fileElement.getAttribute(ARGS_ATTR),
+ metadata).split(",")));
+ }
+
+ }
+
+ public static String fillIn(String value, Metadata inputMetadata)
+ throws Exception {
+ return fillIn(value, inputMetadata, true);
+ }
+
+ public static String fillIn(String value, Metadata inputMetadata,
+ boolean envReplaceRecur) throws Exception {
+ try {
+ while ((value = PathUtils
+ .doDynamicReplacement(value, inputMetadata)).contains("[")
+ && envReplaceRecur)
+ ;
+ if (value.toUpperCase().matches(
+ "^\\s*SQL\\s*\\(.*\\)\\s*\\{.*\\}\\s*$"))
+ value = QueryUtils
+ .getQueryResultsAsString(new XmlRpcFileManagerClient(
+ new URL(inputMetadata
+ .getMetadata(QUERY_FILE_MANAGER_URL
+ .getName())))
+ .complexQuery(SqlParser
+ .parseSqlQueryMethod(value)));
+ return value;
+ } catch (Exception e) {
+ throw new Exception("Failed to parse value: " + value, e);
+ }
+ }
}
Modified: oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/CsvConfigFileWriter.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/CsvConfigFileWriter.java?rev=1617057&r1=1617056&r2=1617057&view=diff
==============================================================================
--- oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/CsvConfigFileWriter.java (original)
+++ oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/CsvConfigFileWriter.java Sun Aug 10 07:33:37 2014
@@ -65,7 +65,7 @@ import org.apache.oodt.cas.metadata.Meta
*
* @author bfoster (Brian Foster)
*/
-public class CsvConfigFileWriter implements DynamicConfigFileWriter {
+public class CsvConfigFileWriter extends DynamicConfigFileWriter {
private static final int HEADER_INDEX = 0;
private static final int DELIM_INDEX = 0;
Modified: oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/DynamicConfigFileWriter.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/DynamicConfigFileWriter.java?rev=1617057&r1=1617056&r2=1617057&view=diff
==============================================================================
--- oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/DynamicConfigFileWriter.java (original)
+++ oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/DynamicConfigFileWriter.java Sun Aug 10 07:33:37 2014
@@ -18,6 +18,7 @@ package org.apache.oodt.cas.pge.writers;
//JDK imports
import java.io.File;
+import java.io.IOException;
import java.util.logging.Logger;
//OODT imports
@@ -26,11 +27,52 @@ import org.apache.oodt.cas.metadata.Meta
/**
* Abstract interface for generating PGE config input files defining the input
* necessary to run the underlying PGE.
- *
+ *
* @author bfoster (Brian Foster)
+ * @author mattmann (Chris Mattmann)
*/
-public interface DynamicConfigFileWriter {
+public abstract class DynamicConfigFileWriter implements SciPgeConfigFileWriter {
- public File generateFile(String filePath, Metadata metadata, Logger logger,
- Object... args) throws Exception;
+ private static final Logger logger = Logger
+ .getLogger(DynamicConfigFileWriter.class.getName());
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.oodt.cas.pge.writers.SciPgeConfigFileWriter#createConfigFile
+ * (java.lang.String, org.apache.oodt.cas.metadata.Metadata,
+ * java.lang.Object[])
+ */
+ @Override
+ public File createConfigFile(String sciPgeConfigFilePath,
+ Metadata inputMetadata, Object... customArgs) throws IOException {
+ try {
+ return this.generateFile(sciPgeConfigFilePath, inputMetadata,
+ logger, customArgs);
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw new IOException(e);
+ }
+ }
+
+ /**
+ * Generates a config file for CAS-PGE to use as PGE input with the given
+ * default logger.
+ *
+ * @param filePath
+ * The name of the config file to generate.
+ * @param metadata
+ * Input CAS-PGE metadata.
+ * @param logger
+ * The logger to write any status information to.
+ * @param args
+ * Any custom parameters needed for the writer to write the input
+ * config file.
+ * @return The newly generated CAS-PGE input config file.
+ * @throws Exception
+ * If any error occurs.
+ */
+ public abstract File generateFile(String filePath, Metadata metadata,
+ Logger logger, Object... args) throws Exception;
}
Added: oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/ExternExtractorMetWriter.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/ExternExtractorMetWriter.java?rev=1617057&view=auto
==============================================================================
--- oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/ExternExtractorMetWriter.java (added)
+++ oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/ExternExtractorMetWriter.java Sun Aug 10 07:33:37 2014
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oodt.cas.pge.writers;
+
+//JDK imports
+import java.io.File;
+
+//OODT imports
+import org.apache.oodt.cas.metadata.Metadata;
+import org.apache.oodt.cas.metadata.extractors.ExternConfigReader;
+import org.apache.oodt.cas.metadata.extractors.ExternMetExtractor;
+import org.apache.oodt.cas.pge.writers.PcsMetFileWriter;
+
+/**
+ *
+ * Wraps the OODT CAS {@link ExternMetExtractor} and exposes it as a CAS-PGE
+ * {@link PcsMetFileWriter}.
+ *
+ */
+public class ExternExtractorMetWriter extends PcsMetFileWriter {
+
+ @Override
+ protected Metadata getSciPgeSpecificMetadata(File sciPgeConfigFilePath,
+ Metadata inputMetadata, Object... customArgs) throws Exception {
+ ExternMetExtractor extractor = new ExternMetExtractor();
+ extractor.setConfigFile(new ExternConfigReader().parseConfigFile(new File(
+ (String) customArgs[0])));
+ Metadata m = new Metadata();
+ m.addMetadata(extractor.extractMetadata(sciPgeConfigFilePath)
+ .getHashtable(), true);
+ return m;
+ }
+
+}
Added: oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/FilenameExtractorWriter.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/FilenameExtractorWriter.java?rev=1617057&view=auto
==============================================================================
--- oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/FilenameExtractorWriter.java (added)
+++ oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/FilenameExtractorWriter.java Sun Aug 10 07:33:37 2014
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oodt.cas.pge.writers;
+
+//JDK imports
+import java.io.File;
+
+//OODT imports
+import org.apache.oodt.cas.metadata.Metadata;
+import org.apache.oodt.cas.metadata.extractors.FilenameTokenMetExtractor;
+import org.apache.oodt.cas.pge.writers.PcsMetFileWriter;
+
+/**
+ *
+ * Wrap CAS-Metadata's {@link FilenameTokenMetExtractor} as a CAS-PGE
+ * {@link PcsMetFileWriter}. First arg passed in is the config file full path.
+ *
+ * @author mattmann
+ * @version $Revision$
+ *
+ */
+public class FilenameExtractorWriter extends PcsMetFileWriter {
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.oodt.cas.pge.writers.PcsMetFileWriter#getSciPgeSpecificMetadata
+ * (java.io.File, org.apache.oodt.cas.metadata.Metadata, java.lang.Object[])
+ */
+ @Override
+ protected Metadata getSciPgeSpecificMetadata(File generatedFile,
+ Metadata workflowMet, Object... args) throws Exception {
+ String metConfFilePath = String.valueOf(args[0]);
+ FilenameTokenMetExtractor extractor = new FilenameTokenMetExtractor();
+ extractor.setConfigFile(metConfFilePath);
+ return extractor.extractMetadata(generatedFile);
+ }
+
+}
Modified: oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/MetadataKeyReplacerTemplateWriter.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/MetadataKeyReplacerTemplateWriter.java?rev=1617057&r1=1617056&r2=1617057&view=diff
==============================================================================
--- oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/MetadataKeyReplacerTemplateWriter.java (original)
+++ oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/MetadataKeyReplacerTemplateWriter.java Sun Aug 10 07:33:37 2014
@@ -41,7 +41,7 @@ import org.apache.oodt.cas.metadata.Meta
* template file to use as a basis.
*
*/
-public class MetadataKeyReplacerTemplateWriter implements
+public class MetadataKeyReplacerTemplateWriter extends
DynamicConfigFileWriter {
private static final String DEFAULT_SEPARATOR = ",";
Added: oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/PcsMetFileWriter.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/PcsMetFileWriter.java?rev=1617057&view=auto
==============================================================================
--- oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/PcsMetFileWriter.java (added)
+++ oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/PcsMetFileWriter.java Sun Aug 10 07:33:37 2014
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.oodt.cas.pge.writers;
+
+//OODT imports
+import org.apache.oodt.cas.pge.metadata.PgeMetadata;
+import org.apache.oodt.cas.filemgr.metadata.CoreMetKeys;
+import org.apache.oodt.cas.metadata.Metadata;
+
+//OODT imports
+import java.io.File;
+
+/**
+ *
+ * @author bfoster
+ * @version $Revision$
+ *
+ * <p>
+ * Writes a PCS metadata file for the give data file
+ * </p>.
+ */
+public abstract class PcsMetFileWriter {
+
+ public static final String FILE_SIZE = "FileSize";
+
+ public Metadata getMetadataForFile(File sciPgeCreatedDataFile,
+ PgeMetadata pgeMetadata, Object... customArgs) throws Exception {
+ try {
+ Metadata inputMetadata = pgeMetadata.asMetadata();
+
+ inputMetadata.replaceMetadata(CoreMetKeys.FILENAME,
+ sciPgeCreatedDataFile.getName());
+ inputMetadata.replaceMetadata(CoreMetKeys.FILE_LOCATION,
+ sciPgeCreatedDataFile.getParentFile().getAbsolutePath());
+ inputMetadata.replaceMetadata(FILE_SIZE, Long.toString(new File(
+ inputMetadata.getMetadata(CoreMetKeys.FILE_LOCATION),
+ inputMetadata.getMetadata(CoreMetKeys.FILENAME)).length()));
+
+ return this.getSciPgeSpecificMetadata(
+ sciPgeCreatedDataFile, inputMetadata, customArgs);
+ } catch (Exception e) {
+ throw new Exception("Failed to create PCS metadata file for '"
+ + sciPgeCreatedDataFile + "' : " + e.getMessage(), e);
+ }
+ }
+
+ protected abstract Metadata getSciPgeSpecificMetadata(
+ File sciPgeCreatedDataFile, Metadata inputMetadata,
+ Object... customArgs) throws Exception;
+
+}
Added: oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/SciPgeConfigFileWriter.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/SciPgeConfigFileWriter.java?rev=1617057&view=auto
==============================================================================
--- oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/SciPgeConfigFileWriter.java (added)
+++ oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/SciPgeConfigFileWriter.java Sun Aug 10 07:33:37 2014
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.oodt.cas.pge.writers;
+
+//JDK imports
+import java.io.File;
+import java.io.IOException;
+
+//OODT imports
+import org.apache.oodt.cas.metadata.Metadata;
+
+/**
+ *
+ * @author bfoster
+ * @version $Revision$
+ *
+ * <p>
+ * Abstract interface for generating PGE config input files defining the input
+ * necessary to run the underlying PGE
+ * </p>.
+ */
+public interface SciPgeConfigFileWriter {
+
+ /**
+ *
+ * @param sciPgeConfigFilePath
+ * @param inputMetadata
+ * @param customArgs
+ * @return
+ * @throws IOException
+ */
+ public File createConfigFile(String sciPgeConfigFilePath,
+ Metadata inputMetadata, Object... customArgs) throws IOException;
+
+}
Modified: oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/TextConfigFileWriter.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/TextConfigFileWriter.java?rev=1617057&r1=1617056&r2=1617057&view=diff
==============================================================================
--- oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/TextConfigFileWriter.java (original)
+++ oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/TextConfigFileWriter.java Sun Aug 10 07:33:37 2014
@@ -42,7 +42,7 @@ import org.apache.oodt.cas.metadata.Meta
*
* @author bfoster (Brian Foster)
*/
-public class TextConfigFileWriter implements DynamicConfigFileWriter {
+public class TextConfigFileWriter extends DynamicConfigFileWriter {
private static final int TEMPLATE_INDEX = 0;
Modified: oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/VelocityConfigFileWriter.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/VelocityConfigFileWriter.java?rev=1617057&r1=1617056&r2=1617057&view=diff
==============================================================================
--- oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/VelocityConfigFileWriter.java (original)
+++ oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/VelocityConfigFileWriter.java Sun Aug 10 07:33:37 2014
@@ -41,7 +41,7 @@ import org.apache.velocity.app.Velocity;
* name.
*
*/
-public class VelocityConfigFileWriter implements DynamicConfigFileWriter {
+public class VelocityConfigFileWriter extends DynamicConfigFileWriter {
/*
* (non-Javadoc)
Added: oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/metlist/MetadataListPcsMetFileWriter.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/metlist/MetadataListPcsMetFileWriter.java?rev=1617057&view=auto
==============================================================================
--- oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/metlist/MetadataListPcsMetFileWriter.java (added)
+++ oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/metlist/MetadataListPcsMetFileWriter.java Sun Aug 10 07:33:37 2014
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.oodt.cas.pge.writers.metlist;
+
+//JDK imports
+import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.*;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.util.Arrays;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+//OODT imports
+import org.apache.oodt.cas.metadata.Metadata;
+import org.apache.oodt.cas.metadata.util.PathUtils;
+import org.apache.oodt.commons.xml.XMLUtils;
+import org.apache.oodt.cas.pge.writers.PcsMetFileWriter;
+
+/**
+ *
+ * @author bfoster
+ * @version $Revision$
+ *
+ * <p>
+ * A {@link PcsMetFileWriter} that generates PCS met files based on a MetList
+ * XML document
+ * </p>.
+ */
+public class MetadataListPcsMetFileWriter extends PcsMetFileWriter {
+
+ @Override
+ protected Metadata getSciPgeSpecificMetadata(File sciPgeCreatedDataFile,
+ Metadata inputMetadata, Object... customArgs) throws Exception {
+ Metadata metadata = new Metadata();
+ for (Object arg : customArgs) {
+ Element root = XMLUtils.getDocumentRoot(
+ new FileInputStream(new File((String) arg)))
+ .getDocumentElement();
+ NodeList metadataNodeList = root.getElementsByTagName(METADATA_TAG);
+ for (int i = 0; i < metadataNodeList.getLength(); i++) {
+ Element metadataElement = (Element) metadataNodeList.item(i);
+ String key = metadataElement.getAttribute(KEY_ATTR);
+ if (key.equals(""))
+ key = PathUtils.doDynamicReplacement(metadataElement.getAttribute(KEY_GEN_ATTR), inputMetadata);
+ String val = metadataElement.getAttribute(VAL_ATTR);
+ if (val.equals(""))
+ val = metadataElement.getTextContent();
+ if (val != null && !val.equals("")) {
+ if (!metadataElement.getAttribute(ENV_REPLACE_ATTR).toLowerCase().equals("false"))
+ val = PathUtils.doDynamicReplacement(val, inputMetadata);
+ String[] vals = null;
+ if (metadataElement.getAttribute(SPLIT_ATTR).toLowerCase().equals("false")) {
+ vals = new String[] { val };
+ } else {
+ String delimiter = metadataElement.getAttribute("delimiter");
+ if (delimiter == null || delimiter.equals(""))
+ delimiter = ",";
+ vals = (val + delimiter).split(delimiter);
+ }
+ metadata.replaceMetadata(key, Arrays.asList(vals));
+ inputMetadata.replaceMetadata(key, Arrays.asList(vals));
+ } else if (inputMetadata.getMetadata(key) != null
+ && !inputMetadata.getMetadata(key).equals("")) {
+ metadata.replaceMetadata(key, inputMetadata
+ .getAllMetadata(key));
+ }
+ }
+ }
+ return metadata;
+ }
+
+}
Added: oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/xslt/XslTransformWriter.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/xslt/XslTransformWriter.java?rev=1617057&view=auto
==============================================================================
--- oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/xslt/XslTransformWriter.java (added)
+++ oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/xslt/XslTransformWriter.java Sun Aug 10 07:33:37 2014
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.oodt.cas.pge.writers.xslt;
+
+//JDK imports
+import java.io.File;
+import java.io.IOException;
+import javax.xml.transform.OutputKeys;
+import javax.xml.transform.Source;
+import javax.xml.transform.Result;
+import javax.xml.transform.stream.StreamSource;
+import javax.xml.transform.stream.StreamResult;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.dom.DOMSource;
+
+//OODT imports
+import org.apache.oodt.cas.metadata.Metadata;
+import org.apache.oodt.cas.metadata.SerializableMetadata;
+import org.apache.oodt.cas.pge.writers.SciPgeConfigFileWriter;
+
+/**
+ *
+ * @author bfoster
+ * @version $Revision$
+ *
+ * <p>
+ * XSL Transformation class which writes Science PGE config files based from the
+ * XML format of SerializableMetadata
+ * </p>.
+ */
+public class XslTransformWriter implements SciPgeConfigFileWriter {
+
+ public File createConfigFile(String sciPgeConfigFilePath,
+ Metadata inputMetadata, Object... customArgs) throws IOException {
+ try {
+ File sciPgeConfigFile = new File(sciPgeConfigFilePath);
+
+ String xsltFilePath = (String) customArgs[0];
+ Source xsltSource = new StreamSource(new File(xsltFilePath));
+ Result result = new StreamResult(sciPgeConfigFile);
+
+ TransformerFactory transFact = TransformerFactory.newInstance();
+ Transformer trans = transFact.newTransformer(xsltSource);
+ boolean useCDATA = customArgs.length > 1 ? ((String) customArgs[1])
+ .toLowerCase().equals("true") : false;
+ Source xmlSource = new DOMSource((new SerializableMetadata(
+ inputMetadata,
+ trans.getOutputProperty(OutputKeys.ENCODING), useCDATA))
+ .toXML());
+
+ trans.setOutputProperty(OutputKeys.INDENT, "yes");
+ trans.transform(xmlSource, result);
+
+ return sciPgeConfigFile;
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw new IOException("Failed to create science PGE config file '"
+ + sciPgeConfigFilePath + "' : " + e.getMessage());
+ }
+ }
+
+}