You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oodt.apache.org by vm...@apache.org on 2015/04/23 17:27:14 UTC
svn commit: r1675624 - in /oodt/trunk: ./ crawler/src/main/resources/
crawler/src/main/resources/extern-scripts/
metadata/src/main/java/org/apache/oodt/cas/metadata/preconditions/
mvn/archetypes/radix/src/main/resources/archetype-resources/crawler/src/...
Author: vmallder
Date: Thu Apr 23 15:27:14 2015
New Revision: 1675624
URL: http://svn.apache.org/r1675624
Log:
Implemented OODT-823 which created a new ExternPreconditionComparator class. Provided example usage as well.
Added:
oodt/trunk/crawler/src/main/resources/extern-scripts/check-data-file-complete.pl (with props)
oodt/trunk/metadata/src/main/java/org/apache/oodt/cas/metadata/preconditions/ExternPreconditionComparator.java
Modified:
oodt/trunk/CHANGES.txt
oodt/trunk/crawler/src/main/resources/precondition-beans.xml
oodt/trunk/mvn/archetypes/radix/src/main/resources/archetype-resources/crawler/src/main/resources/policy/precondition-beans.xml
Modified: oodt/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/oodt/trunk/CHANGES.txt?rev=1675624&r1=1675623&r2=1675624&view=diff
==============================================================================
--- oodt/trunk/CHANGES.txt (original)
+++ oodt/trunk/CHANGES.txt Thu Apr 23 15:27:14 2015
@@ -2,6 +2,8 @@ Apache OODT Change Log
======================
Release 0.9 - Current Development
+* OODT-826 Add the capability to check external preconditions before ingesting a file.
+
* OODT-832 Move streaming items to seperate top-level component (starchmd)
* OODT-831 CAS PGE adapter framework build failed (Aditya Dhulipala via mattmann)
Added: oodt/trunk/crawler/src/main/resources/extern-scripts/check-data-file-complete.pl
URL: http://svn.apache.org/viewvc/oodt/trunk/crawler/src/main/resources/extern-scripts/check-data-file-complete.pl?rev=1675624&view=auto
==============================================================================
--- oodt/trunk/crawler/src/main/resources/extern-scripts/check-data-file-complete.pl (added)
+++ oodt/trunk/crawler/src/main/resources/extern-scripts/check-data-file-complete.pl Thu Apr 23 15:27:14 2015
@@ -0,0 +1,94 @@
+#!/usr/bin/perl -w
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+#limitations under the License.
+
+# This is an example of an extern script that is used for a pre-ingest
+# condition. In this example, if the string 'PARTIAL' or 'INCOMPLETE'
+# are found in
+#
+# Returns: 0 upon success, 1 for failure
+#
+use strict;
+use Env;
+use File::Basename;
+use Time::Piece;
+#
+# Globals
+#
+my $scriptName = $0;
+
+#
+# Function Prototypes
+#
+sub Main();
+
+#
+# Main Program
+#
+Main();
+exit 0; # 0 = success
+
+#
+# Functions
+#
+sub Main()
+{
+
+ # Get the command line argument
+ my $fullSpec = shift(@ARGV);
+
+ #
+ # Check to make sure the input file exists.
+ #
+ unless ( -e $fullSpec ) {
+ print "ERROR: ". $scriptName . ": Input file \[$fullSpec\] Doesn't exist!\n";
+ exit 1;
+ }
+
+ #
+ # Break up the file specification into parts. In this form of the fileparse
+ # command, the file extension includes the '.' and the directory name includes
+ # the '/' at the end.
+ #
+ my ($baseName, $dir, $ext ) = fileparse( $fullSpec , qr/\.[^.]*/);
+ print "INFO: ". $scriptName . ": Processing file: \[$fullSpec\]\n";
+ chop( $dir ); # remove last '/' character.
+
+ #
+ # If the file is a .out science file, check that there is a coresponding .dtl
+ # available to check for the .out file's complete status.
+ #
+ if ( $ext =~ /.*\.out/ ) {
+ my $dtlFile = $fullSpec;
+ $dtlFile =~ s/\.out/\.dtl/;
+ unless ( -e $dtlFile ) {
+ print "ERROR: ". $scriptName . ": .dtl file \[$dtlFile\] Doesn't exist for $baseName$ext!\n";
+ exit 1;
+ }
+ #
+ # If the .dtl file does exist, open it and check for the existence of strings
+ # indicating whether the .out file is a complete file or not.
+ #
+ open DTLFILE, "<$dtlFile";
+ my @dtlInfo = <DTLFILE>;
+ close DTLFILE;
+ if ( grep /DATA_INCOMPLETE/, @dtlInfo or grep /PARTIAL/, @dtlInfo ) {
+ print "INFO: ". $scriptName . ": Input file \[$fullSpec\] Is not a complete file.\n";
+ exit 1;
+ }
+ }
+
+} # End of Main
+
Propchange: oodt/trunk/crawler/src/main/resources/extern-scripts/check-data-file-complete.pl
------------------------------------------------------------------------------
svn:executable = *
Modified: oodt/trunk/crawler/src/main/resources/precondition-beans.xml
URL: http://svn.apache.org/viewvc/oodt/trunk/crawler/src/main/resources/precondition-beans.xml?rev=1675624&r1=1675623&r2=1675624&view=diff
==============================================================================
--- oodt/trunk/crawler/src/main/resources/precondition-beans.xml (original)
+++ oodt/trunk/crawler/src/main/resources/precondition-beans.xml Thu Apr 23 15:27:14 2015
@@ -60,4 +60,13 @@ the License.
<property name="type" value="equal_to" />
<property name="filemgrUrl" value="${crawler.filemgr.url}" />
</bean>
+
+ <bean id="CheckThatDataFileIsComplete" lazy-init="true" class="org.apache.oodt.cas.metadata.preconditions.ExternPreconditionComparator">
+ <property name="description" value="Execute external script to check for file completeness." />
+ <property name="compareItem">
+ <value type="java.lang.Long">0</value>
+ </property>
+ <property name="type" value="equal_to" />
+ <property name="executeCommand" value="check-data-file-complete.pl" />
+ </bean>
</beans>
Added: oodt/trunk/metadata/src/main/java/org/apache/oodt/cas/metadata/preconditions/ExternPreconditionComparator.java
URL: http://svn.apache.org/viewvc/oodt/trunk/metadata/src/main/java/org/apache/oodt/cas/metadata/preconditions/ExternPreconditionComparator.java?rev=1675624&view=auto
==============================================================================
--- oodt/trunk/metadata/src/main/java/org/apache/oodt/cas/metadata/preconditions/ExternPreconditionComparator.java (added)
+++ oodt/trunk/metadata/src/main/java/org/apache/oodt/cas/metadata/preconditions/ExternPreconditionComparator.java Thu Apr 23 15:27:14 2015
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.oodt.cas.metadata.preconditions;
+
+//JDK imports
+import java.io.File;
+import java.io.IOException;
+import java.util.logging.Logger;
+import java.util.logging.Level;
+
+//OODT static imports
+import static org.apache.oodt.cas.metadata.util.PathUtils.doDynamicReplacement;
+
+//OODT imports
+import org.apache.oodt.cas.metadata.exceptions.PreconditionComparatorException;
+import org.apache.oodt.commons.exec.ExecUtils;
+
+
+/**
+ *
+ * @author vmallder
+ * @version $Revision$
+ *
+ * <p>
+ * The pre-condition comparator to use when you have an external script or
+ * application using {@link ExecUtils} that will perform some processing and
+ * return the result to be checked against the compareItem.
+ * </p>.
+ */
+public class ExternPreconditionComparator extends PreConditionComparator<Long> {
+
+ private String executeCommand;
+
+ protected static final Logger LOG = Logger
+ .getLogger( ExternPreconditionComparator.class.getName());
+
+
+ public void setExecuteCommand( String cmd ) {
+ this.executeCommand = cmd;
+ }
+
+ public String getExecuteCommand() {
+ return this.executeCommand;
+ }
+
+ @Override
+ protected int performCheck(File product, Long compareItem)
+ throws PreconditionComparatorException {
+
+ String envReplacedExecuteCommand = "";
+
+ try {
+ envReplacedExecuteCommand = doDynamicReplacement( executeCommand ) +
+ " " + product.getName();
+
+ } catch (Exception e ) {
+ LOG.log(Level.WARNING,
+ "Exception running extern comparator calling doDynamicReplacement with : command ["
+ + executeCommand
+ + "]: Message: " + e.getMessage());
+ return 1;
+ }
+
+ // Determine working directory
+ String workingDirPath = product.getParentFile().getAbsolutePath();
+ File workingDir = new File(workingDirPath);
+ int status = -1;
+
+ try {
+ status = ExecUtils.callProgram(envReplacedExecuteCommand, workingDir);
+
+ } catch (IOException e) {
+ LOG.log(Level.WARNING,
+ "IOException running extern comparator: commandLine: ["
+ + envReplacedExecuteCommand
+ + "]: Message: " + e.getMessage());
+ return 1;
+ }
+
+ return new Long(status).compareTo(compareItem);
+ }
+}
+
Modified: oodt/trunk/mvn/archetypes/radix/src/main/resources/archetype-resources/crawler/src/main/resources/policy/precondition-beans.xml
URL: http://svn.apache.org/viewvc/oodt/trunk/mvn/archetypes/radix/src/main/resources/archetype-resources/crawler/src/main/resources/policy/precondition-beans.xml?rev=1675624&r1=1675623&r2=1675624&view=diff
==============================================================================
--- oodt/trunk/mvn/archetypes/radix/src/main/resources/archetype-resources/crawler/src/main/resources/policy/precondition-beans.xml (original)
+++ oodt/trunk/mvn/archetypes/radix/src/main/resources/archetype-resources/crawler/src/main/resources/policy/precondition-beans.xml Thu Apr 23 15:27:14 2015
@@ -58,4 +58,13 @@ the License.
</property>
<property name="type" value="equal_to" />
</bean>
-</beans>
\ No newline at end of file
+
+ <bean id="CheckThatDataFileIsComplete" lazy-init="true" class="org.apache.oodt.cas.metadata.preconditions.ExternPreconditionComparator">
+ <property name="description" value="Execute external script to check for file completeness." />
+ <property name="compareItem">
+ <value type="java.lang.Long">0</value>
+ </property>
+ <property name="type" value="equal_to" />
+ <property name="executeCommand" value="check-data-file-complete.pl" />
+ </bean>
+</beans>