You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oodt.apache.org by bf...@apache.org on 2012/02/28 00:14:04 UTC
svn commit: r1294389 [1/3] - in /oodt/trunk/crawler: ./ src/main/assembly/
src/main/bin/ src/main/java/org/apache/oodt/cas/crawl/
src/main/java/org/apache/oodt/cas/crawl/action/
src/main/java/org/apache/oodt/cas/crawl/cli/ src/main/java/org/apache/oodt...
Author: bfoster
Date: Mon Feb 27 23:14:03 2012
New Revision: 1294389
URL: http://svn.apache.org/viewvc?rev=1294389&view=rev
Log:
- Integrated CAS-CLI into CAS-Crawler
----------------
OODT-352
Added:
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/cli/
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/cli/action/
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/cli/action/CrawlerLauncherCliAction.java (with props)
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/cli/option/
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/cli/option/handler/
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/cli/option/handler/BeanInfoHandler.java (with props)
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/cli/option/handler/CrawlerActionInfoHandler.java (with props)
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/cli/option/handler/CrawlerBeansPropHandler.java (with props)
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/cli/option/handler/PreconditionInfoHandler.java (with props)
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/util/ActionBeanProperties.java (with props)
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/util/CasPropertyOverrideConfigurer.java (with props)
oodt/trunk/crawler/src/main/resources/cmd-line-actions.xml (with props)
oodt/trunk/crawler/src/main/resources/cmd-line-options.xml (with props)
oodt/trunk/crawler/src/main/resources/legacy-cmd-line-actions.xml (with props)
oodt/trunk/crawler/src/main/resources/legacy-cmd-line-options.xml (with props)
Removed:
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/option/
oodt/trunk/crawler/src/main/resources/cmd-line-option-beans.xml
oodt/trunk/crawler/src/main/resources/examples/JavaPropHandlerXmlFiles/
Modified:
oodt/trunk/crawler/pom.xml
oodt/trunk/crawler/src/main/assembly/assembly.xml
oodt/trunk/crawler/src/main/bin/crawler_launcher
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/CrawlerLauncher.java
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/ProductCrawler.java
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/StdProductCrawler.java
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/CrawlerAction.java
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/CrawlerActionPhases.java
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/CrawlerActionRepo.java
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/DeleteFile.java
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/EmailNotification.java
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/ExternAction.java
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/FileBasedAction.java
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/FileSettings.java
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/FilemgrUniquenessChecker.java
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/GroupAction.java
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/IngestAncillary.java
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/MimeTypeCrawlerAction.java
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/MoveFile.java
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/TernaryAction.java
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/ToggleAction.java
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/WorkflowMgrStatusUpdate.java
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/config/ProductCrawlerBean.java
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/structs/exceptions/CrawlerActionException.java
oodt/trunk/crawler/src/main/resources/action-beans.xml
oodt/trunk/crawler/src/main/resources/crawler-beans.xml
oodt/trunk/crawler/src/main/resources/crawler-config.xml
oodt/trunk/crawler/src/main/resources/precondition-beans.xml
Modified: oodt/trunk/crawler/pom.xml
URL: http://svn.apache.org/viewvc/oodt/trunk/crawler/pom.xml?rev=1294389&r1=1294388&r2=1294389&view=diff
==============================================================================
--- oodt/trunk/crawler/pom.xml (original)
+++ oodt/trunk/crawler/pom.xml Mon Feb 27 23:14:03 2012
@@ -15,186 +15,173 @@ WARRANTIES OR CONDITIONS OF ANY KIND, ei
License for the specific language governing permissions and limitations under
the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.oodt</groupId>
- <artifactId>oodt-core</artifactId>
- <version>0.4-SNAPSHOT</version>
- <relativePath>../core/pom.xml</relativePath>
- </parent>
- <artifactId>cas-crawler</artifactId>
- <name>Catalog and Archive Crawling Framework</name>
- <description>The Catalog and Archive Service Crawling Framework.</description>
- <scm>
- <connection>scm:svn:https://svn.apache.org/repos/asf/oodt/trunk/crawler</connection>
- <developerConnection>scm:svn:https://svn.apache.org/repos/asf/oodt/trunk/crawler</developerConnection>
- <url>http://svn.apache.org/viewvc/oodt/trunk/crawler</url>
- </scm>
- <build>
- <plugins>
- <plugin>
- <artifactId>maven-surefire-plugin</artifactId>
- <version>2.4</version>
- <configuration>
- <forkMode>pertest</forkMode>
- <useSystemClassLoader>false</useSystemClassLoader>
- <systemProperties>
- <property>
- <name>java.util.logging.config.file</name>
- <value>${basedir}/src/testdata/test.logging.properties</value>
- </property>
- </systemProperties>
- <forkedProcessTimeoutInSeconds>0</forkedProcessTimeoutInSeconds>
- <redirectTestOutputToFile>true</redirectTestOutputToFile>
- <includes>
- <include>**/*Test*.java</include>
- </includes>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-assembly-plugin</artifactId>
- <version>2.2-beta-2</version>
- <configuration>
- <descriptors>
- <descriptor>src/main/assembly/assembly.xml</descriptor>
- </descriptors>
- <archive>
- <manifest>
- <mainClass>org.apache.oodt.cas.crawl.CrawlerLauncher</mainClass>
- </manifest>
- </archive>
- </configuration>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- <resources>
- <resource>
- <targetPath>org/apache/oodt/cas/crawl</targetPath>
- <directory>${basedir}/src/main/resources</directory>
- <includes>
- <include>crawler-beans.xml</include>
- <include>crawler-config.xml</include>
- </includes>
- </resource>
- <resource>
- <targetPath>org/apache/oodt/cas/crawl/action</targetPath>
- <directory>${basedir}/src/main/resources</directory>
- <includes>
- <include>action-beans.xml</include>
- </includes>
- </resource>
- <resource>
- <targetPath>org/apache/oodt/cas/crawl/option</targetPath>
- <directory>${basedir}/src/main/resources</directory>
- <includes>
- <include>cmd-line-option-beans.xml</include>
- </includes>
- </resource>
- <resource>
- <targetPath>org/apache/oodt/cas/crawl/comparator</targetPath>
- <directory>${basedir}/src/main/resources</directory>
- <includes>
- <include>precondition-beans.xml</include>
- </includes>
- </resource>
- </resources>
- </build>
- <dependencies>
- <dependency>
- <groupId>org.apache.oodt</groupId>
- <artifactId>oodt-commons</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.oodt</groupId>
- <artifactId>cas-metadata</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.oodt</groupId>
- <artifactId>cas-filemgr</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.oodt</groupId>
- <artifactId>cas-workflow</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- <version>2.1</version>
- </dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- <version>1.1</version>
- </dependency>
- <dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- <version>1.0.3</version>
- </dependency>
- <dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- <version>3.0</version>
- </dependency>
- <dependency>
- <groupId>commons-codec</groupId>
- <artifactId>commons-codec</artifactId>
- <version>1.3</version>
- </dependency>
- <dependency>
- <groupId>commons-collections</groupId>
- <artifactId>commons-collections</artifactId>
- <version>2.1</version>
- </dependency>
- <dependency>
- <groupId>xmlrpc</groupId>
- <artifactId>xmlrpc</artifactId>
- <version>2.0.1</version>
- </dependency>
- <dependency>
- <groupId>org.springframework</groupId>
- <artifactId>spring-core</artifactId>
- <version>2.5.4</version>
- </dependency>
- <dependency>
- <groupId>org.springframework</groupId>
- <artifactId>spring-beans</artifactId>
- <version>2.5.4</version>
- </dependency>
- <dependency>
- <groupId>org.springframework</groupId>
- <artifactId>spring-context</artifactId>
- <version>2.5.4</version>
- </dependency>
- <dependency>
- <groupId>org.apache.geronimo.javamail</groupId>
- <artifactId>geronimo-javamail_1.4_mail</artifactId>
- <version>1.8.1</version>
- </dependency>
- <dependency>
- <groupId>org.apache.geronimo.specs</groupId>
- <artifactId>geronimo-activation_1.1_spec</artifactId>
- <version>1.1</version>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>3.8.2</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.oodt</groupId>
+ <artifactId>oodt-core</artifactId>
+ <version>0.4-SNAPSHOT</version>
+ <relativePath>../core/pom.xml</relativePath>
+ </parent>
+ <artifactId>cas-crawler</artifactId>
+ <name>Catalog and Archive Crawling Framework</name>
+ <description>The Catalog and Archive Service Crawling Framework.</description>
+ <scm>
+ <connection>scm:svn:https://svn.apache.org/repos/asf/oodt/trunk/crawler</connection>
+ <developerConnection>scm:svn:https://svn.apache.org/repos/asf/oodt/trunk/crawler</developerConnection>
+ <url>http://svn.apache.org/viewvc/oodt/trunk/crawler</url>
+ </scm>
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.4</version>
+ <configuration>
+ <forkMode>pertest</forkMode>
+ <useSystemClassLoader>false</useSystemClassLoader>
+ <systemProperties>
+ <property>
+ <name>java.util.logging.config.file</name>
+ <value>${basedir}/src/testdata/test.logging.properties</value>
+ </property>
+ </systemProperties>
+ <forkedProcessTimeoutInSeconds>0</forkedProcessTimeoutInSeconds>
+ <redirectTestOutputToFile>true</redirectTestOutputToFile>
+ <includes>
+ <include>**/*Test*.java</include>
+ </includes>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>2.2-beta-2</version>
+ <configuration>
+ <descriptors>
+ <descriptor>src/main/assembly/assembly.xml</descriptor>
+ </descriptors>
+ <archive>
+ <manifest>
+ <mainClass>org.apache.oodt.cas.crawl.CrawlerLauncher</mainClass>
+ </manifest>
+ </archive>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ <resources>
+ <resource>
+ <targetPath>org/apache/oodt/cas/crawl</targetPath>
+ <directory>${basedir}/src/main/resources</directory>
+ <includes>
+ <include>action-beans.xml</include>
+ <include>crawler-beans.xml</include>
+ <include>crawler-config.xml</include>
+ <include>precondition-beans.xml</include>
+ </includes>
+ </resource>
+ </resources>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.oodt</groupId>
+ <artifactId>oodt-commons</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.oodt</groupId>
+ <artifactId>cas-metadata</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.oodt</groupId>
+ <artifactId>cas-cli</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.oodt</groupId>
+ <artifactId>cas-filemgr</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.oodt</groupId>
+ <artifactId>cas-workflow</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-core</artifactId>
+ <version>2.5.4</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-beans</artifactId>
+ <version>2.5.4</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-context</artifactId>
+ <version>2.5.4</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <version>2.1</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>1.1</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ <version>1.0.3</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-httpclient</groupId>
+ <artifactId>commons-httpclient</artifactId>
+ <version>3.0</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ <version>1.3</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-collections</groupId>
+ <artifactId>commons-collections</artifactId>
+ <version>2.1</version>
+ </dependency>
+ <dependency>
+ <groupId>xmlrpc</groupId>
+ <artifactId>xmlrpc</artifactId>
+ <version>2.0.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.geronimo.javamail</groupId>
+ <artifactId>geronimo-javamail_1.4_mail</artifactId>
+ <version>1.8.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.geronimo.specs</groupId>
+ <artifactId>geronimo-activation_1.1_spec</artifactId>
+ <version>1.1</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>3.8.2</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
</project>
Modified: oodt/trunk/crawler/src/main/assembly/assembly.xml
URL: http://svn.apache.org/viewvc/oodt/trunk/crawler/src/main/assembly/assembly.xml?rev=1294389&r1=1294388&r2=1294389&view=diff
==============================================================================
--- oodt/trunk/crawler/src/main/assembly/assembly.xml (original)
+++ oodt/trunk/crawler/src/main/assembly/assembly.xml Mon Feb 27 23:14:03 2012
@@ -16,70 +16,70 @@ License for the specific language govern
the License.
-->
<assembly>
- <id>dist</id>
- <formats>
- <format>tar.gz</format>
- <format>zip</format>
- </formats>
- <includeBaseDirectory>true</includeBaseDirectory>
- <baseDirectory>${project.artifactId}-${project.version}</baseDirectory>
- <includeSiteDirectory>false</includeSiteDirectory>
- <fileSets>
- <fileSet>
- <directory>${basedir}</directory>
- <outputDirectory>.</outputDirectory>
- <includes>
- <include>LICENSE.txt</include>
- <include>CHANGES.txt</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/bin</directory>
- <outputDirectory>bin</outputDirectory>
- <includes/>
- <fileMode>755</fileMode>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources</directory>
- <outputDirectory>etc</outputDirectory>
- <includes>
- <include>logging.properties</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources</directory>
- <outputDirectory>logs</outputDirectory>
- <includes>
- <include>REMOVE.log</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources/examples</directory>
- <outputDirectory>policy</outputDirectory>
- <includes>
- <include>*.xml</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources</directory>
- <outputDirectory>policy</outputDirectory>
- <includes>
- <include>*.xml</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources/extern-scripts</directory>
- <outputDirectory>policy/extern-scripts</outputDirectory>
- <includes/>
- </fileSet>
- </fileSets>
- <dependencySets>
- <dependencySet>
- <outputDirectory>lib</outputDirectory>
- <unpack>false</unpack>
- <useProjectArtifact>true</useProjectArtifact>
- <useTransitiveDependencies>true</useTransitiveDependencies>
- <unpackOptions/>
- </dependencySet>
- </dependencySets>
+ <id>dist</id>
+ <formats>
+ <format>tar.gz</format>
+ <format>zip</format>
+ </formats>
+ <includeBaseDirectory>true</includeBaseDirectory>
+ <baseDirectory>${project.artifactId}-${project.version}</baseDirectory>
+ <includeSiteDirectory>false</includeSiteDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${basedir}</directory>
+ <outputDirectory>.</outputDirectory>
+ <includes>
+ <include>LICENSE.txt</include>
+ <include>CHANGES.txt</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/bin</directory>
+ <outputDirectory>bin</outputDirectory>
+ <includes />
+ <fileMode>755</fileMode>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources</directory>
+ <outputDirectory>etc</outputDirectory>
+ <includes>
+ <include>logging.properties</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources</directory>
+ <outputDirectory>logs</outputDirectory>
+ <includes>
+ <include>REMOVE.log</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/examples</directory>
+ <outputDirectory>policy</outputDirectory>
+ <includes>
+ <include>*.xml</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources</directory>
+ <outputDirectory>policy</outputDirectory>
+ <includes>
+ <include>*.xml</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/extern-scripts</directory>
+ <outputDirectory>policy/extern-scripts</outputDirectory>
+ <includes />
+ </fileSet>
+ </fileSets>
+ <dependencySets>
+ <dependencySet>
+ <outputDirectory>lib</outputDirectory>
+ <unpack>false</unpack>
+ <useProjectArtifact>true</useProjectArtifact>
+ <useTransitiveDependencies>true</useTransitiveDependencies>
+ <unpackOptions />
+ </dependencySet>
+ </dependencySets>
</assembly>
Modified: oodt/trunk/crawler/src/main/bin/crawler_launcher
URL: http://svn.apache.org/viewvc/oodt/trunk/crawler/src/main/bin/crawler_launcher?rev=1294389&r1=1294388&r2=1294389&view=diff
==============================================================================
--- oodt/trunk/crawler/src/main/bin/crawler_launcher (original)
+++ oodt/trunk/crawler/src/main/bin/crawler_launcher Mon Feb 27 23:14:03 2012
@@ -1,18 +1,33 @@
#!/bin/sh
+#
# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE.txt file distributed with
-# this work for additional information regarding copyright ownership. The ASF
-# licenses this file to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-##########################################################################
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+if [ -z $JAVA_HOME ] ; then
+ JAVA_HOME=/path/to/java/home
+else
+ JAVA_HOME=${JAVA_HOME}
+fi
+
+export JAVA_HOME
-java -Djava.util.logging.config.file=../etc/logging.properties -Djava.ext.dirs=../lib org.apache.oodt.cas.crawl.CrawlerLauncher $*
+$JAVA_HOME/bin/java \
+ -Djava.ext.dirs=../lib \
+ -Djava.util.logging.config.file=../etc/logging.properties \
+ -Dorg.apache.oodt.cas.crawl.bean.repo=../policy/crawler-config.xml \
+ -Dorg.apache.oodt.cas.cli.action.spring.config=../policy/cmd-line-actions.xml \
+ -Dorg.apache.oodt.cas.cli.option.spring.config=../policy/cmd-line-options.xml \
+ org.apache.oodt.cas.crawl.CrawlerLauncher $*
Modified: oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/CrawlerLauncher.java
URL: http://svn.apache.org/viewvc/oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/CrawlerLauncher.java?rev=1294389&r1=1294388&r2=1294389&view=diff
==============================================================================
--- oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/CrawlerLauncher.java (original)
+++ oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/CrawlerLauncher.java Mon Feb 27 23:14:03 2012
@@ -14,79 +14,25 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-
package org.apache.oodt.cas.crawl;
//JDK imports
import java.io.IOException;
-import java.util.List;
-import java.util.logging.Level;
-import java.util.logging.Logger;
//OODT imports
-import org.apache.oodt.commons.option.CmdLineOptionInstance;
-import org.apache.oodt.commons.option.util.CmdLineOptionUtils;
-import org.apache.oodt.cas.crawl.daemon.CrawlDaemon;
-
-//Spring import
-import org.springframework.context.support.FileSystemXmlApplicationContext;
+import org.apache.oodt.cas.cli.CmdLineUtility;
/**
+ * A command line interface to the new Spring enabled crawler.
*
- * @author bfoster
+ * @author bfoster (Brian Foster)
* @version $Revision$
* @since OODT-190
- *
- * <p>
- * A command line interface to the new Spring enabled crawler.
- * </p>
*/
public class CrawlerLauncher {
- private static Logger LOG = Logger.getLogger(CrawlerLauncher.class
- .getName());
-
- private String beanRepo;
-
- public CrawlerLauncher() {
- this(System.getProperty("org.apache.oodt.cas.crawl.bean.repo",
- "classpath:/org/apache/oodt/cas/crawl/crawler-config.xml"));
- }
-
- public CrawlerLauncher(String beanRepo) {
- this.beanRepo = beanRepo;
- }
-
- public void processMain(String[] args) {
- FileSystemXmlApplicationContext appContext = new FileSystemXmlApplicationContext(
- this.beanRepo);
-
- try {
- List<CmdLineOptionInstance> optionInstances = CmdLineOptionUtils
- .loadValidateAndHandleInstances(appContext, args);
-
- ProductCrawler pc = (ProductCrawler) appContext
- .getBean(CmdLineOptionUtils.getOptionValues("crawlerId",
- optionInstances).get(0));
- System.out.println(pc.getFilemgrUrl());
- System.out.println(pc.getId());
-
- pc.setApplicationContext(appContext);
- if (pc.getDaemonPort() != -1 && pc.getDaemonWait() != -1)
- new CrawlDaemon(pc.getDaemonWait(), pc, pc.getDaemonPort())
- .startCrawling();
- else
- pc.crawl();
-
- } catch (Exception e) {
- System.err.println("Failed to parse options : "
- + e.getMessage());
- }
- }
-
- public static void main(String[] args) throws IOException {
- new CrawlerLauncher().processMain(args);
- }
-
+ public static void main(String[] args) throws IOException {
+ CmdLineUtility cmdLineUtility = new CmdLineUtility();
+ cmdLineUtility.run(args);
+ }
}
Modified: oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/ProductCrawler.java
URL: http://svn.apache.org/viewvc/oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/ProductCrawler.java?rev=1294389&r1=1294388&r2=1294389&view=diff
==============================================================================
--- oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/ProductCrawler.java (original)
+++ oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/ProductCrawler.java Mon Feb 27 23:14:03 2012
@@ -14,8 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-
package org.apache.oodt.cas.crawl;
//OODT imports
@@ -40,8 +38,8 @@ import java.util.logging.Level;
import java.util.logging.Logger;
/**
- * @author mattmann
- * @author bfoster
+ * @author mattmann (Chris Mattmann)
+ * @author bfoster (Brian Foster)
* @version $Revision$
*
* <p>
@@ -49,7 +47,6 @@ import java.util.logging.Logger;
* communicate with the file manager and parse met files that show how to ingest
* a particular Product into the File Manager.
* </p>
- *
*/
public abstract class ProductCrawler extends ProductCrawlerBean {
@@ -76,17 +73,19 @@ public abstract class ProductCrawler ext
private Ingester ingester;
public void crawl() {
- this.crawl(new File(this.getProductPath()));
+ crawl(new File(this.getProductPath()));
}
public void crawl(File dirRoot) {
this.ingestStatus = new Vector<IngestStatus>();
// Load actions
- if (this.getApplicationContext() != null)
+ if (this.getApplicationContext() != null) {
(this.actionRepo = new CrawlerActionRepo())
.loadActionsFromBeanFactory(this.getApplicationContext(), this
.getActionIds());
+ validateActions();
+ }
// create ingester
this.ingester = new StdIngester(this.getClientTransferer());
@@ -129,7 +128,21 @@ public abstract class ProductCrawler ext
public List<IngestStatus> getIngestStatus() {
return Collections.unmodifiableList(this.ingestStatus);
}
-
+
+ private void validateActions() {
+ StringBuffer actionErrors = new StringBuffer("");
+ for (CrawlerAction action : actionRepo.getActions()) {
+ try {
+ action.validate();
+ } catch (Exception e) {
+ actionErrors.append(" " + action.getId() + ": " + e.getMessage() + "\n");
+ }
+ }
+ if (actionErrors.length() > 0) {
+ throw new RuntimeException("Actions failed validation:\n" + actionErrors);
+ }
+ }
+
private synchronized boolean containsRequiredMetadata(
Metadata productMetadata) {
for (int i = 0; i < this.getRequiredMetadata().size(); i++) {
@@ -290,5 +303,4 @@ public abstract class ProductCrawler ext
}
return allSucceeded;
}
-
}
Modified: oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/StdProductCrawler.java
URL: http://svn.apache.org/viewvc/oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/StdProductCrawler.java?rev=1294389&r1=1294388&r2=1294389&view=diff
==============================================================================
--- oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/StdProductCrawler.java (original)
+++ oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/StdProductCrawler.java Mon Feb 27 23:14:03 2012
@@ -14,8 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-
package org.apache.oodt.cas.crawl;
//OODT imports
@@ -27,53 +25,50 @@ import java.io.File;
import java.util.logging.Level;
/**
- * @author mattmann
- * @author bfoster
- * @version $Revision$
- *
- * <p>
* A generic Product Crawler for Products. The Crawler is given a root Product
* Path and it searches through all directories and sub-directories for .met
* files, which it uses to determine products to ingest into the file manger.
* The important .met file fields that this crawler requires are:
*
* <ul>
- * <li><code>FileLocation</code>: directory absolute path to location of
- * product file</li>
+ * <li><code>FileLocation</code>: directory absolute path to location of product
+ * file</li>
* <li><code>Filename</code>: name of the product file to ingest</li>
- * <li><code>ProductType</code>: the ProductType that will be sent to the
- * file manager for the product file described by the .met file.</li>
+ * <li><code>ProductType</code>: the ProductType that will be sent to the file
+ * manager for the product file described by the .met file.</li>
* </ul>
*
- * </p>
- *
+ * @author mattmann (Chris Mattmann)
+ * @author bfoster (Brian Foster)
*/
public class StdProductCrawler extends ProductCrawler {
- String metFileExtension;
-
- public StdProductCrawler() {
- this.metFileExtension = "met";
- }
-
- protected Metadata getMetadataForProduct(File product) {
- try {
- MetReaderExtractor extractor = new MetReaderExtractor(this.metFileExtension);
- return extractor.extractMetadata(product);
- } catch (Exception e) {
- LOG.log(Level.WARNING, "Failed to get metadata for " + product
- + " : " + e.getMessage());
- return new Metadata();
- }
- }
-
- protected boolean passesPreconditions(File product) {
- return new File(product.getAbsolutePath() + "." + this.metFileExtension)
- .exists();
- }
-
- public void setMetFileExtension(String metFileExtension) {
- this.metFileExtension = metFileExtension;
- }
+ String metFileExtension;
+ public StdProductCrawler() {
+ this.metFileExtension = "met";
+ }
+
+ @Override
+ protected Metadata getMetadataForProduct(File product) {
+ try {
+ MetReaderExtractor extractor = new MetReaderExtractor(
+ this.metFileExtension);
+ return extractor.extractMetadata(product);
+ } catch (Exception e) {
+ LOG.log(Level.WARNING, "Failed to get metadata for " + product + " : "
+ + e.getMessage());
+ return new Metadata();
+ }
+ }
+
+ @Override
+ protected boolean passesPreconditions(File product) {
+ return new File(product.getAbsolutePath() + "." + this.metFileExtension)
+ .exists();
+ }
+
+ public void setMetFileExtension(String metFileExtension) {
+ this.metFileExtension = metFileExtension;
+ }
}
Modified: oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/CrawlerAction.java
URL: http://svn.apache.org/viewvc/oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/CrawlerAction.java?rev=1294389&r1=1294388&r2=1294389&view=diff
==============================================================================
--- oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/CrawlerAction.java (original)
+++ oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/CrawlerAction.java Mon Feb 27 23:14:03 2012
@@ -15,7 +15,6 @@
* limitations under the License.
*/
-
package org.apache.oodt.cas.crawl.action;
//JDK imports
@@ -24,6 +23,7 @@ import java.util.List;
import java.util.logging.Logger;
//OODT imports
+import org.apache.commons.lang.Validate;
import org.apache.oodt.commons.spring.SpringSetIdInjectionType;
import org.apache.oodt.cas.crawl.structs.exceptions.CrawlerActionException;
import org.apache.oodt.cas.metadata.Metadata;
@@ -32,55 +32,76 @@ import org.apache.oodt.cas.metadata.Meta
import org.springframework.beans.factory.annotation.Required;
/**
- *
- * @author bfoster
- * @author mattmann
- * @version $Revision$
- *
- * <p>
* An action taken by the crawler during success or fail of one of its lifecycle
- * states: preIngest, postIngestSuccess, or postIngestFail
- * </p>.
+ * states: preIngest, postIngestSuccess, or postIngestFail .
+ *
+ * @author bfoster (Brian Foster)
+ * @author mattmann (Chris Mattmann)
*/
public abstract class CrawlerAction implements SpringSetIdInjectionType {
- public List<String> phases;
-
- public static Logger LOG = Logger.getLogger(CrawlerAction.class.getName());
-
- private String description;
-
- private String id;
-
- public CrawlerAction() {
- }
-
- public String getId() {
- return id;
- }
-
- public void setId(String id) {
- this.id = id;
- }
+ public List<String> phases;
- @Required
- public void setPhases(List<String> phases) {
- this.phases = phases;
- }
+ public static Logger LOG = Logger.getLogger(CrawlerAction.class.getName());
- public List<String> getPhases() {
- return this.phases;
- }
+ private String description;
- public void setDescription(String description) {
- this.description = description;
- }
+ private String id;
- public String getDescription() {
- return this.description;
- }
+ public CrawlerAction() {
+ }
+
+ @Override
+ public String getId() {
+ return id;
+ }
+
+ @Override
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ @Required
+ public void setPhases(List<String> phases) {
+ this.phases = phases;
+ }
+
+ public List<String> getPhases() {
+ return this.phases;
+ }
+
+ public void setDescription(String description) {
+ this.description = description;
+ }
+
+ public String getDescription() {
+ return this.description;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof CrawlerAction) {
+ return ((CrawlerAction) obj).id.equals(id);
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return id.hashCode();
+ }
+
+ public void validate() throws CrawlerActionException {
+ try {
+ Validate.notNull(id, "Must specify id");
+ Validate.notNull(phases, "Must specify phases");
+ } catch (Exception e) {
+ throw new CrawlerActionException(e);
+ }
+ }
- public abstract boolean performAction(File product, Metadata productMetadata)
- throws CrawlerActionException;
+ public abstract boolean performAction(File product, Metadata productMetadata)
+ throws CrawlerActionException;
}
Modified: oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/CrawlerActionPhases.java
URL: http://svn.apache.org/viewvc/oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/CrawlerActionPhases.java?rev=1294389&r1=1294388&r2=1294389&view=diff
==============================================================================
--- oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/CrawlerActionPhases.java (original)
+++ oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/CrawlerActionPhases.java Mon Feb 27 23:14:03 2012
@@ -14,24 +14,35 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-
package org.apache.oodt.cas.crawl.action;
/**
- * @author mattmann
- * @version $Revision$
- *
- * <p>
- * Met keys defining the 3 key Crawler Phases
- * </p>.
+ * Supported Crawler Phases.
+ *
+ * @author mattmann (Chris Mattmann)
+ * @author bfoster (Brian Foster)
*/
-public interface CrawlerActionPhases {
-
- public static final String PRE_INGEST = "preIngest";
-
- public static final String POST_INGEST_SUCCESS = "postIngestSuccess";
-
- public static final String POST_INGEST_FAILURE = "postIngestFailure";
-
+public enum CrawlerActionPhases {
+ PRE_INGEST("preIngest"),
+ POST_INGEST_SUCCESS("postIngestSuccess"),
+ POST_INGEST_FAILURE("postIngestFailure");
+
+ private String name;
+
+ private CrawlerActionPhases(String name) {
+ this.name = name;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public static CrawlerActionPhases getPhaseByName(String name) {
+ for (CrawlerActionPhases phase : CrawlerActionPhases.values()) {
+ if (phase.getName().equals(name)) {
+ return phase;
+ }
+ }
+ return null;
+ }
}
Modified: oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/CrawlerActionRepo.java
URL: http://svn.apache.org/viewvc/oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/CrawlerActionRepo.java?rev=1294389&r1=1294388&r2=1294389&view=diff
==============================================================================
--- oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/CrawlerActionRepo.java (original)
+++ oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/CrawlerActionRepo.java Mon Feb 27 23:14:03 2012
@@ -14,69 +14,79 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-
package org.apache.oodt.cas.crawl.action;
//JDK imports
import java.util.LinkedList;
import java.util.List;
+import java.util.Set;
//Spring imports
import org.springframework.context.ApplicationContext;
+import com.google.common.collect.Sets;
+
/**
- *
- * @author bfoster
- * @author mattmann
- * @version $Revision$
- *
- * <p>
* A repository and reader for {@link CrawlerAction}s associated with Crawler
- * lifecycle phases: preIngest, postIngestSuccess and postIngestFail
- * </p>.
+ * lifecycle phases: preIngest, postIngestSuccess and postIngestFail.
+ *
+ * @author bfoster (Brian Foster)
+ * @author mattmann (Chris Mattmann)
*/
-public class CrawlerActionRepo implements CrawlerActionPhases {
-
- LinkedList<CrawlerAction> preIngestActions;
-
- LinkedList<CrawlerAction> postIngestOnFailActions;
+public class CrawlerActionRepo {
- LinkedList<CrawlerAction> postIngestOnSuccessActions;
-
- public CrawlerActionRepo() {
- this.preIngestActions = new LinkedList<CrawlerAction>();
- this.postIngestOnFailActions = new LinkedList<CrawlerAction>();
- this.postIngestOnSuccessActions = new LinkedList<CrawlerAction>();
- }
-
- public List<CrawlerAction> getPreIngestActions() {
- return this.preIngestActions;
- }
-
- public List<CrawlerAction> getPostIngestOnFailActions() {
- return this.postIngestOnFailActions;
- }
-
- public List<CrawlerAction> getPostIngestOnSuccessActions() {
- return this.postIngestOnSuccessActions;
- }
-
- public void loadActionsFromBeanFactory(ApplicationContext context,
- List<String> actionIds) {
- for (String actionId : actionIds) {
- CrawlerAction action = ((CrawlerAction) context.getBean(actionId,
- CrawlerAction.class));
- List<String> phases = action.getPhases();
- for (String phase : phases) {
- if (phase.equals(PRE_INGEST))
- this.preIngestActions.add(action);
- else if (phase.equals(POST_INGEST_SUCCESS))
- this.postIngestOnSuccessActions.add(action);
- else if (phase.equals(POST_INGEST_FAILURE))
- this.postIngestOnFailActions.add(action);
+ private LinkedList<CrawlerAction> preIngestActions;
+ private LinkedList<CrawlerAction> postIngestOnFailActions;
+ private LinkedList<CrawlerAction> postIngestOnSuccessActions;
+
+ public CrawlerActionRepo() {
+ this.preIngestActions = new LinkedList<CrawlerAction>();
+ this.postIngestOnFailActions = new LinkedList<CrawlerAction>();
+ this.postIngestOnSuccessActions = new LinkedList<CrawlerAction>();
+ }
+
+ public Set<CrawlerAction> getActions() {
+ Set<CrawlerAction> actions = Sets.newHashSet();
+ actions.addAll(preIngestActions);
+ actions.addAll(postIngestOnFailActions);
+ actions.addAll(postIngestOnSuccessActions);
+ return actions;
+ }
+
+ public List<CrawlerAction> getPreIngestActions() {
+ return this.preIngestActions;
+ }
+
+ public List<CrawlerAction> getPostIngestOnFailActions() {
+ return this.postIngestOnFailActions;
+ }
+
+ public List<CrawlerAction> getPostIngestOnSuccessActions() {
+ return this.postIngestOnSuccessActions;
+ }
+
+ public void loadActionsFromBeanFactory(ApplicationContext context,
+ List<String> actionIds) {
+ for (String actionId : actionIds) {
+ CrawlerAction action = ((CrawlerAction) context.getBean(actionId,
+ CrawlerAction.class));
+ List<String> phases = action.getPhases();
+ for (String phase : phases) {
+ switch (CrawlerActionPhases.getPhaseByName(phase)) {
+ case PRE_INGEST:
+ preIngestActions.add(action);
+ break;
+ case POST_INGEST_SUCCESS:
+ postIngestOnSuccessActions.add(action);
+ break;
+ case POST_INGEST_FAILURE:
+ postIngestOnFailActions.add(action);
+ break;
+ default:
+ throw new RuntimeException("Phase '" + phase
+ + "' is not supported");
}
- }
- }
-
+ }
+ }
+ }
}
Modified: oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/DeleteFile.java
URL: http://svn.apache.org/viewvc/oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/DeleteFile.java?rev=1294389&r1=1294388&r2=1294389&view=diff
==============================================================================
--- oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/DeleteFile.java (original)
+++ oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/DeleteFile.java Mon Feb 27 23:14:03 2012
@@ -14,62 +14,66 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-
package org.apache.oodt.cas.crawl.action;
//JDK imports
import java.io.File;
import java.util.logging.Level;
+//Apache imports
+import org.apache.commons.lang.Validate;
+
//OODT imports
import org.apache.oodt.cas.crawl.structs.exceptions.CrawlerActionException;
import org.apache.oodt.cas.metadata.Metadata;
/**
+ * Deletes a product file as a {@link CrawlerAction} response.
*
- * @author bfoster
- * @author mattmann
- * @version $Revision$
- *
- * <p>
- * Deletes a product file as a {@link CrawlerAction} reponse
- * </p>.
+ * @author bfoster (Brian Foster)
+ * @author mattmann (Chris Mattmann)
*/
public class DeleteFile extends CrawlerAction {
- private String file;
-
- private String fileExtension;
-
- public DeleteFile() {
- }
-
- public boolean performAction(File product, Metadata productMetadata)
- throws CrawlerActionException {
- File fileToDelete = product;
- try {
- if (file != null)
- fileToDelete = new File(file);
- else if (fileExtension != null)
- fileToDelete = new File(fileToDelete.getAbsolutePath() + "."
- + fileExtension);
-
- LOG.log(Level.INFO, "Deleting file "
- + fileToDelete.getAbsolutePath());
- return fileToDelete.delete();
- } catch (Exception e) {
- throw new CrawlerActionException("Error while deleting file "
- + fileToDelete + " : " + e.getMessage());
- }
- }
-
- public void setFile(String file) {
- this.file = file;
- }
-
- public void setFileExtension(String fileExtension) {
- this.fileExtension = fileExtension;
- }
+ private File file;
+ private String fileExtension;
+ @Override
+ public boolean performAction(File product, Metadata productMetadata)
+ throws CrawlerActionException {
+ try {
+ File fileToDelete = product;
+ if (file != null) {
+ fileToDelete = file;
+ } else if (fileExtension != null) {
+ fileToDelete = new File(product.getAbsolutePath() + "." + fileExtension);
+ }
+
+ LOG.log(Level.INFO, "Deleting file " + fileToDelete.getAbsolutePath());
+ return fileToDelete.delete();
+ } catch (Exception e) {
+ LOG.log(Level.SEVERE, "Error while deleting file for product '"
+ + product + "' : " + e.getMessage(), e);
+ return false;
+ }
+ }
+
+ @Override
+ public void validate() throws CrawlerActionException {
+ super.validate();
+ try {
+ Validate.isTrue(file == null || fileExtension == null,
+ "Must specify either file or fileExtension");
+ } catch (Exception e) {
+ throw new CrawlerActionException(e);
+ }
+ }
+
+ public void setFile(File file) {
+ this.file = file;
+ }
+
+ public void setFileExtension(String fileExtension) {
+ this.fileExtension = fileExtension;
+ }
}
Modified: oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/EmailNotification.java
URL: http://svn.apache.org/viewvc/oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/EmailNotification.java?rev=1294389&r1=1294388&r2=1294389&view=diff
==============================================================================
--- oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/EmailNotification.java (original)
+++ oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/EmailNotification.java Mon Feb 27 23:14:03 2012
@@ -14,7 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.oodt.cas.crawl.action;
//JDK imports
@@ -30,6 +29,9 @@ import javax.mail.internet.MimeMessage;
//Spring imports
import org.springframework.beans.factory.annotation.Required;
+//Apache imports
+import org.apache.commons.lang.Validate;
+
//OODT imports
import org.apache.oodt.cas.crawl.action.CrawlerAction;
import org.apache.oodt.cas.crawl.structs.exceptions.CrawlerActionException;
@@ -41,105 +43,118 @@ import org.apache.oodt.cas.metadata.util
* property replacement on the subject, recipients, and message. This allows the
* notifications to be dynamically generated.
*
- * @author pramirez
- * @author mattmann
+ * @author pramirez (Paul Ramirez)
+ * @author mattmann (Chris Mattmann)
*
*/
public class EmailNotification extends CrawlerAction {
- private String mailHost;
- private String subject;
- private String message;
- private String recipients;
- private String sender;
- private boolean ignoreInvalidAddresses;
-
- public EmailNotification() {
- this.ignoreInvalidAddresses = true;
- }
-
- @Override
- public boolean performAction(File product, Metadata metadata)
- throws CrawlerActionException {
- Properties props = new Properties();
- props.put("mail.host", this.mailHost);
- props.put("mail.transport.protocol", "smtp");
- props.put("mail.from", this.sender);
-
- Session session = Session.getDefaultInstance(props);
- Message msg = new MimeMessage(session);
- try {
- msg.setSubject(PathUtils.doDynamicReplacement(subject, metadata));
- msg.setText(new String(PathUtils.doDynamicReplacement(message, metadata)
- .getBytes()));
- String[] recips = recipients.split(",");
- for (String recipient : recips) {
- try {
- msg.addRecipient(Message.RecipientType.TO, new InternetAddress(
- PathUtils.replaceEnvVariables(recipient.trim(), metadata),
- ignoreInvalidAddresses));
- LOG.fine("Recipient: "
- + PathUtils.replaceEnvVariables(recipient.trim(), metadata));
- } catch (AddressException ae) {
- LOG.fine("Recipient: "
- + PathUtils.replaceEnvVariables(recipient.trim(), metadata));
- LOG.warning(ae.getMessage());
- }
+ private String mailHost;
+ private String subject;
+ private String message;
+ private String recipients;
+ private String sender;
+ private boolean ignoreInvalidAddresses;
+
+ public EmailNotification() {
+ this.ignoreInvalidAddresses = true;
+ }
+
+ @Override
+ public boolean performAction(File product, Metadata metadata)
+ throws CrawlerActionException {
+ try {
+ Properties props = new Properties();
+ props.put("mail.host", this.mailHost);
+ props.put("mail.transport.protocol", "smtp");
+ props.put("mail.from", this.sender);
+
+ Session session = Session.getDefaultInstance(props);
+ Message msg = new MimeMessage(session);
+ msg.setSubject(PathUtils.doDynamicReplacement(subject, metadata));
+ msg.setText(new String(PathUtils.doDynamicReplacement(message,
+ metadata).getBytes()));
+ String[] recips = recipients.split(",");
+ for (String recipient : recips) {
+ try {
+ msg.addRecipient(Message.RecipientType.TO, new InternetAddress(
+ PathUtils.replaceEnvVariables(recipient.trim(), metadata),
+ ignoreInvalidAddresses));
+ LOG.fine("Recipient: "
+ + PathUtils.replaceEnvVariables(recipient.trim(), metadata));
+ } catch (AddressException ae) {
+ LOG.fine("Recipient: "
+ + PathUtils.replaceEnvVariables(recipient.trim(), metadata));
+ LOG.warning(ae.getMessage());
+ }
+ }
+ LOG.fine("Subject: " + msg.getSubject());
+ LOG.fine("Message: "
+ + new String(PathUtils.doDynamicReplacement(message, metadata)
+ .getBytes()));
+ Transport.send(msg);
+ return true;
+ } catch (Exception e) {
+ LOG.severe(e.getMessage());
+ return false;
+ }
+ }
+
+ @Override
+ public void validate() throws CrawlerActionException {
+ super.validate();
+ try {
+ Validate.notNull(mailHost, "Must specify mailHost");
+ Validate.notNull(subject, "Must specify subject");
+ Validate.notNull(message, "Must specify message");
+ Validate.notNull(recipients, "Must specify recipients");
+ Validate.notNull(sender, "Must specify sender");
+ } catch (Exception e) {
+ throw new CrawlerActionException(e);
+ }
+ }
+
+ @Required
+ public void setMailHost(String smtpHost) {
+ this.mailHost = smtpHost;
+ }
+
+ @Required
+ public void setSubject(String subject) {
+ this.subject = subject;
+ }
+
+ @Required
+ public void setMessage(String message) {
+ this.message = message;
+ }
+
+ @Required
+ public void setRecipients(String recipients) {
+ this.recipients = recipients;
+ }
+
+ @Required
+ public void setSender(String sender) {
+ this.sender = sender;
+ }
+
+ public void setIgnoreInvalidAddresses(Boolean ignoreInvalidAddresses) {
+ this.ignoreInvalidAddresses = ignoreInvalidAddresses;
+ }
+
+ public static void main(String[] args) throws Exception {
+ if (args.length != 5) {
+ System.out.println("Usage: java " + EmailNotification.class.getName()
+ + " <mailhost> <sender> <recipients> <subject> <message>");
+ System.exit(-1);
}
- LOG.fine("Subject: " + msg.getSubject());
- LOG.fine("Message: "
- + new String(PathUtils.doDynamicReplacement(message, metadata)
- .getBytes()));
- Transport.send(msg);
- } catch (Exception e) {
- LOG.severe(e.getMessage());
- return false;
- }
-
- return true;
- }
-
- @Required
- public void setMailHost(String smtpHost) {
- this.mailHost = smtpHost;
- }
-
- @Required
- public void setSubject(String subject) {
- this.subject = subject;
- }
-
- @Required
- public void setMessage(String message) {
- this.message = message;
- }
-
- @Required
- public void setRecipients(String recipients) {
- this.recipients = recipients;
- }
-
- @Required
- public void setSender(String sender) {
- this.sender = sender;
- }
-
- public void setIgnoreInvalidAddresses(Boolean ignoreInvalidAddresses) {
- this.ignoreInvalidAddresses = ignoreInvalidAddresses;
- }
-
- public static void main(String[] args) throws Exception {
- if (args.length != 5) {
- System.out.println("Usage: java " + EmailNotification.class.getName()
- + " <mailhost> <sender> <recipients> <subject> <message>");
- System.exit(-1);
- }
- EmailNotification notification = new EmailNotification();
- notification.setMailHost(args[0]);
- notification.setSender(args[1]);
- notification.setRecipients(args[2]);
- notification.setSubject(args[3]);
- notification.setMessage(args[4]);
- notification.performAction(new File(""), new Metadata());
- }
+ EmailNotification notification = new EmailNotification();
+ notification.setMailHost(args[0]);
+ notification.setSender(args[1]);
+ notification.setRecipients(args[2]);
+ notification.setSubject(args[3]);
+ notification.setMessage(args[4]);
+ notification.performAction(new File(""), new Metadata());
+ }
}
Modified: oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/ExternAction.java
URL: http://svn.apache.org/viewvc/oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/ExternAction.java?rev=1294389&r1=1294388&r2=1294389&view=diff
==============================================================================
--- oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/ExternAction.java (original)
+++ oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/ExternAction.java Mon Feb 27 23:14:03 2012
@@ -16,50 +16,64 @@
*/
package org.apache.oodt.cas.crawl.action;
+//OODT static imports
+import static org.apache.oodt.cas.metadata.util.PathUtils.doDynamicReplacement;
+
//JDK imports
import java.io.File;
import java.util.logging.Level;
+//Apache imports
+import org.apache.commons.lang.Validate;
+
//OODT imports
import org.apache.oodt.cas.crawl.structs.exceptions.CrawlerActionException;
import org.apache.oodt.cas.metadata.Metadata;
-import org.apache.oodt.cas.metadata.util.PathUtils;
import org.apache.oodt.commons.exec.ExecUtils;
/**
+ * Execute some external command as a {@link CrawlerAction} response.
*
- * @author bfoster
- * @version $Revision$
- *
- * <p>
- * Execute some external command as a {@link CrawlerAction} reponse
- * </p>.
+ * @author bfoster (Brian Foster)
*/
public class ExternAction extends CrawlerAction {
- private String executeCommand;
- private String workingDir;
-
- @Override
- public boolean performAction(File product, Metadata productMetadata)
- throws CrawlerActionException {
- String currentExcecuteCommand = this.executeCommand;
- try {
- if (currentExcecuteCommand == null)
- throw new Exception("Must specify execute command");
- return ExecUtils.callProgram(currentExcecuteCommand = PathUtils.doDynamicReplacement(currentExcecuteCommand, productMetadata), LOG, new File(workingDir != null ? workingDir : product.getParent())) == 0;
- }catch (Exception e) {
- LOG.log(Level.SEVERE, "Failed to execute extern command '" + currentExcecuteCommand + "' : " + e.getMessage(), e);
- return false;
- }
- }
-
- public void setExecuteCommand(String executeCommand) {
- this.executeCommand = executeCommand;
- }
-
- public void setWorkingDir(String workingDir) throws Exception {
- this.workingDir = PathUtils.doDynamicReplacement(workingDir);
- }
+ private String executeCommand;
+ private String workingDir;
+
+ @Override
+ public boolean performAction(File product, Metadata productMetadata)
+ throws CrawlerActionException {
+ try {
+ String envReplacedExecuteCommand = doDynamicReplacement(
+ executeCommand, productMetadata);
+ return ExecUtils.callProgram(
+ envReplacedExecuteCommand,
+ LOG,
+ new File(workingDir != null ? doDynamicReplacement(workingDir,
+ productMetadata) : product.getParent())) == 0;
+ } catch (Exception e) {
+ LOG.log(Level.SEVERE, "Failed to execute extern command '"
+ + executeCommand + "' : " + e.getMessage(), e);
+ return false;
+ }
+ }
+
+ @Override
+ public void validate() throws CrawlerActionException {
+ super.validate();
+ try {
+ Validate.notNull(executeCommand, "Must specify executeCommand");
+ } catch (Exception e) {
+ throw new CrawlerActionException(e);
+ }
+ }
+
+ public void setExecuteCommand(String executeCommand) {
+ this.executeCommand = executeCommand;
+ }
+ public void setWorkingDir(String workingDir) throws Exception {
+ this.workingDir = workingDir;
+ }
}
Modified: oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/FileBasedAction.java
URL: http://svn.apache.org/viewvc/oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/FileBasedAction.java?rev=1294389&r1=1294388&r2=1294389&view=diff
==============================================================================
--- oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/FileBasedAction.java (original)
+++ oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/FileBasedAction.java Mon Feb 27 23:14:03 2012
@@ -14,7 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.oodt.cas.crawl.action;
//JDK imports
@@ -33,82 +32,82 @@ import org.apache.oodt.cas.metadata.Meta
* Finally, it captures a flag for indicating whether the action should fail
* should the file not exist.
*
- * @author pramirez
- * @author mattmann
+ * @author pramirez (Paul Ramirez)
+ * @author mattmann (Chris Mattmann)
*
*/
public abstract class FileBasedAction extends CrawlerAction {
- protected String file;
- protected String fileKey;
- protected FileSettings fileSettings;
- protected boolean failMissingFile;
-
- public FileBasedAction() {
- fileSettings = new FileSettings();
- this.failMissingFile = false;
- }
-
- public void setFilePrefix(String filePrefix) {
- fileSettings.setFilePrefix(filePrefix);
- }
-
- public void setFileSuffix(String fileSuffix) {
- fileSettings.setFileSuffix(fileSuffix);
- }
-
- public void setFileExtension(String fileExtension) {
- fileSettings.setFileExtension(fileExtension);
- }
-
- public void setKeepExistingExtension(boolean keepExistingExtension) {
- fileSettings.setKeepExistingExtension(keepExistingExtension);
- }
-
- public void setFile(String file) {
- this.file = file;
- }
-
- public void setFileKey(String fileKey) {
- this.fileKey = fileKey;
- }
-
- public void setFailMissingFile(boolean failMissingFile) {
- this.failMissingFile = failMissingFile;
- }
-
- public File getSelectedFile(File product, Metadata metadata) {
- File selectedFile = null;
- if (file == null && fileKey == null) {
- selectedFile = product;
- } else if (file != null) {
- selectedFile = new File(file);
- } else {
- selectedFile = new File(metadata.getMetadata(fileKey));
- }
- return selectedFile;
- }
-
- public boolean performAction(File product, Metadata metadata)
- throws CrawlerActionException {
- File selectedFile = this.getSelectedFile(product, metadata);
- String selectedFileString = this.fileSettings
- .getPreparedFileString(selectedFile);
- File actionFile = new File(selectedFileString);
-
- if (failMissingFile && !actionFile.exists()) {
- LOG.severe("File does not exist: " + actionFile.getAbsolutePath());
- return false;
- }
-
- if (actionFile.exists()) {
- return this.performFileAction(actionFile, metadata);
- } else {
- LOG.fine("File does not exist: " + actionFile.getAbsolutePath());
- }
+ protected String file;
+ protected String fileKey;
+ protected FileSettings fileSettings;
+ protected boolean failMissingFile;
+
+ public FileBasedAction() {
+ fileSettings = new FileSettings();
+ this.failMissingFile = false;
+ }
+
+ public void setFilePrefix(String filePrefix) {
+ fileSettings.setFilePrefix(filePrefix);
+ }
+
+ public void setFileSuffix(String fileSuffix) {
+ fileSettings.setFileSuffix(fileSuffix);
+ }
+
+ public void setFileExtension(String fileExtension) {
+ fileSettings.setFileExtension(fileExtension);
+ }
+
+ public void setKeepExistingExtension(boolean keepExistingExtension) {
+ fileSettings.setKeepExistingExtension(keepExistingExtension);
+ }
+
+ public void setFile(String file) {
+ this.file = file;
+ }
+
+ public void setFileKey(String fileKey) {
+ this.fileKey = fileKey;
+ }
+
+ public void setFailMissingFile(boolean failMissingFile) {
+ this.failMissingFile = failMissingFile;
+ }
+
+ public File getSelectedFile(File product, Metadata metadata) {
+ File selectedFile = null;
+ if (file == null && fileKey == null) {
+ selectedFile = product;
+ } else if (file != null) {
+ selectedFile = new File(file);
+ } else {
+ selectedFile = new File(metadata.getMetadata(fileKey));
+ }
+ return selectedFile;
+ }
+
+ public boolean performAction(File product, Metadata metadata)
+ throws CrawlerActionException {
+ File selectedFile = this.getSelectedFile(product, metadata);
+ String selectedFileString = this.fileSettings
+ .getPreparedFileString(selectedFile);
+ File actionFile = new File(selectedFileString);
+
+ if (failMissingFile && !actionFile.exists()) {
+ LOG.severe("File does not exist: " + actionFile.getAbsolutePath());
+ return false;
+ }
+
+ if (actionFile.exists()) {
+ return this.performFileAction(actionFile, metadata);
+ } else {
+ LOG.fine("File does not exist: " + actionFile.getAbsolutePath());
+ }
- return true;
- }
+ return true;
+ }
- public abstract boolean performFileAction(File actionFile, Metadata metadata)
- throws CrawlerActionException;
+ public abstract boolean performFileAction(File actionFile, Metadata metadata)
+ throws CrawlerActionException;
}
Modified: oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/FileSettings.java
URL: http://svn.apache.org/viewvc/oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/FileSettings.java?rev=1294389&r1=1294388&r2=1294389&view=diff
==============================================================================
--- oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/FileSettings.java (original)
+++ oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/FileSettings.java Mon Feb 27 23:14:03 2012
@@ -14,7 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.oodt.cas.crawl.action;
//JDK imports
@@ -29,72 +28,72 @@ import java.io.File;
* fileExtension="png", and keepExistingExtension=true. This conversion could
* take place by passing calling the getPreparedFileString method.
*
- * @author pramirez
- * @author mattmann
+ * @author pramirez (Paul Ramirez)
+ * @author mattmann (Chris Mattmann)
*
*/
public class FileSettings {
- private String filePrefix;
- private String fileSuffix;
- private String fileExtension;
- private boolean keepExistingExtension;
-
- public FileSettings() {
- this.keepExistingExtension = true;
- }
-
- public void setFilePrefix(String filePrefix) {
- this.filePrefix = filePrefix;
- }
-
- public void setFileSuffix(String fileSuffix) {
- this.fileSuffix = fileSuffix;
- }
-
- public void setFileExtension(String fileExtension) {
- this.fileExtension = fileExtension;
- }
-
- public void setKeepExistingExtension(boolean keepExistingExtension) {
- this.keepExistingExtension = keepExistingExtension;
- }
-
- public String getPreparedFileString(File file) {
- StringBuffer fileString = new StringBuffer();
-
- if (file.getParent() != null) {
- fileString.append(file.getParent());
- fileString.append(System.getProperties().getProperty("file.separator",
- "/"));
- }
-
- if (filePrefix != null) {
- fileString.append(filePrefix);
- }
-
- String existingExtension = "";
- String filename = file.getName();
- int existingIndex = filename.lastIndexOf(".");
- if (existingIndex != -1) {
- existingExtension = filename.substring(existingIndex);
- filename = filename.substring(0, existingIndex);
- }
-
- fileString.append(filename);
-
- if (fileSuffix != null) {
- fileString.append(fileSuffix);
- }
-
- if (keepExistingExtension) {
- fileString.append(existingExtension);
- }
-
- if (fileExtension != null) {
- fileString.append(".");
- fileString.append(fileExtension);
- }
+ private String filePrefix;
+ private String fileSuffix;
+ private String fileExtension;
+ private boolean keepExistingExtension;
+
+ public FileSettings() {
+ this.keepExistingExtension = true;
+ }
+
+ public void setFilePrefix(String filePrefix) {
+ this.filePrefix = filePrefix;
+ }
+
+ public void setFileSuffix(String fileSuffix) {
+ this.fileSuffix = fileSuffix;
+ }
+
+ public void setFileExtension(String fileExtension) {
+ this.fileExtension = fileExtension;
+ }
+
+ public void setKeepExistingExtension(boolean keepExistingExtension) {
+ this.keepExistingExtension = keepExistingExtension;
+ }
+
+ public String getPreparedFileString(File file) {
+ StringBuffer fileString = new StringBuffer();
+
+ if (file.getParent() != null) {
+ fileString.append(file.getParent());
+ fileString.append(System.getProperties().getProperty("file.separator",
+ "/"));
+ }
+
+ if (filePrefix != null) {
+ fileString.append(filePrefix);
+ }
+
+ String existingExtension = "";
+ String filename = file.getName();
+ int existingIndex = filename.lastIndexOf(".");
+ if (existingIndex != -1) {
+ existingExtension = filename.substring(existingIndex);
+ filename = filename.substring(0, existingIndex);
+ }
+
+ fileString.append(filename);
+
+ if (fileSuffix != null) {
+ fileString.append(fileSuffix);
+ }
+
+ if (keepExistingExtension) {
+ fileString.append(existingExtension);
+ }
+
+ if (fileExtension != null) {
+ fileString.append(".");
+ fileString.append(fileExtension);
+ }
- return fileString.toString();
- }
+ return fileString.toString();
+ }
}
Modified: oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/FilemgrUniquenessChecker.java
URL: http://svn.apache.org/viewvc/oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/FilemgrUniquenessChecker.java?rev=1294389&r1=1294388&r2=1294389&view=diff
==============================================================================
--- oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/FilemgrUniquenessChecker.java (original)
+++ oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/FilemgrUniquenessChecker.java Mon Feb 27 23:14:03 2012
@@ -14,61 +14,59 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-
package org.apache.oodt.cas.crawl.action;
+//OODT static imports
+import static org.apache.oodt.cas.filemgr.metadata.CoreMetKeys.PRODUCT_NAME;
+
//JDK imports
import java.io.File;
import java.net.URL;
+//Apache imports
+import org.apache.commons.lang.Validate;
+
//OODT imports
import org.apache.oodt.cas.crawl.structs.exceptions.CrawlerActionException;
-import org.apache.oodt.cas.filemgr.metadata.CoreMetKeys;
import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
import org.apache.oodt.cas.metadata.Metadata;
-import org.springframework.beans.factory.annotation.Required;
-
/**
- *
- * @author bfoster
- * @version $Revision$
- *
- * <p>
* Check whether a product exists in the database already
- * </p>.
+ *
+ * @author bfoster (Brian Foster)
*/
public class FilemgrUniquenessChecker extends CrawlerAction {
- private String filemgrUrl;
-
- public FilemgrUniquenessChecker() {}
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.oodt.cas.crawl.action.CrawlerAction#performAction(java.io.File,
- * org.apache.oodt.cas.metadata.Metadata)
- */
- public boolean performAction(File product, Metadata productMetadata)
- throws CrawlerActionException {
- try {
- URL filemgrURL = new URL(this.filemgrUrl);
- XmlRpcFileManagerClient fmClient = new XmlRpcFileManagerClient(
- filemgrURL);
- return !fmClient.hasProduct(productMetadata
- .getMetadata(CoreMetKeys.PRODUCT_NAME));
- } catch (Exception e) {
- throw new CrawlerActionException(
- "Product failed uniqueness check : [" + product + "] : "
- + e.getMessage());
- }
- }
-
- @Required
- public void setFilemgrUrl(String filemgrUrl) {
- this.filemgrUrl = filemgrUrl;
- }
+ private String filemgrUrl;
+ @Override
+ public boolean performAction(File product, Metadata productMetadata)
+ throws CrawlerActionException {
+ try {
+ Validate.notNull(productMetadata.getMetadata(PRODUCT_NAME),
+ PRODUCT_NAME + " was not found in metadata");
+
+ XmlRpcFileManagerClient fmClient = new XmlRpcFileManagerClient(
+ new URL(this.filemgrUrl));
+ return !fmClient.hasProduct(productMetadata.getMetadata(PRODUCT_NAME));
+ } catch (Exception e) {
+ throw new CrawlerActionException("Product failed uniqueness check : ["
+ + product + "] : " + e.getMessage());
+ }
+ }
+
+ @Override
+ public void validate() throws CrawlerActionException {
+ super.validate();
+ try {
+ Validate.notNull(filemgrUrl, "Must specify filemgrUrl");
+ } catch (Exception e) {
+ throw new CrawlerActionException(e);
+ }
+ }
+
+ public void setFilemgrUrl(String filemgrUrl) {
+ this.filemgrUrl = filemgrUrl;
+ }
}
Modified: oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/GroupAction.java
URL: http://svn.apache.org/viewvc/oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/GroupAction.java?rev=1294389&r1=1294388&r2=1294389&view=diff
==============================================================================
--- oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/GroupAction.java (original)
+++ oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/GroupAction.java Mon Feb 27 23:14:03 2012
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@@ -14,21 +14,21 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.oodt.cas.crawl.action;
//JDK imports
import java.io.File;
import java.util.List;
+//Apache imports
+import org.apache.commons.lang.Validate;
+
//OODT imports
import org.apache.oodt.cas.crawl.action.CrawlerAction;
import org.apache.oodt.cas.crawl.structs.exceptions.CrawlerActionException;
import org.apache.oodt.cas.metadata.Metadata;
/**
- *
- *
* Support of a GroupAction could go a ways to simplify command line
* specification of actions to run. Instead of having to list actions to run one
* would simply end up referencing the group. In addition, this may end up
@@ -37,33 +37,43 @@ import org.apache.oodt.cas.metadata.Meta
* executing actions even if an action fails this could be a way to tailor that
* behavior.
*
+ * @author pramirez (Paul Ramirez)
*/
public class GroupAction extends CrawlerAction {
- private List<CrawlerAction> actionsToCall;
+ private List<CrawlerAction> actionsToCall;
- @Override
- public boolean performAction(File product, Metadata metadata)
- throws CrawlerActionException {
- boolean allSucceeded = true;
- for (CrawlerAction action : actionsToCall) {
+ @Override
+ public boolean performAction(File product, Metadata metadata)
+ throws CrawlerActionException {
+ boolean allSucceeded = true;
+ for (CrawlerAction action : actionsToCall) {
+ try {
+ LOG.info("Performing action (id = " + action.getId()
+ + " : description = " + action.getDescription() + ")");
+ if (!action.performAction(product, metadata))
+ throw new Exception("Action (id = " + action.getId()
+ + " : description = " + action.getDescription()
+ + ") returned false");
+ } catch (Exception e) {
+ allSucceeded = false;
+ LOG.warning("Failed to perform crawler action : " + e.getMessage());
+ }
+
+ }
+ return allSucceeded;
+ }
+
+ @Override
+ public void validate() throws CrawlerActionException {
+ super.validate();
try {
- LOG.info("Performing action (id = " + action.getId()
- + " : description = " + action.getDescription() + ")");
- if (!action.performAction(product, metadata))
- throw new Exception("Action (id = " + action.getId()
- + " : description = " + action.getDescription()
- + ") returned false");
+ Validate.notNull(actionsToCall, "Must specify actionsToCall");
} catch (Exception e) {
- allSucceeded = false;
- LOG.warning("Failed to perform crawler action : " + e.getMessage());
+ throw new CrawlerActionException(e);
}
+ }
- }
- return allSucceeded;
- }
-
- public void setActionsToCall(List<CrawlerAction> actionsToCall) {
- this.actionsToCall = actionsToCall;
- }
-
+ public void setActionsToCall(List<CrawlerAction> actionsToCall) {
+ this.actionsToCall = actionsToCall;
+ }
}
Modified: oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/IngestAncillary.java
URL: http://svn.apache.org/viewvc/oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/IngestAncillary.java?rev=1294389&r1=1294388&r2=1294389&view=diff
==============================================================================
--- oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/IngestAncillary.java (original)
+++ oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/IngestAncillary.java Mon Feb 27 23:14:03 2012
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@@ -14,7 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.oodt.cas.crawl.action;
//JDK imports
@@ -45,9 +44,8 @@ import org.springframework.beans.factory
* should consider using a server side metadata extractor configured with the
* file manager.
*
- * @author pramirez
- * @author mattmann
- *
+ * @author pramirez (Paul Ramirez)
+ * @author mattmann (Chris Mattmann)
*/
public class IngestAncillary extends FileBasedAction {
private String fileManagerUrl;
Modified: oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/MimeTypeCrawlerAction.java
URL: http://svn.apache.org/viewvc/oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/MimeTypeCrawlerAction.java?rev=1294389&r1=1294388&r2=1294389&view=diff
==============================================================================
--- oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/MimeTypeCrawlerAction.java (original)
+++ oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/MimeTypeCrawlerAction.java Mon Feb 27 23:14:03 2012
@@ -14,12 +14,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.oodt.cas.crawl.action;
+//OODT static imports
+import static org.apache.oodt.cas.crawl.config.ProductCrawlerBean.MIME_TYPES_HIERARCHY;
-package org.apache.oodt.cas.crawl.action;
+//Apache imports
+import org.apache.commons.lang.Validate;
//OODT imports
-import org.apache.oodt.cas.crawl.config.ProductCrawlerBean;
import org.apache.oodt.cas.crawl.structs.exceptions.CrawlerActionException;
import org.apache.oodt.cas.metadata.Metadata;
@@ -34,55 +37,58 @@ import java.util.logging.Level;
import org.springframework.beans.factory.annotation.Required;
/**
+ * Calls a {@link CrawlerAction} if this {@link File} matches the specified set
+ * of internal {@link #mimeTypes}.
*
- * @author bfoster
- * @version $Revision$
- *
- * <p>
- * Calls a {@link CrawlerAction} if this {@link File} matches the
- * specified set of internal {@link #mimeTypes}.
- * </p>
- *
+ * @author bfoster (Brian Foster)
*/
public class MimeTypeCrawlerAction extends CrawlerAction {
- private CrawlerAction actionToCall;
-
- private List<String> mimeTypes;
-
- @Override
- public boolean performAction(File product, Metadata productMetadata)
- throws CrawlerActionException {
- List<String> mimeTypeHierarchy = productMetadata
- .getAllMetadata(ProductCrawlerBean.MIME_TYPES_HIERARCHY);
- if (mimeTypeHierarchy == null)
- mimeTypeHierarchy = new Vector<String>();
- if (mimeTypes == null
- || (mimeTypes != null && !Collections.disjoint(mimeTypes,
- mimeTypeHierarchy))) {
- return this.actionToCall.performAction(product, productMetadata);
- } else {
- LOG.log(Level.INFO, "Skipping action (id = " + this.getId()
- + " : description = " + this.getDescription()
- + " - doesn't apply to current product");
- return true;
- }
- }
-
- @Required
- public void setActionToCall(CrawlerAction actionToCall) {
- this.actionToCall = actionToCall;
- this.setDescription("Mime-type restricted version of '"
- + this.actionToCall.getId() + "' CrawlerAction");
- this.setPhases(this.actionToCall.getPhases());
- }
-
- public List<String> getMimeTypes() {
- return mimeTypes;
- }
-
- public void setMimeTypes(List<String> mimeTypes) {
- this.mimeTypes = mimeTypes;
- }
+ private CrawlerAction actionToCall;
+ private List<String> mimeTypes;
+ @Override
+ public boolean performAction(File product, Metadata productMetadata)
+ throws CrawlerActionException {
+ List<String> mimeTypeHierarchy = productMetadata
+ .getAllMetadata(MIME_TYPES_HIERARCHY);
+ if (mimeTypeHierarchy == null)
+ mimeTypeHierarchy = new Vector<String>();
+ if (mimeTypes == null
+ || (mimeTypes != null && !Collections.disjoint(mimeTypes,
+ mimeTypeHierarchy))) {
+ return this.actionToCall.performAction(product, productMetadata);
+ } else {
+ LOG.log(Level.INFO, "Skipping action (id = " + this.getId()
+ + " : description = " + this.getDescription()
+ + " - doesn't apply to current product");
+ return true;
+ }
+ }
+
+ @Override
+ public void validate() throws CrawlerActionException {
+ super.validate();
+ try {
+ Validate.notNull(actionToCall, "Must specify actionToCall");
+ } catch (Exception e) {
+ throw new CrawlerActionException(e);
+ }
+ }
+
+ @Required
+ public void setActionToCall(CrawlerAction actionToCall) {
+ this.actionToCall = actionToCall;
+ this.setDescription("Mime-type restricted version of '"
+ + this.actionToCall.getId() + "' CrawlerAction");
+ this.setPhases(this.actionToCall.getPhases());
+ }
+
+ public List<String> getMimeTypes() {
+ return mimeTypes;
+ }
+
+ public void setMimeTypes(List<String> mimeTypes) {
+ this.mimeTypes = mimeTypes;
+ }
}
Modified: oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/MoveFile.java
URL: http://svn.apache.org/viewvc/oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/MoveFile.java?rev=1294389&r1=1294388&r2=1294389&view=diff
==============================================================================
--- oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/MoveFile.java (original)
+++ oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/MoveFile.java Mon Feb 27 23:14:03 2012
@@ -14,8 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-
package org.apache.oodt.cas.crawl.action;
//JDK imports
@@ -23,6 +21,7 @@ import java.io.File;
import java.util.logging.Level;
//OODT imports
+import org.apache.commons.lang.Validate;
import org.apache.oodt.cas.crawl.structs.exceptions.CrawlerActionException;
import org.apache.oodt.cas.metadata.Metadata;
@@ -30,67 +29,70 @@ import org.apache.oodt.cas.metadata.Meta
import org.springframework.beans.factory.annotation.Required;
/**
- *
- * @author bfoster
- * @author mattmann
- * @version $Revision$
- *
- * <p>
* Moves a {@link Product} file as a reponse to a Crawler lifecycle phase
- * </p>.
+ *
+ * @author bfoster (Brian Foster)
+ * @author mattmann (Chris Mattmann)
*/
public class MoveFile extends CrawlerAction {
- private String file;
-
- private String toDir;
-
- private String fileExtension;
-
- private boolean createToDir;
-
- public MoveFile() {
- super();
- this.createToDir = false;
- }
-
- public boolean performAction(File product, Metadata productMetadata)
- throws CrawlerActionException {
- String mvFile = file;
- try {
- if (mvFile == null) {
- mvFile = product.getAbsolutePath();
- if (this.fileExtension != null)
- mvFile += "." + this.fileExtension;
- }
- File srcFile = new File(mvFile);
- File toFile = new File(toDir + "/" + srcFile.getName());
- if (createToDir)
- toFile.getParentFile().mkdirs();
- LOG.log(Level.INFO, "Moving file " + srcFile.getAbsolutePath()
- + " to " + toFile.getAbsolutePath());
- return srcFile.renameTo(toFile);
- } catch (Exception e) {
- throw new CrawlerActionException("Failed to move file from " + mvFile
- + " to " + this.toDir + " : " + e.getMessage());
- }
- }
-
- public void setCreateToDir(boolean createToDir) {
- this.createToDir = createToDir;
- }
-
- public void setFile(String file) {
- this.file = file;
- }
-
- @Required
- public void setToDir(String toDir) {
- this.toDir = toDir;
- }
-
- public void setFileExtension(String fileExtension) {
- this.fileExtension = fileExtension;
- }
-
+ private String file;
+ private String toDir;
+ private String fileExtension;
+ private boolean createToDir;
+
+ public MoveFile() {
+ super();
+ this.createToDir = false;
+ }
+
+ public boolean performAction(File product, Metadata productMetadata)
+ throws CrawlerActionException {
+ String mvFile = file;
+ try {
+ if (mvFile == null) {
+ mvFile = product.getAbsolutePath();
+ if (this.fileExtension != null)
+ mvFile += "." + this.fileExtension;
+ }
+ File srcFile = new File(mvFile);
+ File toFile = new File(toDir + "/" + srcFile.getName());
+ if (createToDir)
+ toFile.getParentFile().mkdirs();
+ LOG.log(Level.INFO, "Moving file " + srcFile.getAbsolutePath()
+ + " to " + toFile.getAbsolutePath());
+ return srcFile.renameTo(toFile);
+ } catch (Exception e) {
+ throw new CrawlerActionException("Failed to move file from " + mvFile
+ + " to " + this.toDir + " : " + e.getMessage());
+ }
+ }
+
+ @Override
+ public void validate() throws CrawlerActionException {
+ super.validate();
+ try {
+ Validate.isTrue(file == null || fileExtension == null,
+ "Must specify either file or fileExtension");
+ } catch (Exception e) {
+ throw new CrawlerActionException(e);
+ }
+ }
+
+ public void setCreateToDir(boolean createToDir) {
+ this.createToDir = createToDir;
+ }
+
+ public void setFile(String file) {
+ this.file = file;
+ }
+
+ @Required
+ public void setToDir(String toDir) {
+ this.toDir = toDir;
+ }
+
+ public void setFileExtension(String fileExtension) {
+ this.fileExtension = fileExtension;
+ }
}