You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stanbol.apache.org by wk...@apache.org on 2012/09/19 10:38:49 UTC
svn commit: r1387485 [1/5] - in
/incubator/stanbol/trunk/enhancer/engines/htmlextractor: ./ src/
src/license/ src/main/ src/main/java/ src/main/java/org/
src/main/java/org/apache/ src/main/java/org/apache/stanbol/
src/main/java/org/apache/stanbol/enhan...
Author: wkasper
Date: Wed Sep 19 08:38:46 2012
New Revision: 1387485
URL: http://svn.apache.org/viewvc?rev=1387485&view=rev
Log:
STANBOL-689: Refactored HTML Extraction engine from Metaxa without Aperture dependency
Added:
incubator/stanbol/trunk/enhancer/engines/htmlextractor/
incubator/stanbol/trunk/enhancer/engines/htmlextractor/README.md
incubator/stanbol/trunk/enhancer/engines/htmlextractor/pom.xml
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/license/
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/HtmlExtractorEngine.java
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/BundleURIResolver.java
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/CharsetRecognizer.java
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/ClerezzaRDFUtils.java
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/DOMUtils.java
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/ExtractorException.java
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/HtmlExtractionComponent.java
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/HtmlExtractionRegistry.java
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/HtmlExtractor.java
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/HtmlParser.java
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/InitializationException.java
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/XsltExtractor.java
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/resources/
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/resources/OSGI-INF/
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/resources/OSGI-INF/metatype/
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/resources/OSGI-INF/metatype/metatype.properties
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/resources/htmlextractors.xml
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/resources/xslt/
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/resources/xslt/Microformats2RDF.xsl
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/resources/xslt/datetime.xsl
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/resources/xslt/functions.xsl
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/resources/xslt/geo2rdf.xsl
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/resources/xslt/hatom2rdf.xsl
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/resources/xslt/hcal2rdf.xsl
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/resources/xslt/hcard2rdf.xsl
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/resources/xslt/hreview2rdf.xsl
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/resources/xslt/htmltextextract.xsl
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/resources/xslt/rdfa.xslt
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/resources/xslt/rel-design-pattern2rdf.xsl
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/resources/xslt/rel-license2rdf.xsl
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/resources/xslt/rel-tag2rdf.xsl
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/resources/xslt/xalan_functions.xsl
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/resources/xslt/xfolk2rdf.xsl
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/test/
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/test/java/
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/test/java/org/
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/test/java/org/apache/
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/test/java/org/apache/stanbol/
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/test/java/org/apache/stanbol/enhancer/
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/test/java/org/apache/stanbol/enhancer/engines/
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/test/java/org/apache/stanbol/enhancer/engines/htmlextractor/
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/test/java/org/apache/stanbol/enhancer/engines/htmlextractor/TestHtmlExtractor.java
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/test/resources/
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/test/resources/README
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/test/resources/test-MF.html
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/test/resources/test-MultiRoot.html
incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/test/resources/test-rdfa.html
Added: incubator/stanbol/trunk/enhancer/engines/htmlextractor/README.md
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/htmlextractor/README.md?rev=1387485&view=auto
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/htmlextractor/README.md (added)
+++ incubator/stanbol/trunk/enhancer/engines/htmlextractor/README.md Wed Sep 19 08:38:46 2012
@@ -0,0 +1,157 @@
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+# Htmlextractor: Metadata extraction from HTML documents
+
+The **Htmlextractor Enhancement Engine** extracts embedded metadata from HTML documents, such as [Microformats](http://microformats.org/) and [RDFa](http://www.w3.org/TR/rdfa-syntax/).
+By providing other extractors it can be configured for any kind of content extraction from HTML pages.
+
+##Technical description
+
+### Supported metadata types
+
+The built-in extractors are defined in the default resource *htmlextractors.xml*. The following metadata types are supported:
+* RDFa
+* geo
+* hAtom
+* hCal
+* hCard
+* hReview
+* rel-license
+* rel-tag
+* xFolk
+
+### Vocabularies
+
+#### HTML Microformat Extractors
+
+The following table describes which vocabularies are used for representing microformat data in Metaxa:
+
+
+<table border="1">
+ <tr>
+ <th>MF</th>
+ <th>Vocabulary (Namespace)</th>
+ </tr>
+ <tr>
+ <td>geo</td>
+ <td>wgs84 (<tt>http://www.w3.org/2003/01/geo/wgs84_pos#</tt>)</td>
+ </tr>
+ <tr>
+ <td>hAtom</td>
+ <td>atom (<tt>http://www.w3.org/2005/Atom#)</td>
+ </tr>
+ <tr>
+ <td/>
+ <td>tagging (<tt>http://aperture.sourceforge.net/ontologies/tagging#</tt>)</td>
+ </tr>
+ <tr>
+ <td>hCal</td>
+ <td> ical (<tt>http://www.w3.org/2002/12/cal/icaltzd#</tt>)</td>
+ </tr>
+ <tr>
+ <td></td>
+ <td>vcard (<tt>http://www.w3.org/2006/vcard/ns#</tt>)</td>
+ </tr>
+ <tr>
+ <td>hCard</td>
+ <td>vcard (<tt>http://www.w3.org/2006/vcard/ns#</tt>)</td>
+ </tr>
+ <tr>
+ <td>hReview</td>
+ <td>review (<tt>http://www.purl.org/stuff/rev#</tt>)</td></tr>
+ <tr>
+ <td></td>
+ <td>wgs84 (<tt>http://www.w3.org/2003/01/geo/wgs84_pos#</tt>)</td>
+ </tr>
+ <tr>
+ <td></td>
+ <td>dc (<tt>http://purl.org/dc/elements/1.1/</tt>)</td>
+ </tr>
+ <tr>
+ <td></td>
+ <td>dcterms (<tt>http://purl.org/dc/dcmitype/</tt>)</td>
+ </tr>
+ <tr>
+ <td></td>
+ <td>foaf (<tt>http://xmlns.com/foaf/0.1/</tt>)</td>
+ </tr>
+ <tr>
+ <td></td>
+ <td>vcard (<tt>http://www.w3.org/2006/vcard/ns#</tt>)</td>
+ </tr>
+ <tr>
+ <td></td>
+ <td>tag (<tt>http://www.holygoat.co.uk/owl/redwood/0.1/tags/</tt>)</td>
+ </tr>
+ <tr>
+ <td>rel-license</td>
+ <td>dc (<tt>http://purl.org/dc/elements/1.1</tt>/)</td>
+ </tr>
+ <tr>
+ <td>rel-tag</td>
+ <td> tagging (<tt>http://aperture.sourceforge.net/ontologies/tagging#</tt>)</td>
+ </tr>
+ <tr>
+ <td>xFolk</td>
+ <td>nfo (<tt>http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#</tt>)</td>
+ </tr>
+ <tr>
+ <td></td>
+ <td>dc (<tt>http://purl.org/dc/elements/1.1</tt>/)</td>
+ </tr>
+ <tr>
+ <td></td>
+ <td>tagging (<tt>http://aperture.sourceforge.net/ontologies/tagging#</tt>)</td>
+ </tr>
+</table>
+
+To prevent the occurrence of unconnected graphs in the metadata extracted subgraphs get connected to the content item by the property:
+
+* http://www.semanticdesktop.org/ontologies/2007/01/19/nie#contains
+
+## Configuration options
+
+By default, the Htmlextractor engine uses the extractors specified in the resource "htmlregistry.xml".
+Alternative configurations and extractors can be attached to the Htmlextractor as fragment bundles, specifying as host bundle
+
+ Fragment-Host: org.apache.stanbol.enhancer.engines.htmlextractor
+
+The alternative configuration files then can be set as values of the property
+
+* <pre><code>org.apache.stanbol.enhancer.engines.htmlextractor.htmlextractors</pre></code>
+
+
+## Usage
+
+Assuming that the Stanbol endpoint with the full launcher is running at
+
+ http://localhost:8080
+
+and the engine is activated, from the command line commands like this can be used for submitting some file as content item, where the mime type must match the document type:
+
+* stateless interface
+
+ curl -i -X POST -H "Content-Type:text/html" -T testpage.html http://localhost:8080/enhancer
+
+* stateful interface
+
+ curl -i -X PUT -H "Content-Type:text/html" -T testpage.html http://localhost:8080/contenthub/content/someFileId
+
+Alternatively, the Stanbol web interface can be used for submitting documents
+and viewing the metadata at
+
+ http://localhost:8080/contenthub
+
Added: incubator/stanbol/trunk/enhancer/engines/htmlextractor/pom.xml
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/htmlextractor/pom.xml?rev=1387485&view=auto
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/htmlextractor/pom.xml (added)
+++ incubator/stanbol/trunk/enhancer/engines/htmlextractor/pom.xml Wed Sep 19 08:38:46 2012
@@ -0,0 +1,148 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>org.apache.stanbol.enhancer.parent</artifactId>
+ <groupId>org.apache.stanbol</groupId>
+ <version>0.10.0-incubating-SNAPSHOT</version>
+ <relativePath>../../parent</relativePath>
+ </parent>
+
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.enhancer.engines.htmlextractor</artifactId>
+ <version>0.10.0-incubating-SNAPSHOT</version>
+ <packaging>bundle</packaging>
+ <name>Apache Stanbol Enhancer Enhancement Engine : HTML Extractors</name>
+ <description>Enhancement Engine that extracts RDFa and Microformat data from HTML pages
+ </description>
+
+ <inceptionYear>2012</inceptionYear>
+
+ <scm>
+ <connection>
+ scm:svn:http://svn.apache.org/repos/asf/incubator/stanbol/trunk/enhancer/engines/html/
+ </connection>
+ <developerConnection>
+ scm:svn:https://svn.apache.org/repos/asf/incubator/stanbol/trunk/enhancer/engines/html/
+ </developerConnection>
+ <url>http://incubator.apache.org/stanbol/</url>
+ </scm>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>maven-bundle-plugin</artifactId>
+ <extensions>true</extensions>
+ <configuration>
+ <instructions>
+ <Export-Package>
+ org.apache.stanbol.enhancer.engines.html;version=${project.version}
+ </Export-Package>
+ <Embed-Dependency>
+ jtidy;scope=compile
+ </Embed-Dependency>
+ <Import-Package>
+ !org.apache.tools.*,
+ *
+ </Import-Package>
+ </instructions>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>maven-scr-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <!-- AL20 License -->
+ <exclude>src/license/THIRD-PARTY.properties</exclude>
+ <!-- AL20 License for test resources (see src/test/resources/README) -->
+ <exclude>src/test/resources/*.html</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.enhancer.servicesapi</artifactId>
+ <version>0.10.0-incubating-SNAPSHOT</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>org.apache.felix.scr.annotations</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.clerezza</groupId>
+ <artifactId>rdf.core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.clerezza</groupId>
+ <artifactId>rdf.jena.parser</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <!-- Test dependencies -->
+ <dependency>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.enhancer.test</artifactId>
+ <version>0.10.0-incubating-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.enhancer.core</artifactId>
+ <version>0.10.0-incubating-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-simple</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.ibm.icu</groupId>
+ <artifactId>icu4j</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>net.sf.jtidy</groupId>
+ <artifactId>jtidy</artifactId>
+ <version>r938</version>
+ </dependency>
+ </dependencies>
+</project>
Added: incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/HtmlExtractorEngine.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/HtmlExtractorEngine.java?rev=1387485&view=auto
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/HtmlExtractorEngine.java (added)
+++ incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/HtmlExtractorEngine.java Wed Sep 19 08:38:46 2012
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.stanbol.enhancer.engines.htmlextractor;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Dictionary;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.clerezza.rdf.core.MGraph;
+import org.apache.clerezza.rdf.core.UriRef;
+import org.apache.clerezza.rdf.core.impl.SimpleMGraph;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Property;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.Service;
+import org.apache.stanbol.enhancer.engines.htmlextractor.impl.BundleURIResolver;
+import org.apache.stanbol.enhancer.engines.htmlextractor.impl.ClerezzaRDFUtils;
+import org.apache.stanbol.enhancer.engines.htmlextractor.impl.ExtractorException;
+import org.apache.stanbol.enhancer.engines.htmlextractor.impl.HtmlExtractionRegistry;
+import org.apache.stanbol.enhancer.engines.htmlextractor.impl.HtmlExtractor;
+import org.apache.stanbol.enhancer.engines.htmlextractor.impl.HtmlParser;
+import org.apache.stanbol.enhancer.engines.htmlextractor.impl.InitializationException;
+import org.apache.stanbol.enhancer.servicesapi.Blob;
+import org.apache.stanbol.enhancer.servicesapi.ContentItem;
+import org.apache.stanbol.enhancer.servicesapi.ContentItemFactory;
+import org.apache.stanbol.enhancer.servicesapi.EngineException;
+import org.apache.stanbol.enhancer.servicesapi.EnhancementEngine;
+import org.apache.stanbol.enhancer.servicesapi.ServiceProperties;
+import org.apache.stanbol.enhancer.servicesapi.impl.AbstractEnhancementEngine;
+import org.apache.stanbol.enhancer.servicesapi.rdf.NamespaceEnum;
+import org.osgi.framework.BundleContext;
+import org.osgi.service.cm.ConfigurationException;
+import org.osgi.service.component.ComponentContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ * @author <a href="mailto:kasper@dfki.de">Walter Kasper</a>
+ *
+ */
+@Component(immediate = true, metatype = true, inherit = true)
+@Service
+@org.apache.felix.scr.annotations.Properties(value={
+ @Property(name=EnhancementEngine.PROPERTY_NAME, value="htmlextractor")
+})
+public class HtmlExtractorEngine extends AbstractEnhancementEngine<IOException,RuntimeException>
+ implements EnhancementEngine, ServiceProperties {
+ private static final Logger LOG = LoggerFactory.getLogger(HtmlExtractorEngine.class);
+
+ /**
+ * The default charset
+ */
+ private static final Charset UTF8 = Charset.forName("UTF-8");
+
+ /**
+ * The default value for the Execution of this Engine. Currently set to
+ * {@link ServiceProperties#ORDERING_PRE_PROCESSING}
+ */
+ public static final Integer defaultOrder = ORDERING_PRE_PROCESSING;
+
+ private static final String DEFAULT_HTML_EXTRACTOR_REGISTRY = "htmlextractors.xml";
+
+ /**
+ * name of a file that defines the set of extractors for HTML documents. By default, the builtin file 'htmlextractors.xml' is used."
+ */
+ @Property(value=HtmlExtractorEngine.DEFAULT_HTML_EXTRACTOR_REGISTRY)
+ public static final String HTML_EXTRACTOR_REGISTRY = "org.apache.stanbol.enhancer.engines.htmlextractor.htmlextractors";
+
+ /**
+ * Internally used to create additional {@link Blob} for transformed
+ * versions af the original content
+ */
+ @Reference
+ private ContentItemFactory ciFactory;
+
+ BundleContext bundleContext;
+
+ private Set<String> supportedMimeTypes = new HashSet<String>(Arrays.asList(new String[]{"text/html","application/xhtml+xml"}));
+
+ private HtmlExtractionRegistry htmlExtractorRegistry;
+ private HtmlParser htmlParser;
+
+ private boolean singleRootRdf = true;
+
+ protected void activate(ComponentContext ce) throws ConfigurationException, IOException {
+ super.activate(ce);
+ this.bundleContext = ce.getBundleContext();
+ BundleURIResolver.BUNDLE = this.bundleContext.getBundle();
+ String htmlExtractors = DEFAULT_HTML_EXTRACTOR_REGISTRY;
+ Dictionary<String, Object> properties = ce.getProperties();
+ String confFile = (String)properties.get(HTML_EXTRACTOR_REGISTRY);
+ if (confFile != null && confFile.trim().length() > 0) {
+ htmlExtractors = confFile;
+ }
+ try {
+ this.htmlExtractorRegistry = new HtmlExtractionRegistry(htmlExtractors);
+ }
+ catch (InitializationException e) {
+ LOG.error("Registry Initialization Error: " + e.getMessage());
+ throw new IOException(e.getMessage());
+ }
+ this.htmlParser = new HtmlParser();
+
+ }
+
+ /**
+ * The deactivate method.
+ *
+ * @param ce the {@link ComponentContext}
+ */
+ protected void deactivate(ComponentContext ce) {
+ super.deactivate(ce);
+ this.htmlParser = null;
+ this.htmlExtractorRegistry = null;
+ }
+
+ @Override
+ public Map<String,Object> getServiceProperties() {
+ return Collections.unmodifiableMap(Collections.singletonMap(ENHANCEMENT_ENGINE_ORDERING, (Object) defaultOrder));
+ }
+
+ @Override
+ public int canEnhance(ContentItem ci) throws EngineException {
+ LOG.info("MimeType: {}", ci.getMimeType());
+ if (isSupported(ci.getMimeType())) {
+ return ENHANCE_ASYNC;
+ }
+ return CANNOT_ENHANCE;
+ }
+
+ @Override
+ public void computeEnhancements(ContentItem ci) throws EngineException {
+ HtmlExtractor extractor = new HtmlExtractor(htmlExtractorRegistry, htmlParser);
+ MGraph model = new SimpleMGraph();
+ ci.getLock().readLock().lock();
+ try {
+ extractor.extract(ci.getUri().getUnicodeString(), ci.getStream(),null, ci.getMimeType(), model);
+ } catch (ExtractorException e) {
+ throw new EngineException("Error while processing ContentItem "
+ + ci.getUri()+" with HtmlExtractor",e);
+ } finally {
+ ci.getLock().readLock().unlock();
+ }
+ ClerezzaRDFUtils.urifyBlankNodes(model);
+ // make the model single rooted
+ if (singleRootRdf) {
+ ClerezzaRDFUtils.makeConnected(model,ci.getUri(),new UriRef(NamespaceEnum.nie+"contains"));
+ }
+ //add the extracted triples to the metadata of the ContentItem
+ ci.getLock().writeLock().lock();
+ try {
+ LOG.info("Model: {}",model);
+ ci.getMetadata().addAll(model);
+ model = null;
+ } finally {
+ ci.getLock().writeLock().unlock();
+ }
+ }
+
+ private boolean isSupported(String mimeType) {
+ return this.supportedMimeTypes.contains(mimeType);
+ }
+
+
+}
Added: incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/BundleURIResolver.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/BundleURIResolver.java?rev=1387485&view=auto
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/BundleURIResolver.java (added)
+++ incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/BundleURIResolver.java Wed Sep 19 08:38:46 2012
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.stanbol.enhancer.engines.htmlextractor.impl;
+
+import java.net.URI;
+import java.net.URL;
+
+import javax.xml.transform.Source;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.URIResolver;
+import javax.xml.transform.stream.StreamSource;
+
+import org.osgi.framework.Bundle;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * BundleURIResolver.java
+ *
+ * @author <a href="mailto:kasper@dfki.de">Walter Kasper</a>
+ */
+public class BundleURIResolver implements URIResolver {
+
+ public static Bundle BUNDLE;
+ private static final Logger LOG = LoggerFactory.getLogger(BundleURIResolver.class);
+
+ public Source resolve(String href, String base) throws TransformerException {
+ //base contains the importing/including script, href the imported/included script
+ //it is assumed here that href is a relative path
+ //TODO href specifying an absolute URI
+ if (base == null) {
+ LOG.error("No base given for: " + href);
+ return null;
+ }
+ String resource;
+ try {
+ LOG.debug("base: " + base + "\n" + "href: " + href);
+ URL newUrl;
+ if (base.startsWith("bundle:")) {
+ URI baseURI = new URI(base);
+ String path = baseURI.getPath();
+ resource = path.substring(1, path.lastIndexOf('/') + 1) + href;
+ newUrl = BUNDLE.getEntry(resource);
+ LOG.debug("Resource: " + resource);
+ if (newUrl != null) {
+ return new StreamSource(newUrl.openStream(), newUrl.toString());
+ } else {
+ return null;
+ }
+ }
+ // for non-bundles assume that we have a normal URL as base
+ resource = base.substring(0, base.lastIndexOf('/') + 1) + href;
+ newUrl = new URL(resource);
+ return new StreamSource(newUrl.openStream(), newUrl.toString());
+ } catch (Exception ex) {
+ throw new TransformerException("BundleURIResolver failed: " + ex.getMessage());
+ }
+ }
+
+}
Added: incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/CharsetRecognizer.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/CharsetRecognizer.java?rev=1387485&view=auto
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/CharsetRecognizer.java (added)
+++ incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/CharsetRecognizer.java Wed Sep 19 08:38:46 2012
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.stanbol.enhancer.engines.htmlextractor.impl;
+
+import java.io.BufferedInputStream;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.ibm.icu.text.CharsetDetector;
+import com.ibm.icu.text.CharsetMatch;
+
+/**
+ * EncodingDetector.java
+ *
+ * @author <a href="mailto:kasper@dfki.de">Walter Kasper</a>
+ *
+ */
+public class CharsetRecognizer {
+
+ /**
+ * This contains the logger.
+ */
+ private static final Logger LOG =
+ LoggerFactory.getLogger(CharsetRecognizer.class);
+
+
+ private static String checkPattern(String str, String pattern, int group) {
+
+ Pattern pat = Pattern.compile(pattern);
+ Matcher m = pat.matcher(str);
+ if (m.find()) {
+ return m.group(group);
+ }
+ return null;
+ }
+
+
+ private static String checkFormat(String format, InputStream in)
+ throws IOException {
+
+ String result = null;
+ String defaultValue = null;
+ byte[] bytes;
+ String decl;
+ in.mark(4096);
+ int read;
+ if (format.equalsIgnoreCase("xml")) {
+ defaultValue = "UTF-8";
+ bytes = new byte[80];
+ read = in.read(bytes);
+ in.reset();
+ decl = new String(bytes, 0, read, "US-ASCII");
+ result = checkPattern(decl, "encoding=\"(\\w[-\\w]+)\"", 1);
+ }
+ else if (format.equalsIgnoreCase("html")) {
+ bytes = new byte[2048];
+ read = in.read(bytes);
+ in.reset();
+ decl = new String(bytes, 0, read, "US-ASCII");
+ result =
+ checkPattern(decl,
+ "<meta .*?content=\".*charset=(\\w[-\\w]+).*?/>", 1);
+ }
+ if (result != null) {
+ result = result.toUpperCase();
+ LOG.debug(format.toUpperCase() + " encoding: " + result);
+ }
+ else {
+ return defaultValue;
+ }
+ return result;
+ }
+
+
+ public static String detect(InputStream in)
+ throws IOException {
+
+ return detect(in, null, null);
+ }
+
+
+ public static String detect(InputStream in, String format, String encoding)
+ throws IOException {
+
+ // the input stream must support marks
+ if (!in.markSupported()) {
+ throw new IOException("Mark not supported by input stream");
+ }
+ String result = null;
+ if (format != null) {
+ result = checkFormat(format, in);
+ if (result != null) {
+ return result;
+ }
+ }
+ // in case of HTML or XML check whether there is a charset
+ // specification; might be too fragile
+ CharsetDetector detector = new CharsetDetector();
+ if (encoding != null) {
+ detector.setDeclaredEncoding(encoding);
+ }
+ detector.setText(in);
+ CharsetMatch found = detector.detect();
+ result = found.getName();
+ LOG.debug("Encoding: " + result);
+ return result;
+ }
+
+ public static void main(String[] args) {
+
+ String format = null;
+ String encoding = null;
+ int argv = 0;
+ while (argv < args.length && args[argv].startsWith("-")) {
+ String option = args[argv].substring(1);
+ if (option.startsWith("f")) {
+ format = args[++argv];
+ }
+ else if (option.startsWith("e")) {
+ encoding = args[++argv];
+ }
+ else {
+ System.err.println("illegal option: " + option);
+ System.exit(1);
+ }
+ ++argv;
+ }
+ for (int i = argv; i < args.length; ++i) {
+ try {
+ BufferedInputStream fstream =
+ new BufferedInputStream(new FileInputStream(args[i]));
+ String found =
+ CharsetRecognizer.detect(fstream, format, encoding);
+ System.out.println("Encoding: " + found + ": " + args[i]);
+ /*
+ * check whether the stream is reset correctly byte[] bytes =
+ * new byte[50]; int read = fstream.read(bytes);
+ * System.out.println(new String(bytes));
+ */
+ fstream.close();
+ } catch (IOException e) {
+ LOG.error(e.getMessage());
+ }
+ }
+ }
+
+}
Added: incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/ClerezzaRDFUtils.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/ClerezzaRDFUtils.java?rev=1387485&view=auto
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/ClerezzaRDFUtils.java (added)
+++ incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/ClerezzaRDFUtils.java Wed Sep 19 08:38:46 2012
@@ -0,0 +1,112 @@
+package org.apache.stanbol.enhancer.engines.htmlextractor.impl;
+
+import static org.apache.stanbol.enhancer.servicesapi.helper.EnhancementEngineHelper.randomUUID;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Set;
+
+import org.apache.clerezza.rdf.core.BNode;
+import org.apache.clerezza.rdf.core.MGraph;
+import org.apache.clerezza.rdf.core.NonLiteral;
+import org.apache.clerezza.rdf.core.Resource;
+import org.apache.clerezza.rdf.core.Triple;
+import org.apache.clerezza.rdf.core.UriRef;
+import org.apache.clerezza.rdf.core.impl.SimpleMGraph;
+import org.apache.clerezza.rdf.core.impl.TripleImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Utilities functions for RDF Graphs
+ *
+ * @author <a href="mailto:kasper@dfki.de">Walter Kasper</a>
+ *
+ */
+
+public class ClerezzaRDFUtils {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ClerezzaRDFUtils.class);
+
+ public static void urifyBlankNodes(MGraph model) {
+ HashMap<BNode,UriRef> blankNodeMap = new HashMap<BNode,UriRef>();
+ MGraph remove = new SimpleMGraph();
+ MGraph add = new SimpleMGraph();
+ for (Triple t: model) {
+ NonLiteral subj = t.getSubject();
+ Resource obj = t.getObject();
+ UriRef pred = t.getPredicate();
+ boolean match = false;
+ if (subj instanceof BNode) {
+ match = true;
+ UriRef ru = blankNodeMap.get(subj);
+ if (ru == null) {
+ ru = createRandomUri();
+ blankNodeMap.put((BNode)subj, ru);
+ }
+ subj = ru;
+ }
+ if (obj instanceof BNode) {
+ match = true;
+ UriRef ru = blankNodeMap.get(obj);
+ if (ru == null) {
+ ru = createRandomUri();
+ blankNodeMap.put((BNode)obj, ru);
+ }
+ obj = ru;
+ }
+ if (match) {
+ remove.add(t);
+ add.add(new TripleImpl(subj,pred,obj));
+ }
+ }
+ model.removeAll(remove);
+ model.addAll(add);
+ }
+
+ public static UriRef createRandomUri() {
+ return new UriRef("urn:rnd:"+randomUUID());
+ }
+
+ public static void makeConnected(MGraph model, NonLiteral root, UriRef property) {
+ Set<NonLiteral> roots = findRoots(model);
+ LOG.debug("Roots: {}",roots.size());
+ boolean found = roots.remove(root);
+ //connect all hanging roots to root by property
+ for (NonLiteral n: roots) {
+ model.add(new TripleImpl(root,property,n));
+ }
+ }
+
+ public static Set<NonLiteral> findRoots(MGraph model) {
+ Set<NonLiteral> roots = new HashSet<NonLiteral>();
+ Set<NonLiteral> visited = new HashSet<NonLiteral>();
+ for (Triple t: model) {
+ NonLiteral subj = t.getSubject();
+ findRoot(model, subj, roots, visited);
+ }
+ return roots;
+ }
+
+ private static void findRoot(MGraph model, NonLiteral node, Set<NonLiteral> roots, Set<NonLiteral> visited) {
+ if (visited.contains(node)) {
+ return;
+ }
+ visited.add(node);
+ Iterator<Triple> it = model.filter(null,null,node);
+ // something that is not the object of some stement is a root
+ if (!it.hasNext()) {
+ roots.add(node);
+ LOG.debug("Root found: {}",node);
+ return;
+ }
+ while (it.hasNext()) {
+ Triple t = it.next();
+ NonLiteral subj = t.getSubject();
+ findRoot(model, subj, roots, visited);
+ }
+ }
+
+
+}
Added: incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/DOMUtils.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/DOMUtils.java?rev=1387485&view=auto
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/DOMUtils.java (added)
+++ incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/DOMUtils.java Wed Sep 19 08:38:46 2012
@@ -0,0 +1,501 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.stanbol.enhancer.engines.htmlextractor.impl;
+
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.StringWriter;
+import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.transform.OutputKeys;
+import javax.xml.transform.Result;
+import javax.xml.transform.Source;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerConfigurationException;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
+import org.xml.sax.SAXParseException;
+
+/**
+ * <code>DOMUtils</code> provides convenience methods for working with DOM
+ * documents.
+ *
+ * @author Walter Kasper, DFKI
+ * @author Joerg Steffen, DFKI
+ * @version $Id: DOMUtils.java 1068358 2011-02-08 12:58:11Z bdelacretaz $
+ */
+public final class DOMUtils {
+
+ /**
+ * This prints the given DOM document to System.out with indentation and
+ * utf-8 encoding.
+ *
+ * @param doc
+ * a DOM <code>Document</code>
+ */
+ public static void printXML(Document doc) {
+
+ try {
+ // prepare the DOM document for writing
+ Source source = new DOMSource(doc);
+
+ // prepare the output
+ Result result = new StreamResult(System.out);
+
+ // write the DOM document to the file
+ // get Transformer
+ Transformer xformer =
+ TransformerFactory.newInstance().newTransformer();
+ xformer.setOutputProperty(
+ OutputKeys.INDENT, "yes");
+ xformer.setOutputProperty(
+ OutputKeys.ENCODING, "UTF-8");
+ xformer.setOutputProperty(
+ "{http://xml.apache.org/xslt}indent-amount", "2");
+ xformer.setOutputProperty(OutputKeys.METHOD, "xml");
+
+ // write to System.out
+ xformer.transform(source, result);
+
+ } catch (TransformerConfigurationException tce) {
+ // error generated during transformer configuration
+ System.err.println(tce.getMessage());
+ // use the contained exception, if any
+ Throwable x = tce;
+ if (tce.getException() != null) {
+ x = tce.getException();
+ }
+ x.printStackTrace();
+ } catch (TransformerException te) {
+ // error generated by the transformer
+ System.err.println(te.getMessage());
+ // use the contained exception, if any
+ Throwable x = te;
+ if (te.getException() != null) {
+ x = te.getException();
+ }
+ x.printStackTrace();
+ }
+ }
+
+
+ /**
+ * This returns a string representation of the given document.
+ *
+ * @param doc
+ * an XML <code>Document</code>
+ * @param encoding
+ * a <code>String</code> with the encoding to use
+ * @param docTypeDef
+ * a <code>String</code> with the DTD name; use <code>null</code>
+ * for no DTD
+ * @return a <code>String</code> with the XML string
+ */
+ public static String getStringFromDoc(
+ Document doc, String encoding, String docTypeDef) {
+
+ try {
+
+ // use a Transformer for output
+ TransformerFactory tFactory = TransformerFactory.newInstance();
+ Transformer xformer = tFactory.newTransformer();
+ xformer.setOutputProperty(
+ OutputKeys.INDENT, "yes");
+ xformer.setOutputProperty(
+ OutputKeys.ENCODING, encoding);
+ xformer.setOutputProperty(
+ "{http://xml.apache.org/xslt}indent-amount", "2");
+ xformer.setOutputProperty(OutputKeys.METHOD, "xml");
+ if (null != docTypeDef) {
+ xformer
+ .setOutputProperty(OutputKeys.DOCTYPE_SYSTEM, docTypeDef);
+ }
+
+ DOMSource source = new DOMSource(doc);
+ StringWriter sw = new StringWriter();
+ StreamResult result = new StreamResult(sw);
+ xformer.transform(source, result);
+ return sw.toString();
+
+ } catch (TransformerConfigurationException tce) {
+ // error generated by the parser
+ System.err.println("** Transformer Factory error");
+ System.err.println(" " + tce.getMessage());
+
+ // use the contained exception, if any
+ Throwable x = tce;
+ if (tce.getException() != null) {
+ x = tce.getException();
+ }
+ x.printStackTrace();
+
+ } catch (TransformerException te) {
+ // error generated by the parser
+ System.err.println("** Transformation error");
+ System.err.println(" " + te.getMessage());
+
+ // use the contained exception, if any
+ Throwable x = te;
+ if (te.getException() != null) {
+ x = te.getException();
+ }
+ x.printStackTrace();
+ }
+
+ return null;
+ }
+
+ /**
+ * This method writes a DOM document to the given output stream.
+ *
+ * @param doc
+ * a DOM <code>Document</code>
+ * @param encoding
+ * a <code>String</code> with the encoding to use
+ * @param docTypeDef
+ * a <code>String</code> with the DTD name; use <code>null</code>
+ * for no DTD
+ * @param out
+ * an <code>OutputStream</code> where to write the DOM document
+ */
+ public static void writeXml(
+ Document doc, String encoding, String docTypeDef, OutputStream out) {
+
+ try {
+ // prepare the DOM document
+ Source source = new DOMSource(doc);
+
+ // prepare the output
+ Result result = new StreamResult(out);
+
+ // write the DOM document to the file
+ // get Transformer
+ Transformer xformer =
+ TransformerFactory.newInstance().newTransformer();
+ xformer.setOutputProperty(
+ OutputKeys.INDENT, "yes");
+ xformer.setOutputProperty(
+ OutputKeys.ENCODING, encoding);
+ xformer.setOutputProperty(
+ "{http://xml.apache.org/xslt}indent-amount", "2");
+ xformer.setOutputProperty(OutputKeys.METHOD, "xml");
+ if (null != docTypeDef) {
+ xformer
+ .setOutputProperty(OutputKeys.DOCTYPE_SYSTEM, docTypeDef);
+ }
+ // write to a file
+ xformer.transform(source, result);
+
+ } catch (TransformerConfigurationException tce) {
+ // error generated during transformer configuration
+ System.err.println(tce.getMessage());
+ // use the contained exception, if any
+ Throwable x = tce;
+ if (tce.getException() != null) {
+ x = tce.getException();
+ }
+ x.printStackTrace();
+ } catch (TransformerException te) {
+ // error generated by the transformer
+ System.err.println(te.getMessage());
+ // use the contained exception, if any
+ Throwable x = te;
+ if (te.getException() != null) {
+ x = te.getException();
+ }
+ x.printStackTrace();
+ }
+ }
+
+
+ /**
+ * This parses the given XML string and creates a DOM Document.
+ *
+ * @param fileName
+ * a <code>String</code> with the source file name
+ * @param encoding
+ * a <code>String</code> denoting the encoding of the XML string
+ * @return Document a DOM <code>Document</code>, <code>null</code> if
+ * parsing fails
+ */
+ public static Document parse(String xml, String encoding) {
+
+ if (encoding == null)
+ encoding = "UTF-8";
+ Document document = null;
+ // initiate DocumentBuilderFactory
+ DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+
+ // to get a validating parser
+ factory.setValidating(false);
+ // to get one that understands namespaces
+ factory.setNamespaceAware(true);
+ factory.setIgnoringElementContentWhitespace(true);
+ try {
+ // get DocumentBuilder
+ DocumentBuilder builder = factory.newDocumentBuilder();
+ // parse and load into memory the Document
+ document =
+ builder.parse(new ByteArrayInputStream(xml.getBytes(encoding)));
+ } catch (UnsupportedEncodingException e) {
+ e.printStackTrace();
+ } catch (ParserConfigurationException e) {
+ e.printStackTrace();
+ } catch (SAXException e) {
+ e.printStackTrace();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ return document;
+ }
+
+ /**
+ * This parses the given XML file and creates a DOM Document.
+ *
+ * @param fileName
+ * a <code>String</code> with the source file name
+ * @param validation
+ * a <code>boolean</code> indicatiing if the parsing uses DTD
+ * valudation
+ * @return Document a DOM <code>Document</code>, <code>null</code> if
+ * parsing fails
+ */
+ public static Document parse(String fileName, boolean validation) {
+
+ Document document = null;
+ // initiate DocumentBuilderFactory
+ DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+
+ // to get a validating parser
+ factory.setValidating(validation);
+ // to get one that understands namespaces
+ factory.setNamespaceAware(true);
+ factory.setIgnoringElementContentWhitespace(true);
+
+ try {
+ // get DocumentBuilder
+ DocumentBuilder builder = factory.newDocumentBuilder();
+ // parse and load into memory the Document
+ document = builder.parse(new File(fileName));
+ return document;
+
+ } catch (SAXParseException spe) {
+ // error generated by the parser
+ System.err.println(
+ "Parsing error, line " + spe.getLineNumber() + ", uri "
+ + spe.getSystemId());
+ System.err.println(" " + spe.getMessage());
+ // use the contained exception, if any
+ Exception x = spe;
+ if (spe.getException() != null) {
+ x = spe.getException();
+ }
+ x.printStackTrace();
+ } catch (SAXException sxe) {
+ // error generated during parsing
+ System.err.println(sxe.getMessage());
+ // use the contained exception, if any
+ Exception x = sxe;
+ if (sxe.getException() != null) {
+ x = sxe.getException();
+ }
+ x.printStackTrace();
+ } catch (ParserConfigurationException pce) {
+ // parser with specified options can't be built
+ System.err.println(pce.getMessage());
+ pce.printStackTrace();
+ } catch (IOException ioe) {
+ // i/o error
+ System.err.println(ioe.getMessage());
+ ioe.printStackTrace();
+ }
+
+ return null;
+ }
+
+ /**
+ * This counts the elements in the given document by tag name.
+ *
+ * @param tag
+ * a <code>String</code> with a tag name
+ * @param doc
+ * a DOM <code>Document</code>
+ * @return number an <code>int</code> with the number of elements by tag
+ * name
+ */
+ public static int countByTagName(String tag, Document doc) {
+
+ NodeList list = doc.getElementsByTagName(tag);
+ return list.getLength();
+ }
+
+ /**
+ * This realizes the <code>indexOf</code> method of the
+ * <code>java.util.List</code> interface for <code>NodeList</code>.
+ *
+ * @param list
+ * a <code>NodeList</code> value
+ * @param node
+ * a <code>Node</code> value
+ * @return an <code>int</code> value, giving the position of
+ * <code>node</code> in <code>list</code> or -1, if node is not
+ * contained in the list
+ */
+ public static int indexOf(NodeList list, Node node) {
+
+ for (int i = 0, j = list.getLength(); i < j; ++i) {
+ if (list.item(i) == node) {
+ return i;
+ }
+ }
+ return -1;
+ }
+
+ /**
+ * This concatenates the string values of all text nodes which are direct
+ * children of the given node. If <code>node</code> is a text or attribute
+ * node, its value is returned. Otherwise <code>null</code> is returned
+ * (improvement potential!).
+ *
+ * @param node
+ * a <code>Node</code> value
+ * @return a <code>String</code> with the concatenated text
+ */
+ public static String getText(Node node) {
+
+ short nodeType = node.getNodeType();
+ if ((nodeType == Node.TEXT_NODE) || (nodeType == Node.ATTRIBUTE_NODE)
+ || (nodeType == Node.CDATA_SECTION_NODE)) {
+ return node.getNodeValue();
+ }
+ else if (nodeType == Node.ELEMENT_NODE) {
+ NodeList dtrs = node.getChildNodes();
+ StringBuffer sb = new StringBuffer();
+ for (int i = 0, j = dtrs.getLength(); i < j; ++i) {
+ Node item = dtrs.item(i);
+ if (item.getNodeType() == Node.TEXT_NODE
+ || item.getNodeType() == Node.CDATA_SECTION_NODE) {
+ sb.append(item.getNodeValue());
+ }
+ }
+ return sb.toString();
+ }
+ return null;
+ }
+
+ /**
+ * This selects all direct children of the given element with the given
+ * name. If the name is <code>null</code>, all children are returned.
+ *
+ * @param ele
+ * an <code>Element</code> value
+ * @param name
+ * a <code>String</code> with the children's name
+ * @return a <code>List</code> of <code>Node</code>s with the children
+ */
+ public static List<org.w3c.dom.Node> getChildren(
+ Element ele, String name) {
+
+ NodeList dtrs = ele.getChildNodes();
+ List<org.w3c.dom.Node> eles = new ArrayList<org.w3c.dom.Node>();
+ for (int i = 0, j = dtrs.getLength(); i < j; ++i) {
+ org.w3c.dom.Node item = dtrs.item(i);
+ if (name == null || item.getNodeName().equals(name)) {
+ eles.add(item);
+ }
+ }
+
+ return eles;
+ }
+
+ /**
+ * This selects all direct children of type 'Element' of the given element.
+ *
+ * @param ele
+ * an <code>Element</code> value
+ * @return a <code>List</code> of <code>Elmenet</code>s with the element
+ * children
+ */
+ public static List<Element> getChildrenElements(Element ele) {
+ NodeList dtrs = ele.getChildNodes();
+ List<Element> eles = new ArrayList<Element>();
+ for (int i = 0, j = dtrs.getLength(); i < j; ++i) {
+ org.w3c.dom.Node item = dtrs.item(i);
+ if (item.getNodeType() == Node.ELEMENT_NODE) {
+ eles.add((Element)item);
+ }
+ }
+
+ return eles;
+ }
+
+ /**
+ * This returns the first child element with the given name found at the
+ * given element.
+ *
+ * @param ele
+ * an <code>Element</code> value
+ * @param name
+ * a <code>String</code> with the name of the child element
+ * @return a <code>Element</code> with the child or <code>null</code> if no
+ * such child was found
+ */
+ public static Element getFirstChild(Element ele, String name) {
+ NodeList dtrs = ele.getChildNodes();
+ for (int i = 0, iMax = dtrs.getLength(); i < iMax; ++i) {
+ org.w3c.dom.Node item = dtrs.item(i);
+ if (item.getNodeName().equals(name)) {
+ return (Element)item;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * This adds a new child with the given name to the given element.
+ *
+ * @param ele
+ * an <code>Element</code>
+ * @param name
+ * a <code>String</code> with the name of the child
+ * @return a <code>Element</code> with the newly created child
+ */
+ public static Element addChild(Element ele, String name) {
+
+ Element child = ele.getOwnerDocument().createElement(name);
+ ele.appendChild(child);
+ return child;
+ }
+
+}
Added: incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/ExtractorException.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/ExtractorException.java?rev=1387485&view=auto
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/ExtractorException.java (added)
+++ incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/ExtractorException.java Wed Sep 19 08:38:46 2012
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.stanbol.enhancer.engines.htmlextractor.impl;
+
+/**
+ *
+ * @author <a href="mailto:kasper@dfki.de">Walter Kasper</a>
+ *
+ */
+
+public class ExtractorException extends Exception {
+
+ public ExtractorException() {
+ super();
+ }
+
+ public ExtractorException(String message) {
+ super(message);
+ }
+
+ public ExtractorException(Throwable cause) {
+
+ super(cause);
+ }
+
+ public ExtractorException(String message, Throwable cause) {
+
+ super(message, cause);
+ }
+
+}
Added: incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/HtmlExtractionComponent.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/HtmlExtractionComponent.java?rev=1387485&view=auto
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/HtmlExtractionComponent.java (added)
+++ incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/HtmlExtractionComponent.java Wed Sep 19 08:38:46 2012
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.stanbol.enhancer.engines.htmlextractor.impl;
+
+import java.util.Map;
+
+import org.apache.clerezza.rdf.core.MGraph;
+import org.w3c.dom.Document;
+
+/**
+ * HtmlExtractionComponent.java
+ *
+ * @author <a href="mailto:kasper@dfki.de">Walter Kasper</a>
+ *
+ */
+public interface HtmlExtractionComponent {
+
+ void extract(String id, Document doc, Map<String, Object> params, MGraph result)
+ throws ExtractorException;
+
+}
Added: incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/HtmlExtractionRegistry.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/HtmlExtractionRegistry.java?rev=1387485&view=auto
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/HtmlExtractionRegistry.java (added)
+++ incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/HtmlExtractionRegistry.java Wed Sep 19 08:38:46 2012
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.stanbol.enhancer.engines.htmlextractor.impl;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Set;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.xpath.XPath;
+import javax.xml.xpath.XPathConstants;
+import javax.xml.xpath.XPathExpressionException;
+import javax.xml.xpath.XPathFactory;
+
+import org.apache.clerezza.rdf.core.serializedform.SupportedFormat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.DOMException;
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+
+/**
+ * HtmlExtractionRegistry.java
+ *
+ * @author <a href="mailto:kasper@dfki.de">Walter Kasper</a>
+ */
+public class HtmlExtractionRegistry {
+
+ /**
+ * This contains the logger.
+ */
+ private static final Logger LOG =
+ LoggerFactory.getLogger(HtmlExtractionRegistry.class);
+
+ private HashMap<String, HtmlExtractionComponent> registry;
+ private HashSet<String> activeExtractors;
+
+
+ public HtmlExtractionRegistry() {
+ registry = new HashMap<String, HtmlExtractionComponent>();
+ activeExtractors = new HashSet<String>();
+ }
+
+ public HtmlExtractionRegistry(String configFileName)
+ throws InitializationException {
+ this();
+ InputStream config = getClass().getClassLoader().getResourceAsStream(configFileName);
+ if (config == null) {
+ throw new InitializationException("File not found: "+configFileName);
+ }
+ initialize(config);
+ }
+
+ public HtmlExtractionRegistry(InputStream config) throws InitializationException {
+ this();
+ initialize(config);
+ }
+
+
+ public void initialize(InputStream configFileStream)
+ throws InitializationException {
+
+ try {
+ XPathFactory factory = XPathFactory.newInstance();
+ XPath xPath = factory.newXPath();
+ DocumentBuilder parser =
+ DocumentBuilderFactory.newInstance().newDocumentBuilder();
+ Document document = parser.parse(new InputSource(configFileStream));
+ Node node;
+ NodeList nodes = (NodeList) xPath.evaluate("/htmlextractors/extractor", document, XPathConstants.NODESET);
+ if (nodes != null) {
+ TransformerFactory transFac = TransformerFactory.newInstance();
+ transFac.setURIResolver(new BundleURIResolver());
+ for (int j = 0, iCnt = nodes.getLength(); j < iCnt; j++) {
+ Node nd = nodes.item(j);
+ node = (Node)xPath.evaluate("@id", nd, XPathConstants.NODE);
+ String id = node.getNodeValue();
+ Node srcNode =
+ (Node)xPath.evaluate("source", nd, XPathConstants.NODE);
+ if (srcNode != null) {
+ node = (Node) xPath.evaluate("@type", srcNode, XPathConstants.NODE);
+ String srcType = node.getNodeValue();
+ if (srcType.equals("xslt")) {
+ String rdfFormat = "rdfxml";
+ String rdfSyntax = SupportedFormat.RDF_XML;
+ node =
+ (Node)xPath.evaluate("@syntax", srcNode,
+ XPathConstants.NODE);
+ if (node != null) {
+ //TODO check syntax types
+ rdfFormat = node.getNodeValue();
+ if (rdfFormat.equalsIgnoreCase("turtle")) {
+ rdfSyntax = SupportedFormat.TURTLE;
+ }
+ else if (rdfFormat.equalsIgnoreCase("xturtle")) {
+ rdfSyntax = SupportedFormat.X_TURTLE;
+ }
+ else if (rdfFormat.equalsIgnoreCase("ntriple")) {
+ rdfSyntax = SupportedFormat.N_TRIPLE;
+ }
+ else if (rdfFormat.equalsIgnoreCase("n3")) {
+ rdfSyntax = SupportedFormat.N3;
+ }
+ else if (!rdfFormat.equalsIgnoreCase("rdfxml")) {
+ throw new InitializationException(
+ "Unknown RDF Syntax: " + rdfFormat
+ + " for " + id + " extractor");
+ }
+ }
+ String fileName = DOMUtils.getText(srcNode);
+ XsltExtractor xsltExtractor =
+ new XsltExtractor(id, fileName,transFac);
+ xsltExtractor.setSyntax(rdfSyntax);
+ // name of URI/URL parameter of the script (default
+ // "uri")
+ node =
+ (Node)xPath.evaluate("@uri", srcNode,
+ XPathConstants.NODE);
+ if (node != null) {
+ xsltExtractor.setUriParameter(node
+ .getNodeValue());
+ }
+ registry.put(id, xsltExtractor);
+ activeExtractors.add(id);
+ }
+ else if (srcType.equals("java")) {
+ String clsName = srcNode.getNodeValue();
+ Object extractor =
+ Class.forName(clsName).newInstance();
+ if (extractor instanceof HtmlExtractionComponent) {
+ registry.put(id,
+ (HtmlExtractionComponent)extractor);
+ activeExtractors.add(id);
+ }
+ else {
+ throw new InitializationException(
+ "clsName is not an HtmlExtractionComponent");
+ }
+ }
+ else {
+ LOG.warn("No valid type for extractor found: "
+ + id);
+ }
+ LOG.info("Extractor for: " + id);
+ }
+
+ }
+ }
+ } catch (FileNotFoundException e) {
+ throw new InitializationException(e.getMessage(), e);
+ } catch (XPathExpressionException e) {
+ throw new InitializationException(e.getMessage(), e);
+ } catch (DOMException e) {
+ throw new InitializationException(e.getMessage(), e);
+ } catch (ParserConfigurationException e) {
+ throw new InitializationException(e.getMessage(), e);
+ } catch (SAXException e) {
+ throw new InitializationException(e.getMessage(), e);
+ } catch (IOException e) {
+ throw new InitializationException(e.getMessage(), e);
+ } catch (ClassNotFoundException e) {
+ throw new InitializationException(e.getMessage(), e);
+ } catch (InstantiationException e) {
+ throw new InitializationException(e.getMessage(), e);
+ } catch (IllegalAccessException e) {
+ throw new InitializationException(e.getMessage(), e);
+ }
+ }
+
+ public HashMap<String, HtmlExtractionComponent> getRegistry() {
+ return registry;
+ }
+
+ public void setRegistry(HashMap<String, HtmlExtractionComponent> registry) {
+ this.registry = registry;
+ }
+
+ public Set<String> getActiveExtractors() {
+ return activeExtractors;
+ }
+
+ public void setActiveExtractors(HashSet<String> activeExtractors) {
+ this.activeExtractors = activeExtractors;
+ }
+
+ public static void main(String[] args) throws Exception {
+ int argv = 0;
+ HtmlExtractionRegistry inst = new HtmlExtractionRegistry(args[0]);
+ System.err.println("Active Components: " + inst.activeExtractors.size());
+ for (String s : inst.activeExtractors) {
+ System.err.println(s);
+ }
+ }
+
+ public void add(String id, String resourceName, String type)
+ throws InitializationException {
+ }
+
+ public void remove(String id) {
+ }
+
+ public void activate(String id) {
+ }
+
+ public void deactivate(String id) {
+ }
+
+}
Added: incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/HtmlExtractor.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/HtmlExtractor.java?rev=1387485&view=auto
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/HtmlExtractor.java (added)
+++ incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/HtmlExtractor.java Wed Sep 19 08:38:46 2012
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.stanbol.enhancer.engines.htmlextractor.impl;
+
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.clerezza.rdf.core.MGraph;
+import org.apache.clerezza.rdf.core.UriRef;
+import org.apache.clerezza.rdf.core.impl.SimpleMGraph;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.Document;
+
+/**
+ * HtmlExtractor.java
+ *
+ * @author <a href="mailto:kasper@dfki.de">Walter Kasper</a>
+ */
+public class HtmlExtractor {
+
+ private static final Logger LOG = LoggerFactory.getLogger(HtmlExtractor.class);
+
+ public static String DEFAULT_CONFIGURATION = "htmlextractors.xml";
+
+ private HtmlParser htmlParser;
+
+ public HtmlExtractionRegistry registry = null;
+
+ public HtmlExtractor() {
+ // lazy initialization when used first
+ if (registry == null) {
+ try {
+ this.htmlParser = new HtmlParser();
+ this.registry = new HtmlExtractionRegistry(DEFAULT_CONFIGURATION);
+ } catch (InitializationException e) {
+ LOG.error("Registry Initialization Error: " + e.getMessage());
+ }
+ }
+ }
+ public HtmlExtractor(HtmlExtractionRegistry registry, HtmlParser parser) {
+ this.registry = registry;
+ this.htmlParser = parser;
+ }
+
+ public HtmlExtractor(String configFileName)
+ throws InitializationException {
+ this.htmlParser = new HtmlParser();
+ this.registry = new HtmlExtractionRegistry(configFileName);
+ }
+
+ public void extract(String id,
+ InputStream input, Charset charset, String mimeType,
+ MGraph result)
+ throws ExtractorException {
+ if (registry == null)
+ return;
+ String encoding;
+ if (charset == null) {
+ if (!input.markSupported()) {
+ input = new BufferedInputStream(input);
+ }
+ try {
+ encoding = CharsetRecognizer.detect(input, "html", "UTF-8");
+ } catch (IOException e) {
+ LOG.error("Charset detection problem: " + e.getMessage());
+ throw new ExtractorException("Charset detection problem: "
+ + e.getMessage());
+ }
+ }
+ else {
+ encoding = charset.name();
+ }
+ Document doc = htmlParser.getDOM(input, encoding);
+ HashMap<String, HtmlExtractionComponent> extractors =
+ registry.getRegistry();
+ List<String> formats = new ArrayList<String>();
+ long modelSize = result.size();
+ for (String s : registry.getActiveExtractors()) {
+ LOG.debug("Extractor: {}", s);
+ HtmlExtractionComponent extractor = extractors.get(s);
+ // TODO: Handle dependencies between Microformat extractors, e.g.
+ // formats used also in other formats
+ if (extractor != null) {
+ extractor.extract(id, doc, null, result);
+ long tmpSize = result.size();
+ if (modelSize < tmpSize) {
+ LOG.debug("{} Statements added: {}",(tmpSize - modelSize),s);
+ modelSize = tmpSize;
+ }
+ }
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+ int argv = 0;
+ HtmlExtractor inst = new HtmlExtractor();
+ for (int i = argv; i < args.length; ++i) {
+ File file = new File(args[i]);
+ InputStream input = new FileInputStream(file);
+ Charset charset = Charset.forName("UTF-8");
+ String mimeType = "text/html";
+ UriRef uri = new UriRef(file.toURI().toString());
+ MGraph container = new SimpleMGraph();
+ inst.extract(uri.getUnicodeString(), input, charset, mimeType, container);
+ System.out.println("Model for " + args[i]);
+ //TODO
+// container.writeTo(System.out);
+ System.out.println();
+ }
+ }
+
+}
Added: incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/HtmlParser.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/HtmlParser.java?rev=1387485&view=auto
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/HtmlParser.java (added)
+++ incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/HtmlParser.java Wed Sep 19 08:38:46 2012
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.stanbol.enhancer.engines.htmlextractor.impl;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+import javax.xml.parsers.ParserConfigurationException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.Document;
+import org.w3c.tidy.Tidy;
+
+/**
+ * HtmlParser.java
+ *
+ * @author <a href="mailto:kasper@dfki.de">Walter Kasper</a>
+ *
+ */
+public class HtmlParser {
+
+ /**
+ * This contains the logger.
+ */
+ private static final Logger LOG = LoggerFactory.getLogger(HtmlParser.class);
+
+ private Tidy htmlToXmlParser;
+
+ public HtmlParser() {
+ this.htmlToXmlParser = new Tidy();
+ this.htmlToXmlParser.setTidyMark(false);
+ this.htmlToXmlParser.setDropEmptyParas(true);
+ this.htmlToXmlParser.setQuiet(true);
+ this.htmlToXmlParser.setQuoteAmpersand(true);
+ this.htmlToXmlParser.setShowWarnings(false);
+ this.htmlToXmlParser.setShowErrors(0);
+ this.htmlToXmlParser.setNumEntities(true);
+ this.htmlToXmlParser.setHideComments(true);
+ this.htmlToXmlParser.setOutputEncoding("UTF-8");
+ this.htmlToXmlParser.setXmlOut(true);
+ }
+
+
+ public Document getDOM(String html) {
+ if (html != null) {
+ return getDOM(new ByteArrayInputStream(html.getBytes()), null);
+ }
+ return null;
+ }
+
+
+ public synchronized Document getDOM(InputStream html, String charset) {
+ if (charset != null) {
+ htmlToXmlParser.setInputEncoding(charset);
+ }
+ Document doc = htmlToXmlParser.parseDOM(html, null);
+ return doc;
+ }
+}
Added: incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/InitializationException.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/InitializationException.java?rev=1387485&view=auto
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/InitializationException.java (added)
+++ incubator/stanbol/trunk/enhancer/engines/htmlextractor/src/main/java/org/apache/stanbol/enhancer/engines/htmlextractor/impl/InitializationException.java Wed Sep 19 08:38:46 2012
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.stanbol.enhancer.engines.htmlextractor.impl;
+
+/**
+ * <code>InitializationException</code> is thrown when an initialization step
+ * fails.
+ *
+ * @author Joerg Steffen, DFKI
+ * @version $Id: InitializationException.java 1068358 2011-02-08 12:58:11Z bdelacretaz $
+ */
+public class InitializationException extends Exception {
+
+ /**
+ * This creates a new instance of <code>InitializationException</code> with
+ * null as its detail message. The cause is not initialized.
+ */
+ public InitializationException() {
+ super();
+ }
+
+ /**
+ * This creates a new instance of <code>InitializationException</code> with
+ * the given detail message. The cause is not initialized.
+ *
+ * @param message
+ * a <code>String</code> with the detail message
+ */
+ public InitializationException(String message) {
+ super(message);
+ }
+
+ /**
+ * This creates a new instance of <code>InitializationException</code> with
+ * the specified cause and a detail message of (cause==null ? null :
+ * cause.toString()) (which typically contains the class and detail message
+ * of cause).
+ *
+ * @param cause
+ * a <code>Throwable</code> with the cause of the exception
+ * (which is saved for later retrieval by the {@link #getCause()}
+ * method). (A <tt>null</tt> value is permitted, and indicates
+ * that the cause is nonexistent or unknown.)
+ */
+ public InitializationException(Throwable cause) {
+
+ super(cause);
+ }
+
+ /**
+ * This creates a new instance of <code>InitializationException</code> with
+ * the given detail message and the given cause.
+ *
+ * @param message
+ * a <code>String</code> with the detail message
+ * @param cause
+ * a <code>Throwable</code> with the cause of the exception
+ * (which is saved for later retrieval by the {@link #getCause()}
+ * method). (A <tt>null</tt> value is permitted, and indicates
+ * that the cause is nonexistent or unknown.)
+ */
+ public InitializationException(String message, Throwable cause) {
+
+ super(message, cause);
+ }
+
+}