You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stanbol.apache.org by rw...@apache.org on 2012/12/22 18:52:43 UTC
svn commit: r1425326 - in /stanbol/trunk/enhancement-engines/smartcn-token:
./ src/ src/main/ src/main/java/ src/main/java/org/
src/main/java/org/apache/ src/main/java/org/apache/stanbol/
src/main/java/org/apache/stanbol/enhancer/ src/main/java/org/apa...
Author: rwesten
Date: Sat Dec 22 17:52:42 2012
New Revision: 1425326
URL: http://svn.apache.org/viewvc?rev=1425326&view=rev
Log:
STANBOL-860: implementation of a sentence detection and tokenizer engine for Chinese based on the Solr/Lucene smartcn analyzers
Added:
stanbol/trunk/enhancement-engines/smartcn-token/ (with props)
stanbol/trunk/enhancement-engines/smartcn-token/pom.xml
stanbol/trunk/enhancement-engines/smartcn-token/src/
stanbol/trunk/enhancement-engines/smartcn-token/src/main/
stanbol/trunk/enhancement-engines/smartcn-token/src/main/java/
stanbol/trunk/enhancement-engines/smartcn-token/src/main/java/org/
stanbol/trunk/enhancement-engines/smartcn-token/src/main/java/org/apache/
stanbol/trunk/enhancement-engines/smartcn-token/src/main/java/org/apache/stanbol/
stanbol/trunk/enhancement-engines/smartcn-token/src/main/java/org/apache/stanbol/enhancer/
stanbol/trunk/enhancement-engines/smartcn-token/src/main/java/org/apache/stanbol/enhancer/engines/
stanbol/trunk/enhancement-engines/smartcn-token/src/main/java/org/apache/stanbol/enhancer/engines/smartcn/
stanbol/trunk/enhancement-engines/smartcn-token/src/main/java/org/apache/stanbol/enhancer/engines/smartcn/impl/
stanbol/trunk/enhancement-engines/smartcn-token/src/main/java/org/apache/stanbol/enhancer/engines/smartcn/impl/SmartcnTokenizerEngine.java
stanbol/trunk/enhancement-engines/smartcn-token/src/main/resources/
stanbol/trunk/enhancement-engines/smartcn-token/src/main/resources/OSGI-INF/
stanbol/trunk/enhancement-engines/smartcn-token/src/main/resources/OSGI-INF/metatype/
stanbol/trunk/enhancement-engines/smartcn-token/src/main/resources/OSGI-INF/metatype/metatype.properties
Propchange: stanbol/trunk/enhancement-engines/smartcn-token/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Sat Dec 22 17:52:42 2012
@@ -0,0 +1,7 @@
+target
+
+.settings
+
+.project
+
+.classpath
Added: stanbol/trunk/enhancement-engines/smartcn-token/pom.xml
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/smartcn-token/pom.xml?rev=1425326&view=auto
==============================================================================
--- stanbol/trunk/enhancement-engines/smartcn-token/pom.xml (added)
+++ stanbol/trunk/enhancement-engines/smartcn-token/pom.xml Sat Dec 22 17:52:42 2012
@@ -0,0 +1,103 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <artifactId>org.apache.stanbol.enhancer.parent</artifactId>
+ <groupId>org.apache.stanbol</groupId>
+ <version>0.10.0-SNAPSHOT</version>
+ <relativePath>../../enhancer/parent</relativePath>
+ </parent>
+
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.enhancer.engines.smartcn.token</artifactId>
+ <version>0.10.0-SNAPSHOT</version>
+ <packaging>bundle</packaging>
+
+ <name>Apache Stanbol Enhancement Engine : Smart Chinese Tokenizer</name>
+ <description>Sententence detection and Tokenizer for Chinese based on the
+ Solr/Lucene smartcn (Smart Chinese) analyzer module </description>
+
+ <inceptionYear>2012</inceptionYear>
+
+ <scm>
+ <connection>
+ scm:svn:http://svn.apache.org/repos/asf/stanbol/trunk/enhancement-engines/smartcn-token/
+ </connection>
+ <developerConnection>
+ scm:svn:https://svn.apache.org/repos/asf/stanbol/trunk/enhancement-engines/smartcn-token/
+ </developerConnection>
+ <url>http://stanbol.apache.org/</url>
+ </scm>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>maven-bundle-plugin</artifactId>
+ <extensions>true</extensions>
+ <configuration>
+ <instructions>
+ <Import-Package>
+ org.apache.stanbol.enhancer.servicesapi; provide:=true,
+ org.apache.stanbol.enhancer.servicesapi.impl; provide:=true,
+ org.apache.lucene.analysis.cn.smart,
+ *
+ </Import-Package>
+ <Private-Package>
+ org.apache.stanbol.enhancer.engines.smartcn.impl
+ </Private-Package>
+ </instructions>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <!-- AL20 License -->
+ <exclude>src/license/THIRD-PARTY.properties</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>maven-scr-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.enhancer.servicesapi</artifactId>
+ <version>0.10.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.commons.solr.extras.smartcn</artifactId>
+ <version>0.10.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.enhancer.nlp</artifactId>
+ <version>0.10.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>org.apache.felix.scr.annotations</artifactId>
+ </dependency>
+ </dependencies>
+
+</project>
Added: stanbol/trunk/enhancement-engines/smartcn-token/src/main/java/org/apache/stanbol/enhancer/engines/smartcn/impl/SmartcnTokenizerEngine.java
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/smartcn-token/src/main/java/org/apache/stanbol/enhancer/engines/smartcn/impl/SmartcnTokenizerEngine.java?rev=1425326&view=auto
==============================================================================
--- stanbol/trunk/enhancement-engines/smartcn-token/src/main/java/org/apache/stanbol/enhancer/engines/smartcn/impl/SmartcnTokenizerEngine.java (added)
+++ stanbol/trunk/enhancement-engines/smartcn-token/src/main/java/org/apache/stanbol/enhancer/engines/smartcn/impl/SmartcnTokenizerEngine.java Sat Dec 22 17:52:42 2012
@@ -0,0 +1,252 @@
+/*
+ * Copyright (c) 2012 Sebastian Schaffert
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.stanbol.enhancer.engines.smartcn.impl;
+
+import static org.apache.stanbol.enhancer.nlp.utils.NlpEngineHelper.getLanguage;
+import static org.apache.stanbol.enhancer.nlp.utils.NlpEngineHelper.initAnalysedText;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.clerezza.rdf.core.UriRef;
+import org.apache.commons.io.input.CharSequenceReader;
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.ConfigurationPolicy;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Properties;
+import org.apache.felix.scr.annotations.Property;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.Service;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.cn.smart.SentenceTokenizer;
+import org.apache.lucene.analysis.cn.smart.WordTokenFilter;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
+import org.apache.stanbol.enhancer.nlp.NlpProcessingRole;
+import org.apache.stanbol.enhancer.nlp.NlpServiceProperties;
+import org.apache.stanbol.enhancer.nlp.model.AnalysedText;
+import org.apache.stanbol.enhancer.nlp.model.AnalysedTextFactory;
+import org.apache.stanbol.enhancer.nlp.model.Sentence;
+import org.apache.stanbol.enhancer.nlp.model.Token;
+import org.apache.stanbol.enhancer.nlp.utils.NlpEngineHelper;
+import org.apache.stanbol.enhancer.servicesapi.Blob;
+import org.apache.stanbol.enhancer.servicesapi.ContentItem;
+import org.apache.stanbol.enhancer.servicesapi.EngineException;
+import org.apache.stanbol.enhancer.servicesapi.EnhancementEngine;
+import org.apache.stanbol.enhancer.servicesapi.ServiceProperties;
+import org.apache.stanbol.enhancer.servicesapi.impl.AbstractEnhancementEngine;
+import org.osgi.framework.Constants;
+import org.osgi.service.cm.ConfigurationException;
+import org.osgi.service.component.ComponentContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Sentence detection and word tokenizer for Chinese based on the Solr/Lucene
+ * smartcn analysers.
+ *
+ * @author Rupert Westenthaler
+ */
+
+@Component(immediate = true, metatype = true,
+ policy = ConfigurationPolicy.OPTIONAL) //create a default instance with the default configuration
+@Service
+@Properties(value={
+ @Property(name= EnhancementEngine.PROPERTY_NAME,value="smartcn-token"),
+ @Property(name=Constants.SERVICE_RANKING,intValue=0) //give the default instance a ranking < 0
+})
+public class SmartcnTokenizerEngine extends AbstractEnhancementEngine<RuntimeException,RuntimeException> implements ServiceProperties {
+
+ private static final Map<String,Object> SERVICE_PROPERTIES;
+ static {
+ Map<String,Object> props = new HashMap<String,Object>();
+ props.put(ServiceProperties.ENHANCEMENT_ENGINE_ORDERING,
+ ServiceProperties.ORDERING_NLP_TOKENIZING);
+ props.put(NlpServiceProperties.ENHANCEMENT_ENGINE_NLP_ROLE,
+ NlpProcessingRole.Tokenizing);
+ SERVICE_PROPERTIES = Collections.unmodifiableMap(props);
+ }
+
+
+ private static Logger log = LoggerFactory.getLogger(SmartcnTokenizerEngine.class);
+
+ @Reference
+ private AnalysedTextFactory analysedTextFactory;
+
+ /**
+ * Indicate if this engine can enhance supplied ContentItem, and if it
+ * suggests enhancing it synchronously or asynchronously. The
+ * {@link org.apache.stanbol.enhancer.servicesapi.EnhancementJobManager} can force sync/async mode if desired, it is
+ * just a suggestion from the engine.
+ * <p/>
+ * Returns ENHANCE_ASYNC in case there is a text/plain content part and a tagger for the language identified for
+ * the content item, CANNOT_ENHANCE otherwise.
+ *
+ * @throws org.apache.stanbol.enhancer.servicesapi.EngineException
+ * if the introspecting process of the content item
+ * fails
+ */
+ @Override
+ public int canEnhance(ContentItem ci) throws EngineException {
+ // check if content is present
+ Map.Entry<UriRef,Blob> entry = NlpEngineHelper.getPlainText(this, ci, false);
+ if(entry == null || entry.getValue() == null) {
+ return CANNOT_ENHANCE;
+ }
+
+ String language = getLanguage(this,ci,false);
+ if("zh".equals(language) || (language != null && language.startsWith("zh-"))) {
+ log.trace(" > can enhance ContentItem {} with language {}",ci,language);
+ return ENHANCE_ASYNC;
+ } else {
+ return CANNOT_ENHANCE;
+ }
+ }
+
+ /**
+ * Compute enhancements for supplied ContentItem. The results of the process
+ * are expected to be stored in the metadata of the content item.
+ * <p/>
+ * The client (usually an {@link org.apache.stanbol.enhancer.servicesapi.EnhancementJobManager}) should take care of
+ * persistent storage of the enhanced {@link org.apache.stanbol.enhancer.servicesapi.ContentItem}.
+ * <p/>
+ * This method creates a new POSContentPart using {@link org.apache.stanbol.enhancer.engines.pos.api.POSTaggerHelper#createContentPart} from a text/plain part and
+ * stores it as a new part in the content item. The metadata is not changed.
+ *
+ * @throws org.apache.stanbol.enhancer.servicesapi.EngineException
+ * if the underlying process failed to work as
+ * expected
+ */
+ @Override
+ public void computeEnhancements(ContentItem ci) throws EngineException {
+ final AnalysedText at = initAnalysedText(this,analysedTextFactory,ci);
+
+ String language = getLanguage(this,ci,false);
+ if(!("zh".equals(language) || (language != null && language.startsWith("zh-")))) {
+ throw new IllegalStateException("The detected language is NOT 'zh'! "
+ + "As this is also checked within the #canEnhance(..) method this "
+ + "indicates an Bug in the used EnhancementJobManager implementation. "
+ + "Please report this on the dev@apache.stanbol.org or create an "
+ + "JIRA issue about this.");
+ }
+ //first the sentences
+ TokenStream sentences = new SentenceTokenizer(new CharSequenceReader(at.getText()));
+ try {
+ while(sentences.incrementToken()){
+ OffsetAttribute offset = sentences.addAttribute(OffsetAttribute.class);
+ Sentence s = at.addSentence(offset.startOffset(), offset.endOffset());
+ if(log.isTraceEnabled()) {
+ log.trace("detected {}:{}",s,s.getSpan());
+ }
+ }
+ } catch (IOException e) {
+ String message = String.format("IOException while reading from "
+ +"CharSequenceReader of AnalyzedText for ContentItem %s",ci.getUri());
+ log.error(message,e);
+ throw new EngineException(this, ci, message, e);
+ }
+ //now the tokens
+ TokenStream tokens = new WordTokenFilter(new AnalyzedTextSentenceTokenizer(at));
+ try {
+ while(tokens.incrementToken()){
+ OffsetAttribute offset = tokens.addAttribute(OffsetAttribute.class);
+ Token t = at.addToken(offset.startOffset(), offset.endOffset());
+ log.trace("detected {}",t);
+ }
+ } catch (IOException e) {
+ String message = String.format("IOException while reading from "
+ +"CharSequenceReader of AnalyzedText for ContentItem %s",ci.getUri());
+ log.error(message,e);
+ throw new EngineException(this, ci, message, e);
+ }
+ }
+
+ @Override
+ public Map<String,Object> getServiceProperties() {
+ return SERVICE_PROPERTIES;
+ }
+ /**
+ * Activate and read the properties. Configures and initialises a POSTagger for each language configured in
+ * CONFIG_LANGUAGES.
+ *
+ * @param ce the {@link org.osgi.service.component.ComponentContext}
+ */
+ @Activate
+ protected void activate(ComponentContext ce) throws ConfigurationException {
+ log.info("activating smartcn tokenizing engine");
+ super.activate(ce);
+ }
+
+ @Deactivate
+ protected void deactivate(ComponentContext context) {
+ super.deactivate(context);
+ }
+
+ /**
+ * This is an internal helper class that avoids to execute sentences
+ * using the {@link SentenceTokenizer} twice.
+ * @author Rupert Westenthaler
+ *
+ */
+ protected final class AnalyzedTextSentenceTokenizer extends Tokenizer {
+ private final AnalysedText at;
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
+ private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
+ private Iterator<Sentence> sentences;
+ private Sentence sentence = null;
+
+ protected AnalyzedTextSentenceTokenizer(AnalysedText at) {
+ this.at = at;
+ sentences = at.getSentences();
+ }
+
+ @Override
+ public boolean incrementToken() throws IOException {
+ if(sentences.hasNext()){
+ sentence = sentences.next();
+ termAtt.setEmpty().append(sentence.getSpan());
+ offsetAtt.setOffset(sentence.getStart(),sentence.getEnd());
+ typeAtt.setType("sentence");
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public void end() throws IOException {
+ // set final offset
+ offsetAtt.setOffset(at.getEnd(), at.getEnd());
+ }
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ sentences = at.getSentences();
+ termAtt.setEmpty();
+ offsetAtt.setOffset(0, 0);
+ typeAtt.setType(null);
+ }
+ }
+
+}
Added: stanbol/trunk/enhancement-engines/smartcn-token/src/main/resources/OSGI-INF/metatype/metatype.properties
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/smartcn-token/src/main/resources/OSGI-INF/metatype/metatype.properties?rev=1425326&view=auto
==============================================================================
--- stanbol/trunk/enhancement-engines/smartcn-token/src/main/resources/OSGI-INF/metatype/metatype.properties (added)
+++ stanbol/trunk/enhancement-engines/smartcn-token/src/main/resources/OSGI-INF/metatype/metatype.properties Sat Dec 22 17:52:42 2012
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+org.apache.stanbol.enhancer.engines.opennlp.token.impl.OpenNlpTokenizerEngine.name=Apache \
+Stanbol Enhancer Engine: OpenNLP Tokenizer
+org.apache.stanbol.enhancer.engines.opennlp.token.impl.OpenNlpTokenizerEngine.description=Enhancement \
+Engine that tokenizes text by using the OpenNLP Tokenizer. It can be configured to use a \
+custom TokenizerModel or the SimpleTokenizer by configuring 'SIMPLE' as modle name.
+
+
+stanbol.enhancer.engine.name.name=Name
+stanbol.enhancer.engine.name.description=The name of the enhancement engine as \
+used in the RESTful interface '/engine/<name>'
+service.ranking.name=Ranking
+service.ranking.description=If two enhancement engines with the same name are active the \
+one with the higher ranking will be used to process parsed content items.
+
+org.apache.stanbol.enhancer.token.languages.name=Language configuration
+org.apache.stanbol.enhancer.token.languages.description=Takes a list of ISO \
+ language codes. '*' is the Wildcard; '!{lang}' to exclude a language; \
+ '{lang};model={tokenizer-model-file-name}' to configure a \
+ custom OpenNLP tokenizer model for a language. The name 'SIMPLE' can be used to \
+ force the usage of the SimpleTokenizer for a lnaugage. \
+ Models are loaded via the Stanbol DataFileProvider service. So users can e.g. \
+ put models in the datafiles directory \
+ (defaults to '{stanbol-working-dir}/stanbol/datafiles')