You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by dr...@apache.org on 2023/07/13 08:23:31 UTC
[uima-uimacpp] 10/11: scriptators running
This is an automated email from the ASF dual-hosted git repository.
drdub pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/uima-uimacpp.git
commit 87c99761bd4c1f543428d39ba44bc2c366f8cefd
Author: Pablo Duboue <pa...@gmail.com>
AuthorDate: Thu Jul 13 02:59:17 2023 -0400
scriptators running
---
.dockerignore | 1 -
Dockerfile | 40 +
build.Dockerfile | 28 +
docker-entrypoint.sh | 17 +
examples/Dockerfile | 9 +
examples/README.md | 130 ++
examples/lib/.gitkeep | 0
examples/scriptators/README.md | 17 +
.../History_of_natural_language_processing.txt | 333 ++++
examples/scriptators/data/IBM_Watson.txt | 1935 ++++++++++++++++++++
.../data/Natural_language_processing.txt | 1345 ++++++++++++++
examples/scriptators/data/UIMA.txt | 384 ++++
examples/scriptators/perltator/desc/Perltator.xml | 109 ++
examples/scriptators/perltator/scripts/ae.pl | 105 ++
.../scriptators/perltator/scripts/perltator.pm | 546 ++++++
.../scriptators/pythonnator/desc/Pythonnator.xml | 109 ++
examples/scriptators/pythonnator/scripts/ae.py | 95 +
.../scriptators/pythonnator/scripts/pythonnator.py | 399 ++++
examples/src/ExampleApplication.cpp | 1 +
{src/test => examples}/src/SimpleTextSegmenter.cpp | 0
examples/src/SofaDataAnnotator.cpp | 1 +
examples/src/SofaExampleAnnotator.cpp | 1 +
examples/src/SofaExampleApplication.cpp | 1 +
.../src/SofaStreamHandlerFile.cpp | 0
examples/src/XCasWriterCasConsumer.cpp | 1 +
scriptators/perl/Makefile | 10 +-
scriptators/perl/Perltator.cpp | 3 +-
scriptators/perl/Perltator.xml | 109 ++
scriptators/perl/ae.pl | 87 +
scriptators/python/Makefile | 12 +-
scriptators/python/Pythonnator.xml | 109 ++
scriptators/python/ae.py | 77 +
scriptators/uima.i | 6 +-
src/base.mak | 8 +-
src/test/fvt.sh | 2 +-
src/test/src/SimpleTextSegmenter.cpp | 194 +-
src/test/src/SofaStreamHandlerFile.cpp | 257 +--
37 files changed, 6012 insertions(+), 469 deletions(-)
diff --git a/.dockerignore b/.dockerignore
index 8a7a0ce..e73635c 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,3 +1,2 @@
*~
docs/
-examples/
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000..c2a2427
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,40 @@
+# Build SDK
+
+FROM debian:bookworm-slim AS build
+WORKDIR /uima-cpp
+RUN apt-get update && apt-get -y install --no-install-suggests libapr1-dev libxerces-c-dev libicu-dev openjdk-17-jdk-headless build-essential pkgconf autoconf file libtool m4 automake swig4.0 libpython3-dev libperl-dev
+COPY . .
+RUN ./autogen.sh
+RUN mkdir /usr/local/uimacpp
+RUN mkdir /usr/local/uimacpp/desc
+RUN mkdir /usr/local/uimacpp/scripts
+RUN mkdir /usr/local/uimacpp/ae
+RUN ./configure --prefix=/usr/local/uimacpp --without-activemq --with-jdk=/usr/lib/jvm/java-17-openjdk-amd64/
+RUN make -j4
+RUN make install
+# run tests
+RUN make check
+
+# pythonnator
+RUN echo '/usr/local/uimacpp/lib' >> /etc/ld.so.conf.d/uimacpp.conf && ldconfig
+WORKDIR /uima-cpp/scriptators/python
+RUN UIMACPP_HOME=/usr/local/uimacpp make
+RUN cp _pythonnator.so /usr/local/uimacpp/lib; cd /usr/local/uimacpp/scripts; ln -s ../lib/_pythonnator.so .
+RUN cd /usr/local/uimacpp/lib; ln -s ../scripts/_pythonnator.so ./libpythonnator.so
+RUN cp Pythonnator.xml /usr/local/uimacpp/desc/
+RUN cp ae.py pythonnator.py /usr/local/uimacpp/scripts/
+
+# perltator
+WORKDIR /uima-cpp/scriptators/perl
+RUN UIMACPP_HOME=/usr/local/uimacpp make
+RUN cp perltator.so /usr/local/uimacpp/lib; cd /usr/local/uimacpp/lib; ln -s perltator.so libperltator.so
+RUN cp Perltator.xml /usr/local/uimacpp/desc/
+RUN cp ae.pl perltator.pm /usr/local/uimacpp/scripts/
+
+# Generate image with installed files
+FROM debian:bookworm-slim
+WORKDIR /
+COPY --from=build /usr/local/uimacpp /usr/local/uimacpp
+RUN apt-get update && apt-get -y install --no-install-suggests libapr1 libxerces-c3.2 libicu72 libpython3.11 libperl5.36 && echo '/usr/local/uimacpp/lib' >> /etc/ld.so.conf.d/uimacpp.conf && ldconfig
+COPY docker-entrypoint.sh /
+ENTRYPOINT ["/docker-entrypoint.sh"]
diff --git a/build.Dockerfile b/build.Dockerfile
new file mode 100644
index 0000000..78ab5e1
--- /dev/null
+++ b/build.Dockerfile
@@ -0,0 +1,28 @@
+# Build SDK
+
+FROM debian:bookworm-slim AS build
+WORKDIR /uima-cpp
+RUN apt-get update && apt-get -y install --no-install-suggests libapr1-dev libxerces-c-dev libicu-dev openjdk-17-jdk-headless build-essential pkgconf autoconf file libtool m4 automake swig4.0 libpython3-dev
+COPY . .
+RUN ./autogen.sh
+RUN mkdir /usr/local/uimacpp
+RUN mkdir /usr/local/uimacpp/ae
+RUN mkdir /usr/local/uimacpp/desc
+RUN mkdir /usr/local/uimacpp/scripts
+RUN ./configure --prefix=/usr/local/uimacpp --without-activemq --with-jdk=/usr/lib/jvm/java-17-openjdk-amd64/
+RUN make -j4
+RUN make install
+# run tests
+RUN make check
+
+# build pythonator
+RUN echo '/usr/local/uimacpp/lib' >> /etc/ld.so.conf.d/uimacpp.conf && ldconfig
+WORKDIR /uima-cpp/scriptators
+RUN UIMACPP_HOME=/usr/local/uimacpp make
+RUN cp libpythonator.so /usr/local/uimacpp/lib/
+RUN cp Pythonator.xml /usr/local/uimacpp/desc/
+RUN cp ae.py pythonnator.py /usr/local/uimacpp/scripts/
+
+# dev
+RUN apt-get install -y --no-install-recommends emacs-nox gdb less
+ENTRYPOINT ["/bin/bash"]
diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh
old mode 100644
new mode 100755
index 5cf9c3f..649da8a
--- a/docker-entrypoint.sh
+++ b/docker-entrypoint.sh
@@ -10,5 +10,22 @@ _is_sourced() {
}
_main() {
+ # for generic AEs .so in /usr/local/uimacpp/ae
+
+ # without LD_LIBRARY_PATH all annotators would need to be named 'lib' something
+ # https://stackoverflow.com/q/11842729
+ export LD_LIBRARY_PATH="/usr/local/uimacpp/ae"
+
+ # for pythonnator, until a package is system installed
+ export PYTHONPATH=/usr/local/uimacpp/scripts
+
+ # for perltator, until the module is system installed
+ export PATH=/usr/local/uimacpp/scripts:$PATH
+ export PERLLIB=/usr/local/uimacpp/scripts:/usr/local/uimacpp/lib
+
/usr/local/uimacpp/bin/runAECpp "$@"
}
+
+if ! _is_sourced; then
+ _main "$@"
+fi
diff --git a/examples/Dockerfile b/examples/Dockerfile
new file mode 100644
index 0000000..01a0100
--- /dev/null
+++ b/examples/Dockerfile
@@ -0,0 +1,9 @@
+FROM apache:uimacpp AS build
+WORKDIR /examples
+RUN apt-get update && apt-get -y install --no-install-suggests libapr1-dev libxerces-c-dev libicu-dev build-essential pkgconf autoconf
+COPY . .
+RUN UIMACPP_HOME=/usr/local/uimacpp make -C src -f all.mak
+
+# copy the annotators
+FROM scratch
+COPY --from=build /examples/src/*.so .
diff --git a/examples/README.md b/examples/README.md
new file mode 100644
index 0000000..af7ec22
--- /dev/null
+++ b/examples/README.md
@@ -0,0 +1,130 @@
+UIMA C++ Examples
+=================
+
+This directory contains example code that illustrate how to use the UIMA
+C++ Framework.
+
+Building the Examples
+---------------------
+
+Build the examples as follows:
+
+```bash
+$ sudo DOCKER_BUILDKIT=1 docker build -t localuser:uimacppex --output lib .
+```
+
+This will create the shared libraries in lib.
+
+To run it:
+
+```bash
+$ sudo docker run -v $PWD/lib:/usr/local/uimacpp/ae -v $PWD/data:/data -v $PWD/out:/out -v $PWD/descriptors:/descriptors apache:uimacpp /descriptors/DaveDetector.xml /data/example.txt /out
+```
+
+
+Running the Sample UIMA Components
+----------------------------------
+
+These components can be run using either in the native C++ environment
+using the `runAECpp` program or run from Java using the `runAE` utility
+or integrated into a CPE. The UIMA C++ descriptors are located in the
+descriptors subdirectory.
+
+### DaveDetector
+
+A UIMA annotator that finds Daves in text and annotates them. It has one
+configuration parameter, `DaveString`, that specifies the string to
+match. It illustrates how to use the CAS APIs to create annotations and
+add them to the index.
+
+To run this annotator in C++:
+
+### SofaExampleAnnotator
+
+A simple multi-Sofa example annotator that expects an English text Sofa
+as input and creates a German text Sofa as output. This annotator has no
+configuration parameters, and requires no initialization method. To run
+this annotator in C++:
+
+### SofaStreamHandlerFile
+
+This component implements the Sofa stream handler interface defined in
+`sofastreamhandler.hpp` to provide stream access to data located using
+the `file:` URI scheme. It enables a UIMA component to access remote
+Sofa data referenced with a `file:` URI. This example may be used as a
+model for building handlers for custom URI schemes. The shared library
+`SofaStreamHandlerFile` must be registered with the framework as
+follows:
+
+On Windows
+
+On Linux
+
+Handlers for several URI schemes may be registered separated by a blank.
+There can be only one handler per URI scheme.
+
+The `SofaDataAnnotator` described below illustrates reading Sofa data as
+a stream.
+
+### SofaDataAnnotator
+
+An annotator that accesses the data in Sofa \"EnglishDocument\" as a
+text stream. It tokenizes the data on whitespace and creates an
+annotation for each token. The annotator may be run with an input Sofa
+where the Sofa data is local or with a Sofa where the Sofa data is
+remote and specified as a URI. To run this annotator in C++ to process a
+Sofa with local data:
+
+To run and process Sofa where the Sofa data is specified as a `file:`
+URL, register the example `sofaStreamFileHandler` handler for the `file`
+URI scheme as described in the section above. and run:
+
+### SimpleTextSegmenter
+
+A simple example CAS Multiplier which is a type of analysis component
+that outputs new CASes. This example illustrates one use of the CAS
+Multiplier which is to break down a large CAS into smaller pieces which
+are put into new CASs. The `SimpleTextSegmenter` breaks down the input
+document into segments based on a delimiter and creates a Sofa for each
+segment in a new CAS. The delimiter to use can be specified by setting
+the value of the configuration parameter `DelimiterString` in the
+descriptor. To run this annotator in C++:
+
+The original input CAS as well as the new CASs containing the segments
+will be written out as separate files in your \<yourOutputDir\> in XMI
+format.
+
+### XCasWriterCasConsumer
+
+A simple example CAS Consumer that generates an XCAS (XML representation
+of a CAS) and writes it to stdout by default. It can be configured to
+write the XCAS output to a file in a directory specified by modifying
+the descriptor and setting a value for the configuration parameter
+`OutputDirectory`. The `XCasWriterCasConsumer` can be inserted at any
+point in a aggregate or CPE flow to dump the contents of CAS and is
+useful for debugging. To run this annotator in C++:
+
+To see the results of the earlier examples:
+
+Running the Sample UIMA Applications
+------------------------------------
+
+These illustrate how to write stand-alone C++ applications that run UIMA
+C++ components. Build the examples and set up environment variables as
+described above.
+
+### ExampleApplication
+
+This application reads all the .txt files in a directory, creates a CAS
+for each in turn and sends them through an AnalysisEngine. The results
+are printed on stdout as an XCAS. The application takes two arguments,
+the path to a UIMA C++ descriptor file, and a file or directory
+containing input data:
+
+### SofaExampleApplication
+
+A multiple Sofa example that creates a text Sofa called EnglishDocument
+and sets its Sofa data to some English text and calls the
+SofaExampleAnnotator which produces a Sofa with German text and writes
+the annotations to stdout. This application takes one argument, the path
+to the SofaExampleAnnotator descriptor file:
diff --git a/examples/lib/.gitkeep b/examples/lib/.gitkeep
new file mode 100644
index 0000000..e69de29
diff --git a/examples/scriptators/README.md b/examples/scriptators/README.md
new file mode 100644
index 0000000..373654e
--- /dev/null
+++ b/examples/scriptators/README.md
@@ -0,0 +1,17 @@
+# Scriptator examples
+
+These examples run with the default image by changing the descriptor (to add types) and the script.
+
+They mark spans of multiple lines separated by empty lines on a few Wikipedia pages (in text format).
+
+## Pythonnator
+
+```bash
+sudo docker run -v $PWD/pythonnator/scripts:/usr/local/uimacpp/scripts -v $PWD/data:/data -v $PWD/out:/out -v $PWD/pythonnator/desc:/usr/local/uimacpp/desc apache:uimacpp /usr/local/uimacpp/desc/Pythonnator.xml /data /out
+```
+
+## Perltator
+
+```bash
+docker run -v $PWD/perltator/scripts:/usr/local/uimacpp/scripts -v $PWD/data:/data -v $PWD/out:/out -v $PWD/perltator/desc:/usr/local/uimacpp/desc apache:uimacpp /usr/local/uimacpp/desc/Perltator.xml /data /out
+```
diff --git a/examples/scriptators/data/History_of_natural_language_processing.txt b/examples/scriptators/data/History_of_natural_language_processing.txt
new file mode 100644
index 0000000..c040b0f
--- /dev/null
+++ b/examples/scriptators/data/History_of_natural_language_processing.txt
@@ -0,0 +1,333 @@
+ #alternate Edit this page Wikipedia (en) Wikipedia Atom feed
+
+ Jump to content
+
+ [ ] Main menu
+ Main menu
+ (BUTTON) move to sidebar (BUTTON) hide
+ Navigation
+ * Main page
+ * Contents
+ * Current events
+ * Random article
+ * About Wikipedia
+ * Contact us
+ * Donate
+
+ Contribute
+ * Help
+ * Learn to edit
+ * Community portal
+ * Recent changes
+ * Upload file
+
+ Languages
+ Language links are at the top of the page across from the title.
+ Wikipedia The Free Encyclopedia
+ Search
+ ____________________
+ (BUTTON) Search
+
+ * Create account
+ * Log in
+
+ [ ] Personal tools
+ * Create account
+ * Log in
+
+ Pages for logged out editors learn more
+ * Contributions
+ * Talk
+
+ [ ]
+
+Contents
+
+ (BUTTON) move to sidebar (BUTTON) hide
+ * (Top)
+ * 1Research and development
+ * 2Software
+ * 3References
+ * 4Bibliography
+
+ Toggle the table of contents
+
+ [ ] Toggle the table of contents
+
+History of natural language processing
+
+ [ ] 2 languages
+ * العربية
+ * فارسی
+
+ Edit links
+
+ * Article
+ * Talk
+
+ [ ] English
+
+ * Read
+ * Edit
+ * View history
+
+ [ ] Tools
+ Tools
+ (BUTTON) move to sidebar (BUTTON) hide
+ Actions
+ * Read
+ * Edit
+ * View history
+
+ General
+ * What links here
+ * Related changes
+ * Upload file
+ * Special pages
+ * Permanent link
+ * Page information
+ * Cite this page
+ * Wikidata item
+
+ Print/export
+ * Download as PDF
+ * Printable version
+
+ From Wikipedia, the free encyclopedia
+ This article needs to be updated. Please help update this article to
+ reflect recent events or newly available information. (April 2023)
+ Aspect of history
+
+ The history of natural language processing describes the advances of
+ natural language processing (Outline of natural language processing).
+ There is some overlap with the history of machine translation, the
+ history of speech recognition, and the history of artificial
+ intelligence.
+
+Research and development[edit]
+
+ The history of machine translation dates back to the seventeenth
+ century, when philosophers such as Leibniz and Descartes put forward
+ proposals for codes which would relate words between languages. All of
+ these proposals remained theoretical, and none resulted in the
+ development of an actual machine.
+
+ The first patents for "translating machines" were applied for in the
+ mid-1930s. One proposal, by Georges Artsrouni was simply an automatic
+ bilingual dictionary using paper tape. The other proposal, by Peter
+ Troyanskii, a Russian, was more detailed. It included both the
+ bilingual dictionary, and a method for dealing with grammatical roles
+ between languages, based on Esperanto.
+
+ In 1950, Alan Turing published his famous article "Computing Machinery
+ and Intelligence" which proposed what is now called the Turing test as
+ a criterion of intelligence. This criterion depends on the ability of a
+ computer program to impersonate a human in a real-time written
+ conversation with a human judge, sufficiently well that the judge is
+ unable to distinguish reliably — on the basis of the conversational
+ content alone — between the program and a real human.
+
+ In 1957, Noam Chomsky’s Syntactic Structures revolutionized Linguistics
+ with 'universal grammar', a rule based system of syntactic
+ structures.^[1]
+
+ The Georgetown experiment in 1954 involved fully automatic translation
+ of more than sixty Russian sentences into English. The authors claimed
+ that within three or five years, machine translation would be a solved
+ problem.^[2] However, real progress was much slower, and after the
+ ALPAC report in 1966, which found that ten years long research had
+ failed to fulfill the expectations, funding for machine translation was
+ dramatically reduced. Little further research in machine translation
+ was conducted until the late 1980s, when the first statistical machine
+ translation systems were developed.
+
+ Some notably successful NLP systems developed in the 1960s were SHRDLU,
+ a natural language system working in restricted "blocks worlds" with
+ restricted vocabularies.
+
+ In 1969 Roger Schank introduced the conceptual dependency theory for
+ natural language understanding.^[3] This model, partially influenced by
+ the work of Sydney Lamb, was extensively used by Schank's students at
+ Yale University, such as Robert Wilensky, Wendy Lehnert, and Janet
+ Kolodner.
+
+ In 1970, William A. Woods introduced the augmented transition network
+ (ATN) to represent natural language input.^[4] Instead of phrase
+ structure rules ATNs used an equivalent set of finite state automata
+ that were called recursively. ATNs and their more general format called
+ "generalized ATNs" continued to be used for a number of years. During
+ the 1970s many programmers began to write 'conceptual ontologies',
+ which structured real-world information into computer-understandable
+ data. Examples are MARGIE (Schank, 1975), SAM (Cullingford, 1978), PAM
+ (Wilensky, 1978), TaleSpin (Meehan, 1976), QUALM (Lehnert, 1977),
+ Politics (Carbonell, 1979), and Plot Units (Lehnert 1981). During this
+ time, many chatterbots were written including PARRY, Racter, and
+ Jabberwacky.
+
+ Up to the 1980s, most NLP systems were based on complex sets of
+ hand-written rules. Starting in the late 1980s, however, there was a
+ revolution in NLP with the introduction of machine learning algorithms
+ for language processing. This was due both to the steady increase in
+ computational power resulting from Moore's Law and the gradual
+ lessening of the dominance of Chomskyan theories of linguistics (e.g.
+ transformational grammar), whose theoretical underpinnings discouraged
+ the sort of corpus linguistics that underlies the machine-learning
+ approach to language processing.^[5] Some of the earliest-used machine
+ learning algorithms, such as decision trees, produced systems of hard
+ if-then rules similar to existing hand-written rules. Increasingly,
+ however, research has focused on statistical models, which make soft,
+ probabilistic decisions based on attaching real-valued weights to the
+ features making up the input data. The cache language models upon which
+ many speech recognition systems now rely are examples of such
+ statistical models. Such models are generally more robust when given
+ unfamiliar input, especially input that contains errors (as is very
+ common for real-world data), and produce more reliable results when
+ integrated into a larger system comprising multiple subtasks.
+
+ Many of the notable early successes occurred in the field of machine
+ translation, due especially to work at IBM Research, where successively
+ more complicated statistical models were developed. These systems were
+ able to take advantage of existing multilingual textual corpora that
+ had been produced by the Parliament of Canada and the European Union as
+ a result of laws calling for the translation of all governmental
+ proceedings into all official languages of the corresponding systems of
+ government. However, most other systems depended on corpora
+ specifically developed for the tasks implemented by these systems,
+ which was (and often continues to be) a major limitation in the success
+ of these systems. As a result, a great deal of research has gone into
+ methods of more effectively learning from limited amounts of data.
+
+ Recent research has increasingly focused on unsupervised and
+ semi-supervised learning algorithms. Such algorithms are able to learn
+ from data that has not been hand-annotated with the desired answers, or
+ using a combination of annotated and non-annotated data. Generally,
+ this task is much more difficult than supervised learning, and
+ typically produces less accurate results for a given amount of input
+ data. However, there is an enormous amount of non-annotated data
+ available (including, among other things, the entire content of the
+ World Wide Web), which can often make up for the inferior results.
+
+Software[edit]
+
+ Software Year Creator Description Reference
+ Georgetown experiment 1954 Georgetown University and IBM involved fully
+ automatic translation of more than sixty Russian sentences into
+ English.
+ STUDENT 1964 Daniel Bobrow could solve high school algebra word
+ problems.^[6]
+ ELIZA 1964 Joseph Weizenbaum a simulation of a Rogerian
+ psychotherapist, rephrasing her response with a few grammar rules.^[7]
+ SHRDLU 1970 Terry Winograd a natural language system working in
+ restricted "blocks worlds" with restricted vocabularies, worked
+ extremely well
+ PARRY 1972 Kenneth Colby A chatterbot
+ KL-ONE 1974 Sondheimer et al. a knowledge representation system in the
+ tradition of semantic networks and frames; it is a frame language.
+ MARGIE 1975 Roger Schank
+ TaleSpin (software) 1976 Meehan
+ QUALM Lehnert
+ LIFER/LADDER 1978 Hendrix a natural language interface to a database of
+ information about US Navy ships.
+ SAM (software) 1978 Cullingford
+ PAM (software) 1978 Robert Wilensky
+ Politics (software) 1979 Carbonell
+ Plot Units (software) 1981 Lehnert
+ Jabberwacky 1982 Rollo Carpenter chatterbot with stated aim to
+ "simulate natural human chat in an interesting, entertaining and
+ humorous manner".
+ MUMBLE (software) 1982 McDonald
+ Racter 1983 William Chamberlain and Thomas Etter chatterbot that
+ generated English language prose at random.
+ MOPTRANS ^[8] 1984 Lytinen
+ KODIAK (software) 1986 Wilensky
+ Absity (software) 1987 Hirst
+ Dr. Sbaitso 1991 Creative Labs
+ Watson (artificial intelligence software) 2006 IBM A question answering
+ system that won the Jeopardy! contest, defeating the best human players
+ in February 2011.
+ Siri 2011 Apple A virtual assistant developed by Apple.
+ Cortana 2014 Microsoft A virtual assistant developed by Microsoft.
+ Amazon Alexa 2014 Amazon A virtual assistant developed by Amazon.
+ Google Assistant 2016 Google A virtual assistant developed by Google.
+
+References[edit]
+
+ 1. ^ "SEM1A5 - Part 1 - A brief history of NLP". Retrieved 2010-06-25.
+ 2. ^ Hutchins, J. (2005)
+ 3. ^ Roger Schank, 1969, A conceptual dependency parser for natural
+ language Proceedings of the 1969 conference on Computational
+ linguistics, Sång-Säby, Sweden, pages 1-3
+ 4. ^ Woods, William A (1970). "Transition Network Grammars for Natural
+ Language Analysis". Communications of the ACM 13 (10): 591–606 [1]
+ 5. ^ Chomskyan linguistics encourages the investigation of "corner
+ cases" that stress the limits of its theoretical models (comparable
+ to pathological phenomena in mathematics), typically created using
+ thought experiments, rather than the systematic investigation of
+ typical phenomena that occur in real-world data, as is the case in
+ corpus linguistics. The creation and use of such corpora of
+ real-world data is a fundamental part of machine-learning
+ algorithms for NLP. In addition, theoretical underpinnings of
+ Chomskyan linguistics such as the so-called "poverty of the
+ stimulus" argument entail that general learning algorithms, as are
+ typically used in machine learning, cannot be successful in
+ language processing. As a result, the Chomskyan paradigm
+ discouraged the application of such models to language processing.
+ 6. ^ McCorduck 2004, p. 286, Crevier 1993, pp. 76−79, Russell & Norvig
+ 2003, p. 19
+ 7. ^ McCorduck 2004, pp. 291–296, Crevier 1993, pp. 134−139
+ 8. ^ Janet L. Kolodner, Christopher K. Riesbeck; Experience, Memory,
+ and Reasoning; Psychology Press; 2014 reprint
+
+Bibliography[edit]
+
+ *
+
+ Crevier, Daniel (1993). AI: The Tumultuous Search for Artificial
+ Intelligence. New York, NY: BasicBooks. ISBN 0-465-02997-3.
+
+ McCorduck, Pamela (2004), Machines Who Think (2nd ed.), Natick, MA:
+ A. K. Peters, Ltd., ISBN 978-1-56881-205-2, OCLC 52197627.
+
+
+
+ Russell, Stuart J.; Norvig, Peter (2003), Artificial Intelligence: A
+ Modern Approach (2nd ed.), Upper Saddle River, New Jersey: Prentice
+ Hall, ISBN 0-13-790395-2.
+
+ Retrieved from
+ "https://en.wikipedia.org/w/index.php?title=History_of_natural_language
+ _processing&oldid=1152004616"
+
+ Categories:
+ * History of artificial intelligence
+ * Natural language processing
+ * History of linguistics
+ * History of software
+
+ Hidden categories:
+ * Wikipedia articles in need of updating from April 2023
+ * All Wikipedia articles in need of updating
+ * Articles with short description
+ * Short description matches Wikidata
+
+ * This page was last edited on 27 April 2023, at 15:39 (UTC).
+ * Text is available under the Creative Commons Attribution-ShareAlike
+ License 4.0; additional terms may apply. By using this site, you
+ agree to the Terms of Use and Privacy Policy. Wikipedia® is a
+ registered trademark of the Wikimedia Foundation, Inc., a
+ non-profit organization.
+
+ * Privacy policy
+ * About Wikipedia
+ * Disclaimers
+ * Contact Wikipedia
+ * Code of Conduct
+ * Mobile view
+ * Developers
+ * Statistics
+ * Cookie statement
+
+ * Wikimedia Foundation
+ * Powered by MediaWiki
+
+ (BUTTON) Toggle limited content width
diff --git a/examples/scriptators/data/IBM_Watson.txt b/examples/scriptators/data/IBM_Watson.txt
new file mode 100644
index 0000000..5a67a71
--- /dev/null
+++ b/examples/scriptators/data/IBM_Watson.txt
@@ -0,0 +1,1935 @@
+ #alternate Edit this page Wikipedia (en) Wikipedia Atom feed
+
+ Jump to content
+
+ [ ] Main menu
+ Main menu
+ (BUTTON) move to sidebar (BUTTON) hide
+ Navigation
+ * Main page
+ * Contents
+ * Current events
+ * Random article
+ * About Wikipedia
+ * Contact us
+ * Donate
+
+ Contribute
+ * Help
+ * Learn to edit
+ * Community portal
+ * Recent changes
+ * Upload file
+
+ Languages
+ Language links are at the top of the page across from the title.
+ Wikipedia The Free Encyclopedia
+ Search
+ ____________________
+ (BUTTON) Search
+
+ * Create account
+ * Log in
+
+ [ ] Personal tools
+ * Create account
+ * Log in
+
+ Pages for logged out editors learn more
+ * Contributions
+ * Talk
+
+ [ ]
+
+Contents
+
+ (BUTTON) move to sidebar (BUTTON) hide
+ * (Top)
+ * 1Description
+ (BUTTON) Toggle Description subsection
+ + 1.1Software
+ + 1.2Hardware
+ + 1.3Data
+ * 2Operation
+ (BUTTON) Toggle Operation subsection
+ + 2.1Comparison with human players
+ * 3History
+ (BUTTON) Toggle History subsection
+ + 3.1Development
+ + 3.2Jeopardy!
+ o 3.2.1Preparation
+ o 3.2.2Practice match
+ o 3.2.3First match
+ o 3.2.4Second match
+ o 3.2.5Final outcome
+ o 3.2.6Philosophy
+ o 3.2.7Match against members of the United States Congress
+ * 4Current and future applications
+ (BUTTON) Toggle Current and future applications subsection
+ + 4.1Healthcare
+ + 4.2IBM Watson Group
+ + 4.3Chef Watson
+ + 4.4Chatbot
+ + 4.5Building codes
+ + 4.6Teaching assistant
+ + 4.7Weather forecasting
+ + 4.8Fashion
+ + 4.9Tax preparation
+ + 4.10Advertising
+ * 5See also
+ * 6References
+ * 7Bibliography
+ * 8Further reading
+ * 9External links
+ (BUTTON) Toggle External links subsection
+ + 9.1J! Archive
+ + 9.2Videos
+
+ Toggle the table of contents
+
+ [ ] Toggle the table of contents
+
+IBM Watson
+
+ [ ] 33 languages
+ * العربية
+ * Български
+ * Català
+ * Deutsch
+ * Eesti
+ * Español
+ * Euskara
+ * فارسی
+ * Français
+ * 한국어
+ * Հայերեն
+ * Italiano
+ * עברית
+ * Magyar
+ * മലയാളം
+ * မြန်မာဘာသာ
+ * Nederlands
+ * 日本語
+ * Norsk bokmål
+ * Oʻzbekcha / ўзбекча
+ * Polski
+ * Português
+ * Română
+ * Русский
+ * Simple English
+ * Slovenčina
+ * Suomi
+ * Svenska
+ * Türkçe
+ * Українська
+ * Tiếng Việt
+ * 吴语
+ * 中文
+
+ Edit links
+
+ * Article
+ * Talk
+
+ [ ] English
+
+ * Read
+ * Edit
+ * View history
+
+ [ ] Tools
+ Tools
+ (BUTTON) move to sidebar (BUTTON) hide
+ Actions
+ * Read
+ * Edit
+ * View history
+
+ General
+ * What links here
+ * Related changes
+ * Upload file
+ * Special pages
+ * Permanent link
+ * Page information
+ * Cite this page
+ * Wikidata item
+
+ Print/export
+ * Download as PDF
+ * Printable version
+
+ In other projects
+ * Wikimedia Commons
+
+ From Wikipedia, the free encyclopedia
+ Artificial intelligence computer system made by IBM
+ For the IBM laboratory, see Thomas J. Watson Research Center.
+ [220px-IBM_Watson_Logo_2017.png] Watson's avatar, inspired by the IBM
+ "Smarter Planet" logo
+
+ IBM Watson is a question-answering computer system capable of answering
+ questions posed in natural language,^[1] developed in IBM's DeepQA
+ project by a research team led by principal investigator David
+ Ferrucci.^[2] Watson was named after IBM's founder and first CEO,
+ industrialist Thomas J. Watson.^[3]^[4]
+
+ The computer system was initially developed to answer questions on the
+ quiz show Jeopardy!^[5] and in 2011, the Watson computer system
+ competed on Jeopardy! against champions Brad Rutter and Ken
+ Jennings,^[3]^[6] winning the first place prize of 1 million USD.^[7]
+
+ In February 2013, IBM announced that Watson's first commercial
+ application would be for utilization management decisions in lung
+ cancer treatment at Memorial Sloan Kettering Cancer Center, New York
+ City, in conjunction with WellPoint (now Elevance Health).^[8].
+
+Description[edit]
+
+ [220px-DeepQA.svg.png] The high-level architecture of IBM's DeepQA used
+ in Watson^[9]
+
+ Watson was created as a question answering (QA) computing system that
+ IBM built to apply advanced natural language processing, information
+ retrieval, knowledge representation, automated reasoning, and machine
+ learning technologies to the field of open domain question
+ answering.^[1]
+
+ IBM stated that Watson uses "more than 100 different techniques to
+ analyze natural language, identify sources, find and generate
+ hypotheses, find and score evidence, and merge and rank
+ hypotheses."^[10]
+
+ In recent years, Watson's capabilities have been extended and the way
+ in which Watson works has been changed to take advantage of new
+ deployment models (Watson on IBM Cloud), evolved machine learning
+ capabilities, and optimized hardware available to developers and
+ researchers. It is no longer purely a question answering (QA) computing
+ system designed from Q&A pairs but can now 'see', 'hear', 'read',
+ 'talk', 'taste', 'interpret', 'learn' and 'recommend'.^[citation
+ needed]
+
+Software[edit]
+
+ Watson uses IBM's DeepQA software and the Apache UIMA (Unstructured
+ Information Management Architecture) framework implementation. The
+ system was written in various languages, including Java, C++, and
+ Prolog, and runs on the SUSE Linux Enterprise Server 11 operating
+ system using the Apache Hadoop framework to provide distributed
+ computing.^[11]^[12]^[13]
+
+Hardware[edit]
+
+ The system is workload-optimized, integrating massively parallel POWER7
+ processors and built on IBM's DeepQA technology,^[14] which it uses to
+ generate hypotheses, gather massive evidence, and analyze data.^[1]
+ Watson employs a cluster of ninety IBM Power 750 servers, each of which
+ uses a 3.5 GHz POWER7 eight-core processor, with four threads per core.
+ In total, the system has 2,880 POWER7 processor threads and 16
+ terabytes of RAM.^[14]
+
+ According to John Rennie, Watson can process 500 gigabytes (the
+ equivalent of a million books) per second.^[15] IBM master inventor and
+ senior consultant Tony Pearson estimated Watson's hardware cost at
+ about three million dollars.^[16] Its Linpack performance stands at 80
+ TeraFLOPs, which is about half as fast as the cut-off line for the Top
+ 500 Supercomputers list.^[17] According to Rennie, all content was
+ stored in Watson's RAM for the Jeopardy game because data stored on
+ hard drives would be too slow to compete with human Jeopardy
+ champions.^[15]
+
+Data[edit]
+
+ The sources of information for Watson include encyclopedias,
+ dictionaries, thesauri, newswire articles and literary works. Watson
+ also used databases, taxonomies and ontologies including DBPedia,
+ WordNet and Yago.^[18] The IBM team provided Watson with millions of
+ documents, including dictionaries, encyclopedias and other reference
+ material, that it could use to build its knowledge.^[19]
+
+Operation[edit]
+
+ Watson parses questions into different keywords and sentence fragments
+ in order to find statistically related phrases.^[19] Watson's main
+ innovation was not in the creation of a new algorithm for this
+ operation but rather its ability to quickly execute hundreds of proven
+ language analysis algorithms simultaneously.^[19]^[20] The more
+ algorithms that find the same answer independently, the more likely
+ Watson is to be correct. Once Watson has a small number of potential
+ solutions, it is able to check against its database to ascertain
+ whether the solution makes sense or not.^[19]
+
+Comparison with human players[edit]
+
+ [220px-Watson_Jeopardy.jpg] Ken Jennings, Watson, and Brad Rutter in
+ their Jeopardy! exhibition match
+
+ Watson's basic working principle is to parse keywords in a clue while
+ searching for related terms as responses. This gives Watson some
+ advantages and disadvantages compared with human Jeopardy!
+ players.^[21] Watson has deficiencies in understanding the contexts of
+ the clues. Watson can read, analyze, and learn from natural language
+ which gives it the ability to make human-like decisions.^[22] As a
+ result, human players usually generate responses faster than Watson,
+ especially to short clues.^[19] Watson's programming prevents it from
+ using the popular tactic of buzzing before it is sure of its
+ response.^[19] However, Watson has consistently better reaction time on
+ the buzzer once it has generated a response, and is immune to human
+ players' psychological tactics, such as jumping between categories on
+ every clue.^[19]^[23]
+
+ In a sequence of 20 mock games of Jeopardy, human participants were
+ able to use the six to seven seconds that Watson needed to hear the
+ clue and decide whether to signal for responding.^[19] During that
+ time, Watson also has to evaluate the response and determine whether it
+ is sufficiently confident in the result to signal.^[19] Part of the
+ system used to win the Jeopardy! contest was the electronic circuitry
+ that receives the "ready" signal and then examined whether Watson's
+ confidence level was great enough to activate the buzzer. Given the
+ speed of this circuitry compared to the speed of human reaction times,
+ Watson's reaction time was faster than the human contestants except
+ when the human anticipated (instead of reacted to) the ready
+ signal.^[24] After signaling, Watson speaks with an electronic voice
+ and gives the responses in Jeopardy!'s question format.^[19] Watson's
+ voice was synthesized from recordings that actor Jeff Woodman made for
+ an IBM text-to-speech program in 2004.^[25]
+
+ The Jeopardy! staff used different means to notify Watson and the human
+ players when to buzz,^[24] which was critical in many rounds.^[23] The
+ humans were notified by a light, which took them tenths of a second to
+ perceive.^[26]^[27] Watson was notified by an electronic signal and
+ could activate the buzzer within about eight milliseconds.^[28] The
+ humans tried to compensate for the perception delay by anticipating the
+ light,^[29] but the variation in the anticipation time was generally
+ too great to fall within Watson's response time.^[23] Watson did not
+ attempt to anticipate the notification signal.^[27]^[29]
+
+History[edit]
+
+Development[edit]
+
+ Since Deep Blue's victory over Garry Kasparov in chess in 1997, IBM had
+ been on the hunt for a new challenge. In 2004, IBM Research manager
+ Charles Lickel, over dinner with coworkers, noticed that the restaurant
+ they were in had fallen silent. He soon discovered the cause of this
+ evening's hiatus: Ken Jennings, who was then in the middle of his
+ successful 74-game run on Jeopardy!. Nearly the entire restaurant had
+ piled toward the televisions, mid-meal, to watch Jeopardy!. Intrigued
+ by the quiz show as a possible challenge for IBM, Lickel passed the
+ idea on, and in 2005, IBM Research executive Paul Horn supported
+ Lickel, pushing for someone in his department to take up the challenge
+ of playing Jeopardy! with an IBM system. Though he initially had
+ trouble finding any research staff willing to take on what looked to be
+ a much more complex challenge than the wordless game of chess,
+ eventually David Ferrucci took him up on the offer.^[30] In
+ competitions managed by the United States government, Watson's
+ predecessor, a system named Piquant, was usually able to respond
+ correctly to only about 35% of clues and often required several minutes
+ to respond.^[31]^[32]^[33] To compete successfully on Jeopardy!, Watson
+ would need to respond in no more than a few seconds, and at that time,
+ the problems posed by the game show were deemed to be impossible to
+ solve.^[19]
+
+ In initial tests run during 2006 by David Ferrucci, the senior manager
+ of IBM's Semantic Analysis and Integration department, Watson was given
+ 500 clues from past Jeopardy! programs. While the best real-life
+ competitors buzzed in half the time and responded correctly to as many
+ as 95% of clues, Watson's first pass could get only about 15% correct.
+ During 2007, the IBM team was given three to five years and a staff of
+ 15 people to solve the problems.^[19] John E. Kelly III succeeded Paul
+ Horn as head of IBM Research in 2007.^[34] InformationWeek described
+ Kelly as "the father of Watson" and credited him for encouraging the
+ system to compete against humans on Jeopardy!.^[35] By 2008, the
+ developers had advanced Watson such that it could compete with
+ Jeopardy! champions.^[19] By February 2010, Watson could beat human
+ Jeopardy! contestants on a regular basis.^[36]
+
+ During the game, Watson had access to 200 million pages of structured
+ and unstructured content consuming four terabytes of disk storage^[11]
+ including the full text of the 2011 edition of Wikipedia,^[37] but was
+ not connected to the Internet.^[38]^[19] For each clue, Watson's three
+ most probable responses were displayed on the television screen. Watson
+ consistently outperformed its human opponents on the game's signaling
+ device, but had trouble in a few categories, notably those having short
+ clues containing only a few words.
+
+ Although the system is primarily an IBM effort, Watson's development
+ involved faculty and graduate students from Rensselaer Polytechnic
+ Institute, Carnegie Mellon University, University of Massachusetts
+ Amherst, the University of Southern California's Information Sciences
+ Institute, the University of Texas at Austin, the Massachusetts
+ Institute of Technology, and the University of Trento,^[9] as well as
+ students from New York Medical College.^[39] Among the team of IBM
+ programmers who worked on Watson was Ed Toutant, who himself had
+ appeared on Jeopardy! in 1989 (winning one game).^[40]
+
+Jeopardy![edit]
+
+Preparation[edit]
+
+ [220px-IBMWatson.jpg] Watson demo at an IBM booth at a trade show
+
+ In 2008, IBM representatives communicated with Jeopardy! executive
+ producer Harry Friedman about the possibility of having Watson compete
+ against Ken Jennings and Brad Rutter, two of the most successful
+ contestants on the show, and the program's producers agreed.^[19]^[41]
+ Watson's differences with human players had generated conflicts between
+ IBM and Jeopardy! staff during the planning of the competition.^[21]
+ IBM repeatedly expressed concerns that the show's writers would exploit
+ Watson's cognitive deficiencies when writing the clues, thereby turning
+ the game into a Turing test. To alleviate that claim, a third party
+ randomly picked the clues from previously written shows that were never
+ broadcast.^[21] Jeopardy! staff also showed concerns over Watson's
+ reaction time on the buzzer. Originally Watson signaled electronically,
+ but show staff requested that it press a button physically, as the
+ human contestants would.^[42] Even with a robotic "finger" pressing the
+ buzzer, Watson remained faster than its human competitors. Ken Jennings
+ noted, "If you're trying to win on the show, the buzzer is all", and
+ that Watson "can knock out a microsecond-precise buzz every single time
+ with little or no variation. Human reflexes can't compete with computer
+ circuits in this regard."^[23]^[29]^[43] Stephen Baker, a journalist
+ who recorded Watson's development in his book Final Jeopardy, reported
+ that the conflict between IBM and Jeopardy! became so serious in May
+ 2010 that the competition was almost cancelled.^[21] As part of the
+ preparation, IBM constructed a mock set in a conference room at one of
+ its technology sites to model the one used on Jeopardy!. Human players,
+ including former Jeopardy! contestants, also participated in mock games
+ against Watson with Todd Alan Crain of The Onion playing host.^[19]
+ About 100 test matches were conducted with Watson winning 65% of the
+ games.^[44]
+
+ To provide a physical presence in the televised games, Watson was
+ represented by an "avatar" of a globe, inspired by the IBM "smarter
+ planet" symbol. Jennings described the computer's avatar as a "glowing
+ blue ball crisscrossed by 'threads' of thought—42 threads, to be
+ precise",^[45] and stated that the number of thought threads in the
+ avatar was an in-joke referencing the significance of the number 42 in
+ Douglas Adams' Hitchhiker's Guide to the Galaxy.^[45] Joshua Davis, the
+ artist who designed the avatar for the project, explained to Stephen
+ Baker that there are 36 trigger-able states that Watson was able to use
+ throughout the game to show its confidence in responding to a clue
+ correctly; he had hoped to be able to find forty-two, to add another
+ level to the Hitchhiker's Guide reference, but he was unable to
+ pinpoint enough game states.^[46]
+
+ A practice match was recorded on January 13, 2011, and the official
+ matches were recorded on January 14, 2011. All participants maintained
+ secrecy about the outcome until the match was broadcast in
+ February.^[47]
+
+Practice match[edit]
+
+ In a practice match before the press on January 13, 2011, Watson won a
+ 15-question round against Ken Jennings and Brad Rutter with a score of
+ $4,400 to Jennings's $3,400 and Rutter's $1,200, though Jennings and
+ Watson were tied before the final $1,000 question. None of the three
+ players responded incorrectly to a clue.^[48]
+
+First match[edit]
+
+ The first round was broadcast February 14, 2011, and the second round,
+ on February 15, 2011. The right to choose the first category had been
+ determined by a draw won by Rutter.^[49] Watson, represented by a
+ computer monitor display and artificial voice, responded correctly to
+ the second clue and then selected the fourth clue of the first
+ category, a deliberate strategy to find the Daily Double as quickly as
+ possible.^[50] Watson's guess at the Daily Double location was correct.
+ At the end of the first round, Watson was tied with Rutter at $5,000;
+ Jennings had $2,000.^[49]
+
+ Watson's performance was characterized by some quirks. In one instance,
+ Watson repeated a reworded version of an incorrect response offered by
+ Jennings. (Jennings said "What are the '20s?" in reference to the
+ 1920s. Then Watson said "What is 1920s?") Because Watson could not
+ recognize other contestants' responses, it did not know that Jennings
+ had already given the same response. In another instance, Watson was
+ initially given credit for a response of "What is a leg?" after
+ Jennings incorrectly responded "What is: he only had one hand?" to a
+ clue about George Eyser (the correct response was, "What is: he's
+ missing a leg?"). Because Watson, unlike a human, could not have been
+ responding to Jennings's mistake, it was decided that this response was
+ incorrect. The broadcast version of the episode was edited to omit
+ Trebek's original acceptance of Watson's response.^[51] Watson also
+ demonstrated complex wagering strategies on the Daily Doubles, with one
+ bet at $6,435 and another at $1,246.^[52] Gerald Tesauro, one of the
+ IBM researchers who worked on Watson, explained that Watson's wagers
+ were based on its confidence level for the category and a complex
+ regression model called the Game State Evaluator.^[53]
+
+ Watson took a commanding lead in Double Jeopardy!, correctly responding
+ to both Daily Doubles. Watson responded to the second Daily Double
+ correctly with a 32% confidence score.^[52]
+
+ However, during the Final Jeopardy! round, Watson was the only
+ contestant to miss the clue in the category U.S. Cities ("Its largest
+ airport was named for a World War II hero; its second largest, for a
+ World War II battle"). Rutter and Jennings gave the correct response of
+ Chicago, but Watson's response was "What is Toronto?????" with five
+ question marks appended indicating a lack of confidence.^[52]^[54]^[55]
+ Ferrucci offered reasons why Watson would appear to have guessed a
+ Canadian city: categories only weakly suggest the type of response
+ desired, the phrase "U.S. city" did not appear in the question, there
+ are cities named Toronto in the U.S., and Toronto in Ontario has an
+ American League baseball team.^[56] Chris Welty, who also worked on
+ Watson, suggested that it may not have been able to correctly parse the
+ second part of the clue, "its second largest, for a World War II
+ battle" (which was not a standalone clause despite it following a
+ semicolon, and required context to understand that it was referring to
+ a second-largest airport).^[57] Eric Nyberg, a professor at Carnegie
+ Mellon University and a member of the development team, stated that the
+ error occurred because Watson does not possess the comparative
+ knowledge to discard that potential response as not viable.^[55]
+ Although not displayed to the audience as with non-Final Jeopardy!
+ questions, Watson's second choice was Chicago. Both Toronto and Chicago
+ were well below Watson's confidence threshold, at 14% and 11%
+ respectively. Watson wagered only $947 on the question.
+
+ The game ended with Jennings with $4,800, Rutter with $10,400, and
+ Watson with $35,734.^[52]
+
+Second match[edit]
+
+ During the introduction, Trebek (a Canadian native) joked that he had
+ learned Toronto was a U.S. city, and Watson's error in the first match
+ prompted an IBM engineer to wear a Toronto Blue Jays jacket to the
+ recording of the second match.^[58]
+
+ In the first round, Jennings was finally able to choose a Daily Double
+ clue,^[59] while Watson responded to one Daily Double clue incorrectly
+ for the first time in the Double Jeopardy! Round.^[60] After the first
+ round, Watson placed second for the first time in the competition after
+ Rutter and Jennings were briefly successful in increasing their dollar
+ values before Watson could respond.^[60]^[61] Nonetheless, the final
+ result ended with a victory for Watson with a score of $77,147, besting
+ Jennings who scored $24,000 and Rutter who scored $21,600.^[62]
+
+Final outcome[edit]
+
+ The prizes for the competition were $1 million for first place
+ (Watson), $300,000 for second place (Jennings), and $200,000 for third
+ place (Rutter). As promised, IBM donated 100% of Watson's winnings to
+ charity, with 50% of those winnings going to World Vision and 50% going
+ to World Community Grid.^[63] Similarly, Jennings and Rutter donated
+ 50% of their winnings to their respective charities.^[64]
+
+ In acknowledgement of IBM and Watson's achievements, Jennings made an
+ additional remark in his Final Jeopardy! response: "I for one welcome
+ our new computer overlords."^[65]^[66] Jennings later wrote an article
+ for Slate, in which he stated:
+
+ IBM has bragged to the media that Watson's question-answering skills
+ are good for more than annoying Alex Trebek. The company sees a
+ future in which fields like medical diagnosis, business analytics,
+ and tech support are automated by question-answering software like
+ Watson. Just as factory jobs were eliminated in the 20th century by
+ new assembly-line robots, Brad and I were the first
+ knowledge-industry workers put out of work by the new generation of
+ 'thinking' machines. 'Quiz show contestant' may be the first job
+ made redundant by Watson, but I'm sure it won't be the last.^[45]
+
+Philosophy[edit]
+
+ Philosopher John Searle argues that Watson—despite impressive
+ capabilities—cannot actually think.^[67] Drawing on his Chinese room
+ thought experiment, Searle claims that Watson, like other computational
+ machines, is capable only of manipulating symbols, but has no ability
+ to understand the meaning of those symbols; however, Searle's
+ experiment has its detractors.^[68]
+
+Match against members of the United States Congress[edit]
+
+ On February 28, 2011, Watson played an untelevised exhibition match of
+ Jeopardy! against members of the United States House of
+ Representatives. In the first round, Rush D. Holt, Jr. (D-NJ, a former
+ Jeopardy! contestant), who was challenging the computer with Bill
+ Cassidy (R-LA, later Senator from Louisiana), led with Watson in second
+ place. However, combining the scores between all matches, the final
+ score was $40,300 for Watson and $30,000 for the congressional players
+ combined.^[69]
+
+ IBM's Christopher Padilla said of the match, "The technology behind
+ Watson represents a major advancement in computing. In the
+ data-intensive environment of government, this type of technology can
+ help organizations make better decisions and improve how government
+ helps its citizens."^[69]
+
+Current and future applications[edit]
+
+ This section contains content that is written like an advertisement.
+ Please help improve it by removing promotional content and
+ inappropriate external links, and by adding encyclopedic content
+ written from a neutral point of view. (April 2019) (Learn how and when
+ to remove this template message)
+
+ According to IBM, "The goal is to have computers start to interact in
+ natural human terms across a range of applications and processes,
+ understanding the questions that humans ask and providing answers that
+ humans can understand and justify."^[36] It has been suggested by
+ Robert C. Weber, IBM's general counsel, that Watson may be used for
+ legal research.^[70] The company also intends to use Watson in other
+ information-intensive fields, such as telecommunications, financial
+ services, and government.^[71]
+
+ Watson is based on commercially available IBM Power 750 servers that
+ have been marketed since February 2010.^[19]
+
+ Commentator Rick Merritt said that "there's another really important
+ reason why it is strategic for IBM to be seen very broadly by the
+ American public as a company that can tackle tough computer problems. A
+ big slice of [IBM's profit] comes from selling to the U.S. government
+ some of the biggest, most expensive systems in the world."^[72]
+
+ In 2013, it was reported that three companies were working with IBM to
+ create apps embedded with Watson technology. Fluid is developing an app
+ for retailers, one called "The North Face", which is designed to
+ provide advice to online shoppers. Welltok is developing an app
+ designed to give people advice on ways to engage in activities to
+ improve their health. MD Buyline is developing an app for the purpose
+ of advising medical institutions on equipment procurement
+ decisions.^[73]^[74]
+
+ In November 2013, IBM announced it would make Watson's API available to
+ software application providers, enabling them to build apps and
+ services that are embedded in Watson's capabilities. To build out its
+ base of partners who create applications on the Watson platform, IBM
+ consults with a network of venture capital firms, which advise IBM on
+ which of their portfolio companies may be a logical fit for what IBM
+ calls the Watson Ecosystem. Thus far, roughly 800 organizations and
+ individuals have signed up with IBM, with interest in creating
+ applications that could use the Watson platform.^[75]
+
+ On January 30, 2013, it was announced that Rensselaer Polytechnic
+ Institute would receive a successor version of Watson, which would be
+ housed at the Institute's technology park and be available to
+ researchers and students.^[76] By summer 2013, Rensselaer had become
+ the first university to receive a Watson computer.^[77]
+
+ On February 6, 2014, it was reported that IBM plans to invest $100
+ million in a 10-year initiative to use Watson and other IBM
+ technologies to help countries in Africa address development problems,
+ beginning with healthcare and education.^[78]
+
+ On June 3, 2014, three new Watson Ecosystem partners were chosen from
+ more than 400 business concepts submitted by teams spanning 18
+ industries from 43 countries. "These bright and enterprising
+ organizations have discovered innovative ways to apply Watson that can
+ deliver demonstrable business benefits", said Steve Gold, vice
+ president, IBM Watson Group. The winners were Majestyk Apps with their
+ adaptive educational platform, FANG (Friendly Anthropomorphic Networked
+ Genome);^[79]^[80] Red Ant with their retail sales trainer;^[81] and
+ GenieMD^[82] with their medical recommendation service.^[83]
+
+ On July 9, 2014, Genesys Telecommunications Laboratories announced
+ plans to integrate Watson to improve their customer experience
+ platform, citing the sheer volume of customer data to analyze.^[84]
+
+ Watson has been integrated with databases including Bon Appétit
+ magazine to perform a recipe generating platform.^[85]
+
+ Watson is being used by Decibel, a music discovery startup, in its app
+ MusicGeek which uses the supercomputer to provide music recommendations
+ to its users. The use of Watson has also been found in the hospitality
+ industry. Go Moment uses Watson for its Rev1 app, which gives hotel
+ staff a way to quickly respond to questions from guests.^[86] Arria NLG
+ has built an app that helps energy companies stay within regulatory
+ guidelines, making it easier for managers to make sense of thousands of
+ pages of legal and technical jargon.
+
+ OmniEarth, Inc. uses Watson computer vision services to analyze
+ satellite and aerial imagery, along with other municipal data, to infer
+ water usage on a property-by-property basis, helping districts in
+ California improve water conservation efforts.^[87]
+
+ In September 2016, Condé Nast started using Watson to help build and
+ strategize social influencer campaigns for brands. Using software built
+ by IBM and Influential, Condé Nast's clients will be able to know which
+ influencer's demographics, personality traits and more best align with
+ a marketer and the audience it is targeting.^[88]
+
+ In February 2017, Rare Carat, a New York City-based startup and
+ e-commerce platform for buying diamonds and diamond rings, introduced
+ an IBM Watson-powered artificial intelligence chatbot called "Rocky" to
+ assist novice diamond buyers through the daunting process of purchasing
+ a diamond. As part of the IBM Global Entrepreneur Program, Rare Carat
+ received the assistance of IBM in the development of the Rocky Chat
+ Bot.^[89]^[90]^[91] In May 2017, IBM partnered with the Pebble Beach
+ Company to use Watson as a concierge.^[92] Watson's artificial
+ intelligence was added to an app developed by Pebble Beach and was used
+ to guide visitors around the resort. The mobile app was designed by IBM
+ iX and hosted on the IBM Cloud. It uses Watson's Conversation
+ applications programming interface.
+
+ In November 2017, in Mexico City, the Experience Voices of Another Time
+ was opened at the National Museum of Anthropology using IBM Watson as
+ an alternative to visiting a museum.^[93]
+
+Healthcare[edit]
+
+ See also: IBM Watson Health
+
+ In healthcare, Watson has been used to analyze medical data and assist
+ doctors in making diagnoses and treatment decisions, including in areas
+ such as oncology and radiology.^[94] Watson's natural language,
+ hypothesis generation, and evidence-based learning capabilities are
+ being investigated to see how Watson may contribute to clinical
+ decision support systems and the increase in artificial intelligence in
+ healthcare for use by medical professionals.^[95] To aid physicians in
+ the treatment of their patients, once a physician has posed a query to
+ the system describing symptoms and other related factors, Watson first
+ parses the input to identify the most important pieces of information;
+ then mines patient data to find facts relevant to the patient's medical
+ and hereditary history; then examines available data sources to form
+ and test hypotheses;^[95] and finally provides a list of
+ individualized, confidence-scored recommendations.^[96] The sources of
+ data that Watson uses for analysis can include treatment guidelines,
+ electronic medical record data, notes from healthcare providers,
+ research materials, clinical studies, journal articles and patient
+ information.^[95] Despite being developed and marketed as a "diagnosis
+ and treatment advisor", Watson has never been actually involved in the
+ medical diagnosis process, only in assisting with identifying treatment
+ options for patients who have already been diagnosed.^[97]
+
+ In February 2011, it was announced that IBM would be partnering with
+ Nuance Communications for a research project to develop a commercial
+ product during the next 18 to 24 months, designed to exploit Watson's
+ clinical decision support capabilities. Physicians at Columbia
+ University would help to identify critical issues in the practice of
+ medicine where the system's technology may be able to contribute, and
+ physicians at the University of Maryland would work to identify the
+ best way that a technology like Watson could interact with medical
+ practitioners to provide the maximum assistance.^[98]
+
+ In September 2011, IBM and WellPoint (now Anthem) announced a
+ partnership to utilize Watson to help suggest treatment options to
+ physicians.^[99] Then, in February 2013, IBM and WellPoint gave Watson
+ its first commercial application, for utilization management decisions
+ in lung cancer treatment at Memorial Sloan–Kettering Cancer Center.^[8]
+
+ IBM announced a partnership with Cleveland Clinic in October 2012. The
+ company has sent Watson to the Cleveland Clinic Lerner College of
+ Medicine of Case Western Reserve University, where it will increase its
+ health expertise and assist medical professionals in treating patients.
+ The medical facility will utilize Watson's ability to store and process
+ large quantities of information to help speed up and increase the
+ accuracy of the treatment process. "Cleveland Clinic's collaboration
+ with IBM is exciting because it offers us the opportunity to teach
+ Watson to 'think' in ways that have the potential to make it a powerful
+ tool in medicine", said C. Martin Harris, MD, chief information officer
+ of Cleveland Clinic.^[100]
+
+ In 2013, IBM and MD Anderson Cancer Center began a pilot program to
+ further the center's "mission to eradicate cancer".^[101]^[102]
+ However, after spending $62 million, the project did not meet its goals
+ and it has been stopped.^[103]
+
+ On February 8, 2013, IBM announced that oncologists at the Maine Center
+ for Cancer Medicine and Westmed Medical Group in New York have started
+ to test Watson in an effort to recommend treatment for lung
+ cancer.^[104]
+
+ On July 29, 2016, IBM and Manipal Hospitals^[105]^[106]^[107] (a
+ leading hospital chain in India) announced the launch of IBM Watson for
+ Oncology, for cancer patients. This product provides information and
+ insights to physicians and cancer patients to help them identify
+ personalized, evidence-based cancer care options. Manipal Hospitals is
+ the second hospital^[108] in the world to adopt this technology and
+ first in the world to offer it to patients online as an expert second
+ opinion through their website.^[105]^[109] Manipal discontinued this
+ contract in December 2018.
+
+ On January 7, 2017, IBM and Fukoku Mutual Life Insurance entered into a
+ contract for IBM to deliver analysis to compensation payouts via its
+ IBM Watson Explorer AI, which resulted in the loss of 34 jobs. The
+ company said it would speed up compensation payout analysis via
+ analyzing claims and medical records, and increase productivity by 30%.
+ The company also said it would save ¥140m in running costs.^[110]
+
+ Several startups in the healthcare space have been effectively using
+ seven business model archetypes to take solutions based on IBM Watson
+ to the marketplace. These archetypes depends on the value generate for
+ the target user (e.g. patient focus vs. healthcare provider and payer
+ focus) and value capturing mechanisms (e.g. providing information or
+ connecting stakeholders).^[111]
+
+ By 2022, IBM Watson Health was generating about a billion dollars in
+ annual gross revenue,^[112] but was facing a lack of profitability and
+ increased competition. One expert assessed to CNN that "IBM was clearly
+ not gaining much traction in the healthcare market". A 2021 post from
+ the Association for Computing Machinery (ACM) titled "What Happened To
+ Watson Health?" described the portfolio management challenges of IBM
+ Watson Health given the number of acquisitions involved in the Watson
+ Health division creation in 2015, as well as technical limitations that
+ existed at the time regarding where the Watson AI framework could be
+ deployed.^[113] In February 2021, the Wall Street Journal reported that
+ Watson Health was exploring a sale.^[114] On January 21, 2022, IBM
+ announced the sell-off of its Watson Health unit to Francisco
+ Partners.^[115]
+
+IBM Watson Group[edit]
+
+ On January 9, 2014, IBM announced it was creating a business unit
+ around Watson, led by senior vice president Michael Rhodin.^[116] IBM
+ Watson Group will have headquarters in New York's Silicon Alley and
+ will employ 2,000 people. IBM has invested $1 billion to get the
+ division going. Watson Group will develop three new cloud-delivered
+ services: Watson Discovery Advisor, Watson Engagement Advisor, and
+ Watson Explorer. Watson Discovery Advisor will focus on research and
+ development projects in pharmaceutical industry, publishing, and
+ biotechnology, Watson Engagement Advisor will focus on self-service
+ applications using insights on the basis of natural language questions
+ posed by business users, and Watson Explorer will focus on helping
+ enterprise users uncover and share data-driven insights based on
+ federated search more easily.^[116] The company is also launching a
+ $100 million venture fund to spur application development for
+ "cognitive" applications. According to IBM, the cloud-delivered
+ enterprise-ready Watson has seen its speed increase 24 times over—a
+ 2,300 percent improvement in performance and its physical size shrank
+ by 90 percent—from the size of a master bedroom to three stacked pizza
+ boxes.^[116] IBM CEO Virginia Rometty said she wants Watson to generate
+ $10 billion in annual revenue within ten years.^[117] In 2017, IBM and
+ MIT established a new joint research venture in artificial
+ intelligence. IBM invested $240 million to create the MIT–IBM Watson AI
+ Lab in partnership with MIT, which brings together researchers in
+ academia and industry to advance AI research, with projects ranging
+ from computer vision and NLP to devising new ways to ensure that AI
+ systems are fair, reliable and secure.^[118] In March 2018, IBM's CEO
+ Ginni Rometty proposed "Watson's Law," the "use of and application of
+ business, smart cities, consumer applications and life in
+ general."^[119]
+
+Chef Watson[edit]
+
+ Watson helped a team of chefs create five new poutines for the 2015 La
+ Poutine Week food festival in Toronto and Montreal. It analyzed the
+ demographics and popular cuisines of the cities and drew from a
+ database of tens of thousands of recipes to create fusion pairings for
+ each city.^[120] IBM and Bon Appétit magazine co-created an AI cooking
+ app known as Chef Watson.^[121]
+
+Chatbot[edit]
+
+ Watson is being used via IBM partner program as a chatbot to provide
+ the conversation for children's toys.^[122]
+
+Building codes[edit]
+
+ In 2015, the engineering firm ENGEO created an online service via the
+ IBM partner program named GoFetchCode. GoFetchCode applies Watson's
+ natural language processing and question-answering capabilities to the
+ International Code Council's model building codes.^[123]
+
+Teaching assistant[edit]
+
+ IBM Watson is being used for several projects relating to education,
+ and has entered partnerships with Pearson Education, Blackboard, Sesame
+ Workshop and Apple.^[124]^[125]
+
+ In its partnership with Pearson, Watson is being made available inside
+ electronic text books to provide natural language, one-on-one tutoring
+ to students on the reading material.^[126]
+
+ As an individual using the free Watson APIs available to the public,
+ Ashok Goel, a professor at Georgia Tech, used Watson to create a
+ virtual teaching assistant to assist students in his class.^[127]
+ Initially, Goel did not reveal the nature of "Jill", which was created
+ with the help of a few students and IBM. Jill answered questions where
+ it had a 97% certainty of an accurate answer, with the remainder being
+ answered by human assistants.^[128]
+
+ The research group of Sabri Pllana developed an assistant for learning
+ parallel programming using the IBM Watson.^[129] A survey with a number
+ of novice parallel programmers at the Linnaeus University indicated
+ that such assistants will be welcomed by students that learn parallel
+ programming.
+
+Weather forecasting[edit]
+
+ In August 2016, IBM announced it would be using Watson for weather
+ forecasting.^[130] Specifically, the company announced they would use
+ Watson to analyze data from over 200,000 Weather Underground personal
+ weather stations, as well as data from other sources, as a part of
+ Project Deep Thunder.^[131]
+
+Fashion[edit]
+
+ IBM Watson together with Marchesa designed a dress that changed the
+ colour of the fabric depending on the mood of the audience. The dress
+ lit up in different colours based on the sentiment of Tweets about the
+ dress. Tweets were passed through a Watson tone analyzer and then sent
+ back to a small computer inside the waist of the dress.^[132]
+
+Tax preparation[edit]
+
+ On February 5–6, 2017, tax preparation company H&R Block began
+ nationwide use of a Watson-based program.^[133]
+
+Advertising[edit]
+
+ In September 2017, IBM announced that with its acquisition of The
+ Weather Company's advertising sales division, and a partnership with
+ advertising neural network Cognitiv, Watson will provide AI-powered
+ advertising solutions.^[134]^[135]^[136]
+
+See also[edit]
+
+ * Artificial intelligence
+ * Blue Gene
+ * Commonsense knowledge (artificial intelligence)
+ * Glossary of artificial intelligence
+ * Artificial general intelligence
+ * Tech companies in the New York metropolitan area
+ * Wolfram Alpha
+
+References[edit]
+
+ 1. ^ ^a ^b ^c "DeepQA Project: FAQ". IBM. Archived from the original
+ on June 29, 2011. Retrieved February 11, 2011.
+ 2. ^ Ferrucci, David; Levas, Anthony; Bagchi, Sugato; Gondek, David;
+ Mueller, Erik T. (2013-06-01). "Watson: Beyond Jeopardy!".
+ Artificial Intelligence. 199: 93–105.
+ doi:10.1016/j.artint.2012.06.009.
+ 3. ^ ^a ^b Hale, Mike (February 8, 2011). "Actors and Their Roles for
+ $300, HAL? HAL!". The New York Times. Retrieved February 11, 2011.
+ 4. ^ "The DeepQA Project". IBM Research. Archived from the original on
+ June 29, 2011. Retrieved February 18, 2011.
+ 5. ^ "Dave Ferrucci at Computer History Museum – How It All Began and
+ What's Next". IBM Research. December 1, 2011. Archived from the
+ original on March 13, 2012. Retrieved February 11, 2012.
+ 6. ^ Loftus, Jack (April 26, 2009). "IBM Prepping 'Watson' Computer to
+ Compete on Jeopardy!". Gizmodo. Archived from the original on July
+ 31, 2017. Retrieved September 18, 2017.
+ 7. ^ "IBM's "Watson" Computing System to Challenge All Time Henry
+ Lambert Jeopardy! Champions". Sony Pictures Television. December
+ 14, 2010. Archived from the original on June 16, 2013.
+ 8. ^ ^a ^b Upbin, Bruce (February 8, 2013). "IBM's Watson Gets Its
+ First Piece Of Business In Healthcare". Forbes. Archived from the
+ original on September 18, 2017. Retrieved September 18, 2017.
+ 9. ^ ^a ^b Ferrucci, D.; et al. (2010). "Building Watson: An Overview
+ of the DeepQA Project". AI Magazine. 31 (3): 59.
+ doi:10.1609/aimag.v31i3.2303. Archived from the original on
+ December 28, 2017. Retrieved February 19, 2011.
+ 10. ^ "Watson, A System Designed for Answers: The Future of Workload
+ Optimized Systems Design". IBM Systems and Technology. February
+ 2011. p. 3. Archived from the original on March 4, 2016. Retrieved
+ September 9, 2015.
+ 11. ^ ^a ^b Jackson, Joab (February 17, 2011). "IBM Watson Vanquishes
+ Human Jeopardy Foes". PC World. IDG News. Archived from the
+ original on February 20, 2011. Retrieved February 17, 2011.
+ 12. ^ Takahashi, Dean (February 17, 2011). "IBM researcher explains
+ what Watson gets right and wrong". VentureBeat. Archived from the
+ original on February 18, 2011. Retrieved February 18, 2011.
+ 13. ^ Novell (February 2, 2011). "Watson Supercomputer to Compete on
+ 'Jeopardy!' – Powered by SUSE Linux Enterprise Server on IBM
+ POWER7". The Wall Street Journal. Archived from the original on
+ April 21, 2011. Retrieved February 21, 2011.
+ 14. ^ ^a ^b "Is Watson the smartest machine on earth?". Computer
+ Science and Electrical Engineering Department, University of
+ Maryland Baltimore County. February 10, 2011. Archived from the
+ original on September 27, 2011. Retrieved February 11, 2011.
+ 15. ^ ^a ^b Rennie, John (February 14, 2011). "How IBM's Watson
+ Computer Excels at Jeopardy!". PLoS blogs. Archived from the
+ original on February 22, 2011. Retrieved February 19, 2011.
+ 16. ^ Lucas, Mearian (February 21, 2011). "Can anyone afford an IBM
+ Watson supercomputer? (Yes)". Computerworld. Archived from the
+ original on December 12, 2013. Retrieved February 21, 2011.
+ 17. ^ "Top500 List – November 2013". Top500.org. Archived from the
+ original on 2013-12-31. Retrieved 2014-01-04.
+ 18. ^ Ferrucci, David; et al. "The AI Behind Watson – The Technical
+ Article". AI Magazine (Fall 2010). Archived from the original on
+ November 6, 2020. Retrieved November 11, 2013.
+ 19. ^ ^a ^b ^c ^d ^e ^f ^g ^h ^i ^j ^k ^l ^m ^n ^o ^p ^q Thompson,
+ Clive (June 16, 2010). "Smarter Than You Think: What Is I.B.M.'s
+ Watson?". The New York Times Magazine. Archived from the original
+ on June 5, 2011. Retrieved February 18, 2011.
+ 20. ^ "Will Watson Win On Jeopardy!?". Nova ScienceNOW. Public
+ Broadcasting Service. January 20, 2011. Archived from the original
+ on April 14, 2011. Retrieved January 27, 2011.
+ 21. ^ ^a ^b ^c ^d Needleman, Rafe (February 18, 2011). "Reporters'
+ Roundtable: Debating the robobrains". CNET. Retrieved February 18,
+ 2011.^[permanent dead link]
+ 22. ^ Russo-Spena, Tiziana; Mele, Cristina; Marzullo, Marialuisa
+ (2018). "Practising Value Innovation through Artificial
+ Intelligence: The IBM Watson Case". Journal of Creating Value. 5
+ (1): 11–24. doi:10.1177/2394964318805839. ISSN 2394-9643.
+ S2CID 56759835.
+ 23. ^ ^a ^b ^c ^d "Jeopardy! Champ Ken Jennings". The Washington Post.
+ February 15, 2011. Archived from the original on February 14, 2011.
+ Retrieved February 15, 2011.
+ 24. ^ ^a ^b Gondek, David (January 10, 2011). "How Watson "sees,"
+ "hears," and "speaks" to play Jeopardy!". IBM Research News.
+ Retrieved February 21, 2011.
+ 25. ^ Avery, Lise (February 14, 2011). "Interview with Actor Jeff
+ Woodman, Voice of IBM's Watson Computer" (MP3). Anything Goes!!.
+ Archived from the original on September 21, 2019. Retrieved
+ February 15, 2011.
+ 26. ^ Kosinski, Robert J. (2008). "A Literature Review on Reaction
+ Time". Clemson University. Archived from the original on March 17,
+ 2016. Retrieved January 10, 2016.
+ 27. ^ ^a ^b Baker (2011), p. 174.
+ 28. ^ Baker (2011), p. 178.
+ 29. ^ ^a ^b ^c Strachan, Alex (February 12, 2011). "For Jennings, it's
+ a man vs. man competition". The Vancouver Sun. Archived from the
+ original on February 21, 2011. Retrieved February 15, 2011.
+ 30. ^ Baker (2011), pp. 6–8.
+ 31. ^ Baker (2011), p. 30.
+ 32. ^ Radev, Dragomir R.; Prager, John; Samn, Valerie (2000). "Ranking
+ potential answers to natural language questions" (PDF). Proceedings
+ of the 6th Conference on Applied Natural Language Processing.
+ Archived from the original (PDF) on 2011-08-26. Retrieved
+ 2011-02-23.
+ 33. ^ Prager, John; Brown, Eric; Coden, Annie; Radev, Dragomir R. (July
+ 2000). "Question-answering by predictive annotation" (PDF).
+ Proceedings, 23rd Annual International ACM SIGIR Conference on
+ Research and Development in Information Retrieval. Archived from
+ the original (PDF) on 2011-08-23. Retrieved 2011-02-23.
+ 34. ^ Leopold, George (July 18, 2007). "IBM's Paul Horn retires, Kelly
+ named research chief". EE Times. Archived from the original on June
+ 3, 2020. Retrieved May 27, 2020.
+ 35. ^ Babcock, Charles (October 14, 2015). "IBM Cognitive Colloquium
+ Spotlights Uncovering Dark Data". InformationWeek. Archived from
+ the original on June 3, 2020. Retrieved May 27, 2020.
+ 36. ^ ^a ^b Brodkin, Jon (February 10, 2010). "IBM's Jeopardy-playing
+ machine can now beat human contestants". Network World. Archived
+ from the original on June 3, 2013. Retrieved February 19, 2011.
+ 37. ^ Zimmer, Ben (February 17, 2011). "Is It Time to Welcome Our New
+ Computer Overlords?". The Atlantic. Archived from the original on
+ August 29, 2018. Retrieved February 17, 2011.
+ 38. ^ Raz, Guy (January 28, 2011). "Can a Computer Become a Jeopardy!
+ Champ?". National Public Radio. Archived from the original on
+ February 28, 2011. Retrieved February 18, 2011.
+ 39. ^ "Medical Students Offer Expertise to IBM's Jeopardy!-Winning
+ Computer Watson as It Pursues a New Career in Medicine" (PDF).
+ InTouch. New York Medical College. 18: 4. June 2012. Archived from
+ the original (PDF) on 2012-11-23.
+ 40. ^ "'Millionaire' quiz whiz Toutant had passion for trivia, Austin's
+ arts scene". Archived from the original on 2021-09-23. Retrieved
+ 2021-09-23.
+ 41. ^ Stelter, Brian (December 14, 2010). "I.B.M. Supercomputer
+ 'Watson' to Challenge 'Jeopardy' Stars". The New York Times.
+ Retrieved December 14, 2010.
+ 42. ^ Baker (2011), p. 171.
+ 43. ^ Flatow, Ira (February 11, 2011). "IBM Computer Faces Off Against
+ 'Jeopardy' Champs". Talk of the Nation. National Public Radio.
+ Archived from the original on February 17, 2011. Retrieved February
+ 15, 2011.
+ 44. ^ Sostek, Anya (February 13, 2011). "Human champs of 'Jeopardy!'
+ vs. Watson the IBM computer: a close match". Pittsburgh Post
+ Gazette. Archived from the original on February 17, 2011. Retrieved
+ February 19, 2011.
+ 45. ^ ^a ^b ^c Jennings, Ken (February 16, 2011). "My Puny Human
+ Brain". Slate. Newsweek Interactive Co. LLC. Archived from the
+ original on February 18, 2011. Retrieved February 17, 2011.
+ 46. ^ Baker (2011), p. 117.
+ 47. ^ Baker (2011), pp. 232–258.
+ 48. ^ Dignan, Larry (January 13, 2011). "IBM's Watson wins Jeopardy
+ practice round: Can humans hang?". ZDnet. Archived from the
+ original on January 13, 2011. Retrieved January 13, 2011.
+ 49. ^ ^a ^b "The IBM Challenge Day 1". Jeopardy. Season 27. Episode 23.
+ February 14, 2011.
+ 50. ^ Lenchner, Jon (February 3, 2011). "Knowing what it knows:
+ selected nuances of Watson's strategy". IBM Research News. IBM.
+ Archived from the original on February 16, 2011. Retrieved February
+ 16, 2011.
+ 51. ^ Johnston, Casey (February 15, 2011). "Jeopardy: IBM's Watson
+ almost sneaks wrong answer by Trebek". Ars Technica. Archived from
+ the original on February 18, 2011. Retrieved February 15, 2011.
+ 52. ^ ^a ^b ^c ^d "Computer crushes the competition on 'Jeopardy!'".
+ Associated Press. February 15, 2011. Archived from the original on
+ February 19, 2011. Retrieved February 19, 2011.
+ 53. ^ Tesauro, Gerald (February 13, 2011). "Watson's wagering
+ strategies". IBM Research News. IBM. Archived from the original on
+ February 18, 2011. Retrieved February 18, 2011.
+ 54. ^ Staff (February 15, 2011). "IBM's computer wins 'Jeopardy!'
+ but... Toronto?". CTV News. Archived from the original on November
+ 27, 2012. Retrieved February 15, 2011.
+ 55. ^ ^a ^b Robertson, Jordan; Borenstein, Seth (February 16, 2011).
+ "For Watson, Jeopardy! victory was elementary". The Globe and Mail.
+ The Associated Press. Archived from the original on February 20,
+ 2011. Retrieved February 17, 2011.
+ 56. ^ Hamm, Steve (February 15, 2011). "Watson on Jeopardy! Day Two:
+ The Confusion over and Airport Clue". A Smart Planet Blog. Archived
+ from the original on October 24, 2011. Retrieved February 21, 2011.
+ 57. ^ Johnston, Casey (February 15, 2011). "Creators: Watson has no
+ speed advantage as it crushes humans in Jeopardy". Ars Technica.
+ Archived from the original on February 18, 2011. Retrieved February
+ 21, 2011.
+ 58. ^ Oberman, Mira (February 17, 2011). "Computer creams human
+ Jeopardy! champions". Vancouver Sun. Agence France-Presse. Archived
+ from the original on February 20, 2011. Retrieved February 17,
+ 2011.
+ 59. ^ Johnston, Casey (February 17, 2011). "Bug lets humans grab Daily
+ Double as Watson triumphs on Jeopardy". Ars Technica. Archived from
+ the original on February 21, 2011. Retrieved February 21, 2011.
+ 60. ^ ^a ^b Upbin, Bruce (February 17, 2011). "IBM's Supercomputer
+ Watson Wins It All With $367 Bet". Forbes. Archived from the
+ original on February 21, 2011. Retrieved February 21, 2011.
+ 61. ^ Oldenburg, Ann (February 17, 2011). "Ken Jennings: 'My puny
+ brain' did just fine on 'Jeopardy!'". USA Today. Archived from the
+ original on February 20, 2011. Retrieved February 21, 2011.
+ 62. ^ "Show 6088 – The IBM Challenge, Day 2". Jeopardy!. February 16,
+ 2011. Syndicated.
+ 63. ^ "World Community Grid to benefit from Jeopardy! competition".
+ World Community Grid. February 4, 2011. Archived from the original
+ on January 14, 2012. Retrieved February 19, 2011.
+ 64. ^ "Jeopardy! And IBM Announce Charities To Benefit From Watson
+ Competition". IBM Corporation. January 13, 2011. Archived from the
+ original on November 10, 2021. Retrieved February 19, 2011.
+ 65. ^ "IBM's Watson supercomputer crowned Jeopardy king". BBC News.
+ February 17, 2011. Archived from the original on February 18, 2011.
+ Retrieved February 17, 2011.
+ 66. ^ Markoff, John (February 16, 2011). "Computer Wins on 'Jeopardy!':
+ Trivial, It's Not". The New York Times. Yorktown Heights, New York.
+ Archived from the original on October 22, 2014. Retrieved February
+ 17, 2011.
+ 67. ^ Searle, John (February 23, 2011). "Watson Doesn't Know It Won on
+ 'Jeopardy!'". The Wall Street Journal. Archived from the original
+ on November 10, 2021. Retrieved July 26, 2011.
+ 68. ^ Lohr, Steve (December 5, 2011). "Creating AI based on the real
+ thing". The New York Times. Archived from the original on November
+ 10, 2021. Retrieved February 26, 2017..
+ 69. ^ ^a ^b "NJ congressman tops 'Jeopardy' computer Watson".
+ Associated Press. March 2, 2011. Archived from the original on
+ March 7, 2011. Retrieved March 2, 2011.
+ 70. ^ Weber, Robert C. (February 14, 2011). "Why 'Watson' matters to
+ lawyers". The National Law Journal. Archived from the original on
+ November 10, 2021. Retrieved February 18, 2011.
+ 71. ^ Nay, Chris (September 6, 2011). "Putting Watson to work:
+ Interview with GM of Watson Solutions Manoj Saxena". Smarter Planet
+ Blog. IBM. Archived from the original on November 12, 2013.
+ Retrieved November 12, 2013.
+ 72. ^ Merritt, Rick (February 14, 2011). "IBM playing Jeopardy with tax
+ dollars". EE Times. Archived from the original on February 18,
+ 2011. Retrieved February 19, 2011.
+ 73. ^ Dusto, Amy (December 3, 2013). "IBM's Watson computer helps
+ shoppers via a new app". Internet Retailer. Archived from the
+ original on June 12, 2016. Retrieved January 10, 2016.
+ 74. ^ Comstock, Jonah (November 15, 2013). "With Watson API launch, IBM
+ turns to WellTok for patients, MD Buyline for docs".
+ MobiHealthNews. Archived from the original on March 4, 2016.
+ Retrieved January 10, 2016.
+ 75. ^ Upbin, Bruce (November 14, 2013). "IBM Opens Up Its Watson
+ Cognitive Computer For Developers Everywhere". Forbes. Archived
+ from the original on January 26, 2016. Retrieved January 10, 2016.
+ 76. ^ "IBM's Watson to Join Research Team at Rensselaer". Rensselaer
+ Polytechnic Institute. January 30, 2013. Archived from the original
+ on October 4, 2013. Retrieved October 1, 2013.
+ 77. ^ "The Independent Sector: Cultural, Economic and Social
+ Contributions of New York's 100+, Not-for-Profit Colleges and
+ Universities" (PDF). Commission on Independent Colleges and
+ Universities. Summer 2013. p. 12. Archived (PDF) from the original
+ on October 4, 2013. Retrieved October 1, 2013.
+ 78. ^ Cocks, Tim (February 6, 2014). "IBM starts rolling out Watson
+ supercomputer in Africa". Reuters. Archived from the original on
+ March 8, 2016. Retrieved January 10, 2016.
+ 79. ^ Coolidge, Donald (May 29, 2014). "IBM Watson Mobile Developers
+ Challenge Finalists: Majestyk". Majestyk Apps. Archived from the
+ original on September 19, 2015. Retrieved January 10, 2016.
+ 80. ^ "Majestyk Apps – An IBM Watson Mobile Developer Challenge
+ Winner". Flickr. June 3, 2014. Archived from the original on
+ September 27, 2015. Retrieved January 10, 2016.
+ 81. ^ "Red Ant – An IBM Watson Mobile Developer Challenge Winner".
+ Flickr. June 3, 2014. Archived from the original on December 28,
+ 2015. Retrieved January 10, 2016.
+ 82. ^ "GenieMD – An IBM Watson Mobile Developer Challenge Winner".
+ Flickr. June 3, 2014. Archived from the original on September 22,
+ 2015. Retrieved January 10, 2016.
+ 83. ^ "IBM Announces Watson Mobile Developer Challenge Winners". IBM
+ News. June 3, 2014. Archived from the original on January 19, 2016.
+ Retrieved January 10, 2016.
+ 84. ^ All, Ann (July 9, 2014). "Genesys to Put IBM's Watson to Work".
+ Enterprise Apps Today. Archived from the original on March 7, 2016.
+ Retrieved January 10, 2016.
+ 85. ^ Wilson, Mark (June 30, 2014). "IBM's Watson Is Now A Cooking App
+ With Infinite Recipes". fastcodesign.com. Archived from the
+ original on February 28, 2016. Retrieved January 10, 2016.
+ 86. ^ Hardawar, Devindra. "IBM's big bet on Watson is paying off with
+ more apps and DNA analysis". Engadget. Archived from the original
+ on July 9, 2015. Retrieved July 2, 2015.
+ 87. ^ Griggs, Mary Beth (May 20, 2016). "IBM Watson can help find water
+ wasters in drought-stricken California". Popular Science. Archived
+ from the original on June 23, 2016. Retrieved July 4, 2016.
+ 88. ^ Marty Swant (6 September 2016). "Condé Nast Has Started Using
+ IBM's Watson to Find Influencers for Brands". Adweek. Archived from
+ the original on 8 September 2016. Retrieved 8 September 2016.
+ 89. ^ "Rare Carat Releases World's First Artificial Intelligence
+ Jeweler Using IBM Watson Technology". PRNewswire. February 28,
+ 2017. Archived from the original on August 22, 2017. Retrieved
+ August 22, 2017.
+ 90. ^ "Rare Carat's Watson-powered chatbot will help you put a diamond
+ ring on it". TechCrunch. February 15, 2017. Archived from the
+ original on August 22, 2017. Retrieved August 22, 2017.
+ 91. ^ "10 ways you may have already used IBM Watson". VentureBeat.
+ March 10, 2017. Archived from the original on August 22, 2017.
+ Retrieved August 22, 2017.
+ 92. ^ "IBM Watson to help Pebble Beach create a virtual concierge for
+ guests". VentureBeat. 2017-05-09. Archived from the original on
+ 2017-05-09. Retrieved 2017-05-10.
+ 93. ^ IBM Watson: artificial intelligence arrives at the Museum of
+ Anthropology. Aban Tech. October 31, 2017. Archived from the
+ original on 2019-09-21. Retrieved May 11, 2018 – via Youtube.
+ 94. ^ "IBM Watson is AI for Business".
+ 95. ^ ^a ^b ^c "Putting Watson to Work: Watson in Healthcare". IBM.
+ Archived from the original on November 11, 2013. Retrieved November
+ 11, 2013.
+ 96. ^ "IBM Watson Helps Fight Cancer with Evidence-Based Diagnosis and
+ Treatment Suggestions" (PDF). IBM. Archived (PDF) from the original
+ on April 26, 2013. Retrieved November 12, 2013.
+ 97. ^ Saxena, Manoj (February 13, 2013). "IBM Watson Progress and 2013
+ Roadmap (Slide 7)". IBM. Archived from the original on November 13,
+ 2013. Retrieved November 12, 2013.
+ 98. ^ Wakeman, Nick (February 17, 2011). "IBM's Watson heads to medical
+ school". Washington Technology. Archived from the original on
+ February 21, 2011. Retrieved February 19, 2011.
+ 99. ^ Mathews, Anna Wilde (September 12, 2011). "Wellpoint's New Hire:
+ What is Watson?". The Wall Street Journal. Archived from the
+ original on February 22, 2017. Retrieved March 12, 2017.
+ 100. ^ Miliard, Mike (October 30, 2012). "Watson Heads to Medical
+ School: Cleveland Clinic, IBM Send Supercomputer to College".
+ Healthcare IT News. Archived from the original on November 11,
+ 2013. Retrieved November 11, 2013.
+ 101. ^ "MD Anderson Taps IBM Watson to Power "Moon Shots" Mission Aimed
+ at Ending Cancer, Starting with Leukemia" (Press release). IBM.
+ Archived from the original on 2017-02-21. Retrieved 2017-02-20.
+ 102. ^ "IBM's Watson Now Tackles Clinical Trials At MD Anderson Cancer
+ Center". Forbes. Archived from the original on 2017-09-20.
+ Retrieved 2017-09-18.
+ 103. ^ "MD Anderson Benches IBM Watson In Setback For Artificial
+ Intelligence In Medicine". Forbes. Archived from the original on
+ 2017-10-02. Retrieved 2017-09-18.
+ 104. ^ Leske, Nikola (February 9, 2013). "Doctors Seek Help on Cancer
+ Treatment from IBM Supercomputer". Reuters. Archived from the
+ original on February 16, 2016. Retrieved November 11, 2013.
+ 105. ^ ^a ^b "Manipal Hospitals | Watson for Oncology | Cancer
+ Treatment". watsononcology.manipalhospitals.com. Archived from the
+ original on 2017-01-18. Retrieved 2017-01-17.
+ 106. ^ "MANIPAL HOSPITALS ANNOUNCES NATIONAL LAUNCH OF IBM WATSON FOR
+ ONCOLOGY". www-03.ibm.com. 2016-07-29. Archived from the original
+ on 2017-01-18. Retrieved 2017-01-17.
+ 107. ^ "Manipal Hospitals is first adopter of IBM Watson in India".
+ www-03.ibm.com. 2015-12-02. Archived from the original on
+ 2017-01-18. Retrieved 2017-01-17.
+ 108. ^ ANI (2016-10-28). "Manipal Hospitals to adopt IBM's 'Watson for
+ Oncology' supercomputer for cancer treatment". Business Standard
+ India. Archived from the original on 2017-01-18. Retrieved
+ 2017-01-17.
+ 109. ^ "Hospitals in Asia use Watson supercomputer for cancer
+ treatment". STAT. 2016-08-19. Archived from the original on
+ 2017-01-18. Retrieved 2017-01-17.
+ 110. ^ McCurry, Justin (2017-01-05). "Japanese company replaces office
+ workers with artificial intelligence". The Guardian.
+ ISSN 0261-3077. Archived from the original on 2019-05-03. Retrieved
+ 2017-01-29.
+ 111. ^ Garbuio, Massimo; Lin, Nidthida (2019). "Artificial Intelligence
+ as a Growth Engine for Health Care Startups: Emerging Business
+ Models". California Management Review. 61 (2): 59–83.
+ doi:10.1177/0008125618811931. S2CID 158219917.
+ 112. ^ Cooper, Laura (2021-02-18). "IBM Explores Sales of IBM Watson
+ Health".
+ 113. ^ Meil, Doug (2021-04-21). "What Happened To Watson Health?".
+ Association for Computing Machinery:BLOG@CACM. Retrieved
+ 2021-04-21.
+ 114. ^ Cooper, Laura (2021-02-18). "IBM explores sale of IBM Watson
+ Health".
+ 115. ^ "IBM is selling off its Watson Health assets". CNN. 2022.
+ Retrieved 6 February 2022.
+ 116. ^ ^a ^b ^c "IBM Watson Group Unveils Cloud-Delivered Watson
+ Services to Transform Industrial R&D, Visualize Big Data Insights
+ and Fuel Analytics Exploration" (Press release). IBM. January 9,
+ 2014. Archived from the original on October 12, 2020. Retrieved
+ February 14, 2020.
+ 117. ^ Ante, Spencer E. (January 9, 2014). "IBM Set to Expand Watson's
+ Reach". The Wall Street Journal. Archived from the original on May
+ 9, 2015. Retrieved January 9, 2014.
+ 118. ^ "Inside the Lab". September 2017. Archived from the original on
+ October 23, 2020. Retrieved October 6, 2020.
+ 119. ^ "IBM CEO Rometty Proposes 'Watson's Law': AI In Everything"
+ Archived 2021-04-16 at the Wayback Machine, Adrian Bridgewater,
+ Forbes, March 20, 2018
+ 120. ^ Pelley, Lauren (2 February 2015). "Poutine inventions offer new
+ wheys to eat your curds". Toronto Star. Toronto, Ontario. pp. E1,
+ E8. ProQuest 1650084755.
+ 121. ^ Kleeman, Alexandra. "Cooking with Chef Watson, I.B.M.'s
+ Artificial-Intelligence App". The New Yorker. Archived from the
+ original on 25 January 2021. Retrieved 18 April 2021.
+ 122. ^ Takahashi, Dean (23 February 2015). "Elemental's smart connected
+ toy CogniToys taps IBM's Watson supercomputer for its brains".
+ Venture Beat. Archived from the original on 20 May 2015. Retrieved
+ 15 May 2015.
+ 123. ^ "About Us GoFetchCode". GoFetchCode. 2015-10-21. Archived from
+ the original on 2017-07-10. Retrieved 2017-07-04.
+ 124. ^ Leopold, Todd. "A professor built an AI teaching assistant for
+ his courses — and it could shape the future of education". Business
+ Insider. Business Insider. Archived from the original on 26
+ September 2017. Retrieved 26 September 2017.
+ 125. ^ Straumsheim, Carl. "'Augmented Intelligence' for Higher Ed".
+ Inside Higher Ed. Inside Higher Ed. Archived from the original on
+ 27 September 2017. Retrieved 26 September 2017.
+ 126. ^ Plenty, Rebecca (October 25, 2016). "Pearson Taps IBM's Watson
+ as a Virtual Tutor for College Students". No. October 25, 2016.
+ Bloomberg. Bloomberg. Archived from the original on September 27,
+ 2017. Retrieved 26 September 2017.
+ 127. ^ Maderer, Jason. "Artificial Intelligence Course Creates AI
+ Teaching Assistant". Georgia Tech News. Georgia Tech News. Archived
+ from the original on 29 July 2018. Retrieved 26 September 2017.
+ 128. ^ McFarlane, Matt (13 May 2016). "Professor reveals to students
+ that his assistant was an AI all along". Sydney Morning Herald.
+ Archived from the original on May 16, 2016. Retrieved May 14, 2016.
+ 129. ^ Memeti, Suejb; Pllana, Sabri (January 2018). "PAPA: A parallel
+ programming assistant powered by IBM Watson cognitive computing
+ technology". Journal of Computational Science. 26: 275–284.
+ doi:10.1016/j.jocs.2018.01.001.
+ 130. ^ Jancer, Matt (26 August 2016). "IBM's Watson Takes On Yet
+ Another Job, as a Weather Forecaster". Smithsonian. Archived from
+ the original on 1 September 2016. Retrieved 29 August 2016.
+ 131. ^ Booton, Jennifer (15 June 2016). "IBM finally reveals why it
+ bought The Weather Company". Market Watch. Archived from the
+ original on 22 August 2016. Retrieved 29 August 2016.
+ 132. ^ "Cognitive dress by Marchesa and IBM lights up the night at the
+ Met Gala". Business Operations. 2016-10-27. Archived from the
+ original on 2020-06-13. Retrieved 2020-07-06.
+ 133. ^ Moscaritolo, Angela (2 February 2017). "H&R Block Enlists IBM
+ Watson to Find Tax Deductions". PC Magazine. Archived from the
+ original on 15 February 2017. Retrieved 14 February 2017.
+ 134. ^ "Artificial Intelligence Marketing Company". Cognitiv. Retrieved
+ 2022-08-23.
+ 135. ^ Swant, Marty (September 24, 2017). "As IBM Ramps Up Its
+ AI-Powered Advertising, Can Watson Crack the Code of Digital
+ Marketing?". www.adweek.com. Archived from the original on
+ 2019-04-29. Retrieved 2019-03-18.
+ 136. ^ "AI is A Rocket About to Launch - Here's How to Get On Board".
+ www.ibm.com. Archived from the original on 2019-10-29. Retrieved
+ 2019-03-18.
+
+Bibliography[edit]
+
+ *
+
+ Baker, Stephen (2011). Final Jeopardy: Man vs. Machine and the Quest to
+ Know Everything. Boston, New York: Houghton Mifflin Harcourt.
+ ISBN 978-0-547-48316-0.
+
+Further reading[edit]
+
+ * Baker, Stephen (2012) Final Jeopardy: The Story of Watson, the
+ Computer That Will Transform Our World, Mariner Books.
+ * Jackson, Joab (2014). IBM bets big on Watson-branded cognitive
+ computing PCWorld: Jan 9, 2014 2:30 PM
+ * Greenemeier, Larry. (2013). Will IBM's Watson Usher in a New Era of
+ Cognitive Computing? Scientific American. Nov 13, 2013 |* Lazarus,
+ R. S. (1982).
+ * Kelly, J.E. and Hamm, S. ( 2013). Smart Machines: IBM's Watson and
+ the Era of Cognitive Computing. Columbia Business School Publishing
+
+External links[edit]
+
+ This article's use of external links may not follow Wikipedia's
+ policies or guidelines. Please improve this article by removing
+ excessive or inappropriate external links, and converting useful links
+ where appropriate into footnote references. (May 2022) (Learn how and
+ when to remove this template message)
+ Wikimedia Commons has media related to IBM Watson.
+ * Watson homepage
+ * DeepQA homepage
+ * About Watson on Jeopardy.com
+ * Smartest Machine on Earth (PBS NOVA documentary about the making of
+ Watson)
+ * Power Systems
+ * The Watson Trivia Challenge. The New York Times. June 16, 2010.
+ * This is Watson – IBM Journal of Research and Development (published
+ by the IEEE)
+
+J! Archive[edit]
+
+ * Jeopardy! Show #6086 – Game 1, Part 1
+ * Jeopardy! Show #6087 – Game 1, Part 2
+ * Jeopardy! Show #6088 – Game 2
+
+Videos[edit]
+
+ * PBS NOVA documentary on the making of Watson
+ * Building Watson – A Brief Overview of the DeepQA Project on YouTube
+ (21:42), IBMLabs
+ * How Watson Answers a Question on YouTube
+ * David Ferrucci, Dan Cerutti and Ken Jennings on IBM's Watson at
+ Singularity Summit 2011 on YouTube
+ * A Computer Called Watson on YouTube – November 15, 2011, David
+ Ferrucci at Computer History Museum, alternate
+ * IBM Watson and the Future of Healthcare on YouTube – 2012
+ * IBM Watson-Introduction and Future Applications on YouTube – IBM at
+ EDGE 2012
+ * IBM Watson for Healthcare on YouTube – Martin Kohn, 2013
+ * IBM Watson playlist, IBMLabs Watson playlist
+
+ * v
+ * t
+ * e
+
+ IBM
+
+ History
+
+ * History
+ * Mergers and acquisitions
+ + PC business acquisition by Lenovo
+
+ Products
+
+ Hardware
+ Current
+ * Mainframe
+ + IBM Z
+ * Power microprocessors
+ * Power Systems
+ * Storage
+ + FlashSystem
+ + DS8000
+ + Q System One
+
+ Former
+ * Blue Gene
+ * Cell microprocessors
+ * PowerPC
+ * Midrange computer
+ * Personal Computer
+ * Selectric
+ * ThinkPad
+
+ Other
+ * alphaWorks
+ * Carbon Design System
+ * Cloud
+ + Cloudant
+ * Cognos Analytics
+ * Connections
+ * Criminal Reduction Utilising Statistical History
+ * Fortran
+ * ILOG
+ * Information Management Software
+ * Lotus Software
+ * Mainframe operating systems
+ * Mashup Center
+ * Planning Analytics
+ * PureQuery
+ * Quantum Experience
+ * Rational Software
+ * SPSS
+ * Tivoli Software
+ + Service Automation Manager
+ * Watson
+ * WebSphere
+
+ Business
+ entities
+
+ Current
+ * Center for The Business of Government
+ * Consulting
+ * Red Hat
+ * Kenexa
+ * International subsidiaries
+ * Research
+ * The Weather Company (Weather Underground)
+
+ Former
+ * AdStar
+ * AIM alliance
+ * Ambra Computer
+ * EduQuest
+ * Kaleida Labs
+ * Kyndryl
+ * Merative
+ * Microelectronics
+ * Product Center
+ * Service Bureau
+ * Taligent
+
+ Facilities
+
+ * Towers
+ + 1250 René-Lévesque, Montreal, QC
+ + One Atlantic Center, Atlanta, GA
+ * Software Labs
+ + Rome Software Lab
+ + Toronto Software Lab
+ * IBM Buildings
+ + 330 North Wabash, Chicago, IL
+ + Honolulu
+ + Seattle
+ * Facilities
+ + Thomas J. Watson Research Center
+ + Hakozaki Facility
+ + Yamato Facility
+ * Cambridge Scientific Center
+ * IBM Hursley
+ * Canada Head Office Building
+ * IBM Rochester
+
+ Initiatives
+
+ * Academy of Technology
+ * Deep Thunder
+ * Developer
+ + Develothon
+ * Fellow
+ * The Great Mind Challenge
+ * Linux Technology Center
+ * Smarter Planet
+ * Virtual Universe Community
+ * World Community Grid
+
+ Inventions
+
+ * Automated teller machine
+ * Cynefin framework
+ * DRAM
+ * Electronic keypunch
+ * Floppy disk
+ * Hard disk drive
+ * Magnetic stripe card
+ * Relational model
+ * Sabre airline reservation system
+ * Scanning tunneling microscope
+ * Financial swaps
+ * Universal Product Code
+
+ Terminology
+
+ * Big Blue
+ * Commercial Processing Workload
+ * Customer engineer
+ * Globally integrated enterprise
+ * e-business
+ * Think slogan
+
+ CEOs
+
+ * Thomas J. Watson (1914–1956)
+ * Thomas Watson Jr. (1956–1971)
+ * T. Vincent Learson (1971–1973)
+ * Frank T. Cary (1973–1981)
+ * John R. Opel (1981–1985)
+ * John Fellows Akers (1985–1993)
+ * Louis V. Gerstner Jr. (1993–2002)
+ * Samuel J. Palmisano (2002–2011)
+ * Ginni Rometty (2012–2020)
+ * Arvind Krishna (since 2020)
+
+ Board of
+ directors
+
+ * Thomas Buberl
+ * Michael L. Eskew
+ * David Farr
+ * Alex Gorsky
+ * Michelle J. Howard
+ * Arvind Krishna
+ * Andrew N. Liveris
+ * Martha E. Pollack
+ * Virginia M. Rometty
+ * Joseph R. Swedish
+ * Sidney Taurel
+ * Peter R. Voser
+
+ Other
+
+ * A Boy and His Atom
+ * Big Blue sports teams
+ + American football
+ + Rugby union
+ * Common Public License/IBM Public License
+ * Deep Blue
+ * Deep Thought
+ * Dynamic infrastructure
+ * GlobalFoundries
+ * GUIDE International
+ * IBM and the Holocaust
+ * International chess tournament
+ * Lucifer cipher
+ * Mathematica
+ * IBM Plex
+ * SHARE computing
+ * ScicomP
+ * Worker organization
+
+ * Category
+ * Commons
+ * Navigational boxes
+ + FOSS
+ + Midrange computers
+ + Operating systems
+ + Personal computers
+ + System/360
+ + System/370
+ + Typewriters
+ + Vacuum tube computers
+
+ * v
+ * t
+ * e
+
+ Jeopardy!
+
+ Hosts
+
+ * Art Fleming
+ * Alex Trebek
+ * Mike Richards
+ * Mayim Bialik
+ * Ken Jennings
+
+ Concepts
+
+ * Audition process
+ * Broadcast information
+ * Strategies and skills of Jeopardy! champions
+
+ Tournaments
+
+ Current
+ Syndicated
+ * Tournament of Champions
+ * Second Chance Tournament
+ * Professors Tournament
+
+ Primetime
+ * National College Championship
+ * Celebrity Jeopardy!
+ * Jeopardy! Masters
+
+ Past
+ Syndicated
+ * International Tournament of Champions (1996–2001)
+ * College Championship (1989–2020)
+ * Teen Tournament (1987–2019)
+ * Kids Week (1999–2014)
+ * Reunion tournaments
+ * Celebrity (1992–2015)
+ * Power Players Week (1997–2016)
+
+ Special
+ * Tenth Anniversary Tournament (1993)
+ * Million Dollar Masters (2002)
+ * Ultimate Tournament of Champions (2005)
+ * Million Dollar Celebrity Invitational (2009–10)
+ * IBM Challenge (2011)
+ * Battle of the Decades (2014)
+ * All-Star Games (2019)
+
+ Primetime
+ * Super Jeopardy! (1990)
+ * The Greatest of All Time (2020)
+
+ Notable
+ contestants
+
+ * Matt Amodio
+ * Colby Burnett
+ * Arthur Chu
+ * Buzzy Cohen
+ * Julia Collins
+ * Richard Cordray
+ * Karl Coryat
+ * Roger Craig
+ * Chuck Forrest
+ * Jackie Fuchs
+ * Red Gibson
+ * Bob Harris
+ * James Holzhauer
+ * Matt Jackson
+ * Alex Jacob
+ * Ken Jennings
+ * Larissa Kelly
+ * Mark Lowenthal
+ * David Madden
+ * Arthur Phillips
+ * Mattea Roach
+ * Brad Rutter
+ * Amy Schneider
+ * Frank Spangenberg
+ * Eddie Timanus
+ * Jerome Vered
+ * Watson
+ * Jay Wolpert
+
+ Franchise
+
+ * Jep!
+ * Rock & Roll Jeopardy!
+ * Sports Jeopardy!
+ * British version
+
+ In popular
+ culture
+
+ * Saturday Night Live sketch series
+ * "I Lost on Jeopardy"
+ * Ellen's Energy Adventure
+ * Cliff Clavin
+ * Velma Dinkley
+ * Fran Fine
+ * Thelma Harper
+ * Shaggy Rogers
+ * Marge Simpson
+ * Adam West
+ * Dorothy Zbornak
+
+ * Category
+
+ * v
+ * t
+ * e
+
+ Virtual assistants
+
+ Active
+
+ * AliGenie
+ * Alexa
+ * Alice
+ * Bixby
+ + Viv
+ * Braina
+ * Celia
+ * Clova
+ * Cortana
+ * Google Assistant
+ * Maluuba
+ * Mycroft
+ * Siri
+ * Voice Mate
+ * Watson
+ * WolframAlpha
+ * Xiaoice
+
+ Discontinued
+
+ * BlackBerry Assistant
+ * Google Now
+ * M
+ * Microsoft Agent
+ * Microsoft Bob
+ * Microsoft Voice Command
+ * Ms. Dewey
+ * Mya
+ * Office Assistant (Clippy)
+ * S Voice
+ * Speaktoit Assistant
+ * Tafiti
+ * Vlingo
+
+ * v
+ * t
+ * e
+
+ Differentiable computing
+
+ General
+
+ * Differentiable programming
+ * Information geometry
+ * Statistical manifold
+
+ * Automatic differentiation
+ * Neuromorphic engineering
+ * Pattern recognition
+ * Tensor calculus
+ * Computational learning theory
+ * Inductive bias
+
+ Concepts
+
+ * Gradient descent
+ + SGD
+ * Clustering
+ * Regression
+ + Overfitting
+ * Hallucination
+ * Adversary
+ * Attention
+ * Convolution
+ * Loss functions
+ * Backpropagation
+ * Normalization (Batchnorm)
+ * Activation
+ + Softmax
+ + Sigmoid
+ + Rectifier
+ * Regularization
+ * Datasets
+ + Augmentation
+ * Diffusion
+ * Autoregression
+
+ Applications
+
+ * Machine learning
+ + In-context learning
+ * Artificial neural network
+ + Deep learning
+ * Scientific computing
+ * Artificial Intelligence
+ * Language model
+ + Large language model
+
+ Hardware
+
+ * IPU
+ * TPU
+ * VPU
+ * Memristor
+ * SpiNNaker
+
+ Software libraries
+
+ * TensorFlow
+ * PyTorch
+ * Keras
+ * Theano
+ * JAX
+ * LangChain
+
+ Implementations
+
+ Audio–visual
+ * AlexNet
+ * WaveNet
+ * Human image synthesis
+ * HWR
+ * OCR
+ * Speech synthesis
+ * Speech recognition
+ * Facial recognition
+ * AlphaFold
+ * DALL-E
+ * Midjourney
+ * Stable Diffusion
+
+ Verbal
+ * Word2vec
+ * Seq2seq
+ * BERT
+ * LaMDA
+ + Bard
+ * NMT
+ * Project Debater
+ * IBM Watson
+ * GPT-2
+ * GPT-3
+ * ChatGPT
+ * GPT-4
+ * GPT-J
+ * Chinchilla AI
+ * PaLM
+ * BLOOM
+ * LLaMA
+
+ Decisional
+ * AlphaGo
+ * AlphaZero
+ * Q-learning
+ * SARSA
+ * OpenAI Five
+ * Self-driving car
+ * MuZero
+ * Action selection
+ + Auto-GPT
+ * Robot control
+
+ People
+
+ * Yoshua Bengio
+ * Alex Graves
+ * Ian Goodfellow
+ * Stephen Grossberg
+ * Demis Hassabis
+ * Geoffrey Hinton
+ * Yann LeCun
+ * Fei-Fei Li
+ * Andrew Ng
+ * Jürgen Schmidhuber
+ * David Silver
+
+ Organizations
+
+ * Anthropic
+ * EleutherAI
+ * Google DeepMind
+ * Hugging Face
+ * OpenAI
+ * Meta AI
+ * Mila
+ * MIT CSAIL
+
+ Architectures
+
+ * Neural Turing machine
+ * Differentiable neural computer
+ * Transformer
+ * Recurrent neural network (RNN)
+ * Long short-term memory (LSTM)
+ * Gated recurrent unit (GRU)
+ * Echo state network
+ * Multilayer perceptron (MLP)
+ * Convolutional neural network
+ * Residual network
+ * Autoencoder
+ * Variational autoencoder (VAE)
+ * Generative adversarial network (GAN)
+ * Graph neural network
+
+ * Portals
+ + Computer programming
+ + Technology
+ * Categories
+ + Artificial neural networks
+ + Machine learning
+
+ Retrieved from
+ "https://en.wikipedia.org/w/index.php?title=IBM_Watson&oldid=1162661401
+ "
+
+ Categories:
+ * Computer-related introductions in 2006
+ * IBM cloud services
+ * IBM computers
+ * Jeopardy! contestants
+ * Natural language processing software
+ * One-of-a-kind computers
+ * Virtual assistants
+
+ Hidden categories:
+ * All articles with dead external links
+ * Articles with dead external links from January 2023
+ * Articles with permanently dead external links
+ * Webarchive template wayback links
+ * Articles with short description
+ * Short description matches Wikidata
+ * All articles with unsourced statements
+ * Articles with unsourced statements from April 2022
+ * Articles with a promotional tone from April 2019
+ * All articles with a promotional tone
+ * Wikipedia external links cleanup from May 2022
+ * Wikipedia spam cleanup from May 2022
+ * Commons category link is on Wikidata
+
+ * This page was last edited on 30 June 2023, at 12:47 (UTC).
+ * Text is available under the Creative Commons Attribution-ShareAlike
+ License 4.0; additional terms may apply. By using this site, you
+ agree to the Terms of Use and Privacy Policy. Wikipedia® is a
+ registered trademark of the Wikimedia Foundation, Inc., a
+ non-profit organization.
+
+ * Privacy policy
+ * About Wikipedia
+ * Disclaimers
+ * Contact Wikipedia
+ * Code of Conduct
+ * Mobile view
+ * Developers
+ * Statistics
+ * Cookie statement
+
+ * Wikimedia Foundation
+ * Powered by MediaWiki
+
+ (BUTTON) Toggle limited content width
diff --git a/examples/scriptators/data/Natural_language_processing.txt b/examples/scriptators/data/Natural_language_processing.txt
new file mode 100644
index 0000000..cd41154
--- /dev/null
+++ b/examples/scriptators/data/Natural_language_processing.txt
@@ -0,0 +1,1345 @@
+ #alternate Edit this page Wikipedia (en) Wikipedia Atom feed
+
+ Jump to content
+
+ [ ] Main menu
+ Main menu
+ (BUTTON) move to sidebar (BUTTON) hide
+ Navigation
+ * Main page
+ * Contents
+ * Current events
+ * Random article
+ * About Wikipedia
+ * Contact us
+ * Donate
+
+ Contribute
+ * Help
+ * Learn to edit
+ * Community portal
+ * Recent changes
+ * Upload file
+
+ Languages
+ Language links are at the top of the page across from the title.
+ Wikipedia The Free Encyclopedia
+ Search
+ ____________________
+ (BUTTON) Search
+
+ * Create account
+ * Log in
+
+ [ ] Personal tools
+ * Create account
+ * Log in
+
+ Pages for logged out editors learn more
+ * Contributions
+ * Talk
+
+ [ ]
+
+Contents
+
+ (BUTTON) move to sidebar (BUTTON) hide
+ * (Top)
+ * 1History
+ (BUTTON) Toggle History subsection
+ + 1.1Symbolic NLP (1950s – early 1990s)
+ + 1.2Statistical NLP (1990s–2010s)
+ + 1.3Neural NLP (present)
+ * 2Methods: Rules, statistics, neural networks
+ (BUTTON) Toggle Methods: Rules, statistics, neural networks
+ subsection
+ + 2.1Statistical methods
+ + 2.2Neural networks
+ * 3Common NLP tasks
+ (BUTTON) Toggle Common NLP tasks subsection
+ + 3.1Text and speech processing
+ + 3.2Morphological analysis
+ + 3.3Syntactic analysis
+ + 3.4Lexical semantics (of individual words in context)
+ + 3.5Relational semantics (semantics of individual sentences)
+ + 3.6Discourse (semantics beyond individual sentences)
+ + 3.7Higher-level NLP applications
+ * 4General tendencies and (possible) future directions
+ (BUTTON) Toggle General tendencies and (possible) future directions
+ subsection
+ + 4.1Cognition and NLP
+ * 5See also
+ * 6References
+ * 7Further reading
+ * 8External links
+
+ Toggle the table of contents
+
+ [ ] Toggle the table of contents
+
+Natural language processing
+
+ [ ] 63 languages
+ * Afrikaans
+ * العربية
+ * Արեւմտահայերէն
+ * Azərbaycanca
+ * বাংলা
+ * Bân-lâm-gú
+ * Беларуская
+ * Беларуская (тарашкевіца)
+ * Български
+ * Bosanski
+ * Català
+ * Čeština
+ * Dansk
+ * Deutsch
+ * Eesti
+ * Ελληνικά
+ * Español
+ * Esperanto
+ * Euskara
+ * فارسی
+ * Français
+ * Galego
+ * 한국어
+ * Հայերեն
+ * हिन्दी
+ * Hrvatski
+ * Bahasa Indonesia
+ * IsiZulu
+ * Íslenska
+ * Italiano
+ * עברית
+ * ಕನ್ನಡ
+ * ქართული
+ * Latviešu
+ * Lietuvių
+ * Македонски
+ * मराठी
+ * مصرى
+ * Монгол
+ * မြန်မာဘာသာ
+ * 日本語
+ * ଓଡ଼ିଆ
+ * Picard
+ * Piemontèis
+ * Polski
+ * Português
+ * Română
+ * Runa Simi
+ * Русский
+ * Shqip
+ * Simple English
+ * کوردی
+ * Српски / srpski
+ * Srpskohrvatski / српскохрватски
+ * Suomi
+ * தமிழ்
+ * తెలుగు
+ * ไทย
+ * Türkçe
+ * Українська
+ * Tiếng Việt
+ * 粵語
+ * 中文
+
+ Edit links
+
+ * Article
+ * Talk
+
+ [ ] English
+
+ * Read
+ * Edit
+ * View history
+
+ [ ] Tools
+ Tools
+ (BUTTON) move to sidebar (BUTTON) hide
+ Actions
+ * Read
+ * Edit
+ * View history
+
+ General
+ * What links here
+ * Related changes
+ * Upload file
+ * Special pages
+ * Permanent link
+ * Page information
+ * Cite this page
+ * Wikidata item
+
+ Print/export
+ * Download as PDF
+ * Printable version
+
+ In other projects
+ * Wikimedia Commons
+ * Wikiversity
+
+ From Wikipedia, the free encyclopedia
+ Field of linguistics and computer science
+ This article is about natural language processing done by computers.
+ For the natural language processing done by the human brain, see
+ Language processing in the brain.
+
+ Natural language processing (NLP) is an interdisciplinary subfield of
+ linguistics, computer science, and artificial intelligence concerned
+ with the interactions between computers and human language, in
+ particular how to program computers to process and analyze large
+ amounts of natural language data. The goal is a computer capable of
+ "understanding" the contents of documents, including the contextual
+ nuances of the language within them. The technology can then accurately
+ extract information and insights contained in the documents as well as
+ categorize and organize the documents themselves.
+
+ Challenges in natural language processing frequently involve speech
+ recognition, natural-language understanding, and natural-language
+ generation.
+
+History[edit]
+
+ Further information: History of natural language processing
+
+ Natural language processing has its roots in the 1950s. Already in
+ 1950, Alan Turing published an article titled "Computing Machinery and
+ Intelligence" which proposed what is now called the Turing test as a
+ criterion of intelligence, though at the time that was not articulated
+ as a problem separate from artificial intelligence. The proposed test
+ includes a task that involves the automated interpretation and
+ generation of natural language.
+
+Symbolic NLP (1950s – early 1990s)[edit]
+
+ The premise of symbolic NLP is well-summarized by John Searle's Chinese
+ room experiment: Given a collection of rules (e.g., a Chinese
+ phrasebook, with questions and matching answers), the computer emulates
+ natural language understanding (or other NLP tasks) by applying those
+ rules to the data it confronts.
+ * 1950s: The Georgetown experiment in 1954 involved fully automatic
+ translation of more than sixty Russian sentences into English. The
+ authors claimed that within three or five years, machine
+ translation would be a solved problem.^[1] However, real progress
+ was much slower, and after the ALPAC report in 1966, which found
+ that ten-year-long research had failed to fulfill the expectations,
+ funding for machine translation was dramatically reduced. Little
+ further research in machine translation was conducted in America
+ (though some research continued elsewhere, such as Japan and
+ Europe^[2]) until the late 1980s when the first statistical machine
+ translation systems were developed.
+ * 1960s: Some notably successful natural language processing systems
+ developed in the 1960s were SHRDLU, a natural language system
+ working in restricted "blocks worlds" with restricted vocabularies,
+ and ELIZA, a simulation of a Rogerian psychotherapist, written by
+ Joseph Weizenbaum between 1964 and 1966. Using almost no
+ information about human thought or emotion, ELIZA sometimes
+ provided a startlingly human-like interaction. When the "patient"
+ exceeded the very small knowledge base, ELIZA might provide a
+ generic response, for example, responding to "My head hurts" with
+ "Why do you say your head hurts?".
+ * 1970s: During the 1970s, many programmers began to write
+ "conceptual ontologies", which structured real-world information
+ into computer-understandable data. Examples are MARGIE (Schank,
+ 1975), SAM (Cullingford, 1978), PAM (Wilensky, 1978), TaleSpin
+ (Meehan, 1976), QUALM (Lehnert, 1977), Politics (Carbonell, 1979),
+ and Plot Units (Lehnert 1981). During this time, the first
+ chatterbots were written (e.g., PARRY).
+ * 1980s: The 1980s and early 1990s mark the heyday of symbolic
+ methods in NLP. Focus areas of the time included research on
+ rule-based parsing (e.g., the development of HPSG as a
+ computational operationalization of generative grammar), morphology
+ (e.g., two-level morphology^[3]), semantics (e.g., Lesk algorithm),
+ reference (e.g., within Centering Theory^[4]) and other areas of
+ natural language understanding (e.g., in the Rhetorical Structure
+ Theory). Other lines of research were continued, e.g., the
+ development of chatterbots with Racter and Jabberwacky. An
+ important development (that eventually led to the statistical turn
+ in the 1990s) was the rising importance of quantitative evaluation
+ in this period.^[5]
+
+Statistical NLP (1990s–2010s)[edit]
+
+ Up to the 1980s, most natural language processing systems were based on
+ complex sets of hand-written rules. Starting in the late 1980s,
+ however, there was a revolution in natural language processing with the
+ introduction of machine learning algorithms for language processing.
+ This was due to both the steady increase in computational power (see
+ Moore's law) and the gradual lessening of the dominance of Chomskyan
+ theories of linguistics (e.g. transformational grammar), whose
+ theoretical underpinnings discouraged the sort of corpus linguistics
+ that underlies the machine-learning approach to language
+ processing.^[6]
+ * 1990s: Many of the notable early successes on statistical methods
+ in NLP occurred in the field of machine translation, due especially
+ to work at IBM Research, such as IBM alignment models. These
+ systems were able to take advantage of existing multilingual
+ textual corpora that had been produced by the Parliament of Canada
+ and the European Union as a result of laws calling for the
+ translation of all governmental proceedings into all official
+ languages of the corresponding systems of government. However, most
+ other systems depended on corpora specifically developed for the
+ tasks implemented by these systems, which was (and often continues
+ to be) a major limitation in the success of these systems. As a
+ result, a great deal of research has gone into methods of more
+ effectively learning from limited amounts of data.
+ * 2000s: With the growth of the web, increasing amounts of raw
+ (unannotated) language data has become available since the
+ mid-1990s. Research has thus increasingly focused on unsupervised
+ and semi-supervised learning algorithms. Such algorithms can learn
+ from data that has not been hand-annotated with the desired answers
+ or using a combination of annotated and non-annotated data.
+ Generally, this task is much more difficult than supervised
+ learning, and typically produces less accurate results for a given
+ amount of input data. However, there is an enormous amount of
+ non-annotated data available (including, among other things, the
+ entire content of the World Wide Web), which can often make up for
+ the inferior results if the algorithm used has a low enough time
+ complexity to be practical.
+
+Neural NLP (present)[edit]
+
+ In 2003, Yoshua Bengio with co-authors tried to use a multi-layer
+ perceptron with a single hidden layer and context length of several
+ words trained on up to 14 million of words with a CPU cluster in
+ language modelling and overperformed the best of n-gram models (a
+ typical statistical alogrithm) then available.^[7] In 2010, Tomáš
+ Mikolov (then a PhD student at Brno University of Technology) with
+ co-authors applied a simple recurrent neural network with a single
+ hidden layer to language modelling,^[8] and in the following years he
+ went on to develop Word2vec.
+
+ In the 2010s, representation learning and deep neural network-style
+ (featuring many hidden layers) machine learning methods became
+ widespread in natural language processing. That popularity was due
+ partly to a flurry of results showing that such techniques^[9]^[10] can
+ achieve state-of-the-art results in many natural language tasks, e.g.,
+ in language modeling^[11] and parsing.^[12]^[13] This is increasingly
+ important in medicine and healthcare, where NLP helps analyze notes and
+ text in electronic health records that would otherwise be inaccessible
+ for study when seeking to improve care^[14] or protect patient
+ privacy.^[15]
+
+Methods: Rules, statistics, neural networks[edit]
+
+ In the early days, many language-processing systems were designed by
+ symbolic methods, i.e., the hand-coding of a set of rules, coupled with
+ a dictionary lookup:^[16]^[17] such as by writing grammars or devising
+ heuristic rules for stemming.
+
+ More recent systems based on machine-learning algorithms have many
+ advantages over hand-produced rules:
+ * The learning procedures used during machine learning automatically
+ focus on the most common cases, whereas when writing rules by hand
+ it is often not at all obvious where the effort should be directed.
+ * Automatic learning procedures can make use of statistical inference
+ algorithms to produce models that are robust to unfamiliar input
+ (e.g. containing words or structures that have not been seen
+ before) and to erroneous input (e.g. with misspelled words or words
+ accidentally omitted). Generally, handling such input gracefully
+ with handwritten rules, or, more generally, creating systems of
+ handwritten rules that make soft decisions, is extremely difficult,
+ error-prone and time-consuming.
+ * Systems based on automatically learning the rules can be made more
+ accurate simply by supplying more input data. However, systems
+ based on handwritten rules can only be made more accurate by
+ increasing the complexity of the rules, which is a much more
+ difficult task. In particular, there is a limit to the complexity
+ of systems based on handwritten rules, beyond which the systems
+ become more and more unmanageable. However, creating more data to
+ input to machine-learning systems simply requires a corresponding
+ increase in the number of man-hours worked, generally without
+ significant increases in the complexity of the annotation process.
+
+ Despite the popularity of machine learning in NLP research, symbolic
+ methods are still (2020) commonly used:
+ * when the amount of training data is insufficient to successfully
+ apply machine learning methods, e.g., for the machine translation
+ of low-resource languages such as provided by the Apertium system,
+ * for preprocessing in NLP pipelines, e.g., tokenization, or
+ * for postprocessing and transforming the output of NLP pipelines,
+ e.g., for knowledge extraction from syntactic parses.
+
+Statistical methods[edit]
+
+ Since the so-called "statistical revolution"^[18]^[19] in the late
+ 1980s and mid-1990s, much natural language processing research has
+ relied heavily on machine learning. The machine-learning paradigm calls
+ instead for using statistical inference to automatically learn such
+ rules through the analysis of large corpora (the plural form of corpus,
+ is a set of documents, possibly with human or computer annotations) of
+ typical real-world examples.
+
+ Many different classes of machine-learning algorithms have been applied
+ to natural-language-processing tasks. These algorithms take as input a
+ large set of "features" that are generated from the input data.
+ Increasingly, however, research has focused on statistical models,
+ which make soft, probabilistic decisions based on attaching real-valued
+ weights to each input feature (complex-valued embeddings,^[20] and
+ neural networks in general have also been proposed, for e.g.
+ speech^[21]). Such models have the advantage that they can express the
+ relative certainty of many different possible answers rather than only
+ one, producing more reliable results when such a model is included as a
+ component of a larger system.
+
+ Some of the earliest-used machine learning algorithms, such as decision
+ trees, produced systems of hard if–then rules similar to existing
+ handwritten rules. However, part-of-speech tagging introduced the use
+ of hidden Markov models to natural language processing, and
+ increasingly, research has focused on statistical models, which make
+ soft, probabilistic decisions based on attaching real-valued weights to
+ the features making up the input data. The cache language models upon
+ which many speech recognition systems now rely are examples of such
+ statistical models. Such models are generally more robust when given
+ unfamiliar input, especially input that contains errors (as is very
+ common for real-world data), and produce more reliable results when
+ integrated into a larger system comprising multiple subtasks.
+
+ Since the neural turn, statistical methods in NLP research have been
+ largely replaced by neural networks. However, they continue to be
+ relevant for contexts in which statistical interpretability and
+ transparency is required.
+
+Neural networks[edit]
+
+ Further information: Artificial neural network
+
+ A major drawback of statistical methods is that they require elaborate
+ feature engineering. Since 2015,^[22] the field has thus largely
+ abandoned statistical methods and shifted to neural networks for
+ machine learning. Popular techniques include the use of word embeddings
+ to capture semantic properties of words, and an increase in end-to-end
+ learning of a higher-level task (e.g., question answering) instead of
+ relying on a pipeline of separate intermediate tasks (e.g.,
+ part-of-speech tagging and dependency parsing). In some areas, this
+ shift has entailed substantial changes in how NLP systems are designed,
+ such that deep neural network-based approaches may be viewed as a new
+ paradigm distinct from statistical natural language processing. For
+ instance, the term neural machine translation (NMT) emphasizes the fact
+ that deep learning-based approaches to machine translation directly
+ learn sequence-to-sequence transformations, obviating the need for
+ intermediate steps such as word alignment and language modeling that
+ was used in statistical machine translation (SMT).
+
+Common NLP tasks[edit]
+
+ The following is a list of some of the most commonly researched tasks
+ in natural language processing. Some of these tasks have direct
+ real-world applications, while others more commonly serve as subtasks
+ that are used to aid in solving larger tasks.
+
+ Though natural language processing tasks are closely intertwined, they
+ can be subdivided into categories for convenience. A coarse division is
+ given below.
+
+Text and speech processing[edit]
+
+ Optical character recognition (OCR)
+ Given an image representing printed text, determine the
+ corresponding text.
+
+ Speech recognition
+ Given a sound clip of a person or people speaking, determine the
+ textual representation of the speech. This is the opposite of
+ text to speech and is one of the extremely difficult problems
+ colloquially termed "AI-complete" (see above). In natural speech
+ there are hardly any pauses between successive words, and thus
+ speech segmentation is a necessary subtask of speech recognition
+ (see below). In most spoken languages, the sounds representing
+ successive letters blend into each other in a process termed
+ coarticulation, so the conversion of the analog signal to
+ discrete characters can be a very difficult process. Also, given
+ that words in the same language are spoken by people with
+ different accents, the speech recognition software must be able
+ to recognize the wide variety of input as being identical to
+ each other in terms of its textual equivalent.
+
+ Speech segmentation
+ Given a sound clip of a person or people speaking, separate it
+ into words. A subtask of speech recognition and typically
+ grouped with it.
+
+ Text-to-speech
+ Given a text, transform those units and produce a spoken
+ representation. Text-to-speech can be used to aid the visually
+ impaired.^[23]
+
+ Word segmentation (Tokenization)
+ Separate a chunk of continuous text into separate words. For a
+ language like English, this is fairly trivial, since words are
+ usually separated by spaces. However, some written languages
+ like Chinese, Japanese and Thai do not mark word boundaries in
+ such a fashion, and in those languages text segmentation is a
+ significant task requiring knowledge of the vocabulary and
+ morphology of words in the language. Sometimes this process is
+ also used in cases like bag of words (BOW) creation in data
+ mining.
+
+Morphological analysis[edit]
+
+ Lemmatization
+ The task of removing inflectional endings only and to return the
+ base dictionary form of a word which is also known as a lemma.
+ Lemmatization is another technique for reducing words to their
+ normalized form. But in this case, the transformation actually
+ uses a dictionary to map words to their actual form.^[24]
+
+ Morphological segmentation
+ Separate words into individual morphemes and identify the class
+ of the morphemes. The difficulty of this task depends greatly on
+ the complexity of the morphology (i.e., the structure of words)
+ of the language being considered. English has fairly simple
+ morphology, especially inflectional morphology, and thus it is
+ often possible to ignore this task entirely and simply model all
+ possible forms of a word (e.g., "open, opens, opened, opening")
+ as separate words. In languages such as Turkish or Meitei, a
+ highly agglutinated Indian language, however, such an approach
+ is not possible, as each dictionary entry has thousands of
+ possible word forms.^[25]
+
+ Part-of-speech tagging
+ Given a sentence, determine the part of speech (POS) for each
+ word. Many words, especially common ones, can serve as multiple
+ parts of speech. For example, "book" can be a noun ("the book on
+ the table") or verb ("to book a flight"); "set" can be a noun,
+ verb or adjective; and "out" can be any of at least five
+ different parts of speech.
+
+ Stemming
+ The process of reducing inflected (or sometimes derived) words
+ to a base form (e.g., "close" will be the root for "closed",
+ "closing", "close", "closer" etc.). Stemming yields similar
+ results as lemmatization, but does so on grounds of rules, not a
+ dictionary.
+
+Syntactic analysis[edit]
+
+ Grammar induction^[26]
+ Generate a formal grammar that describes a language's syntax.
+
+ Sentence breaking (also known as "sentence boundary disambiguation")
+ Given a chunk of text, find the sentence boundaries. Sentence
+ boundaries are often marked by periods or other punctuation
+ marks, but these same characters can serve other purposes (e.g.,
+ marking abbreviations).
+
+ Parsing
+ Determine the parse tree (grammatical analysis) of a given
+ sentence. The grammar for natural languages is ambiguous and
+ typical sentences have multiple possible analyses: perhaps
+ surprisingly, for a typical sentence there may be thousands of
+ potential parses (most of which will seem completely nonsensical
+ to a human). There are two primary types of parsing: dependency
+ parsing and constituency parsing. Dependency parsing focuses on
+ the relationships between words in a sentence (marking things
+ like primary objects and predicates), whereas constituency
+ parsing focuses on building out the parse tree using a
+ probabilistic context-free grammar (PCFG) (see also stochastic
+ grammar).
+
+Lexical semantics (of individual words in context)[edit]
+
+ Lexical semantics
+ What is the computational meaning of individual words in
+ context?
+
+ Distributional semantics
+ How can we learn semantic representations from data?
+
+ Named entity recognition (NER)
+ Given a stream of text, determine which items in the text map to
+ proper names, such as people or places, and what the type of
+ each such name is (e.g. person, location, organization).
+ Although capitalization can aid in recognizing named entities in
+ languages such as English, this information cannot aid in
+ determining the type of named entity, and in any case, is often
+ inaccurate or insufficient. For example, the first letter of a
+ sentence is also capitalized, and named entities often span
+ several words, only some of which are capitalized. Furthermore,
+ many other languages in non-Western scripts (e.g. Chinese or
+ Arabic) do not have any capitalization at all, and even
+ languages with capitalization may not consistently use it to
+ distinguish names. For example, German capitalizes all nouns,
+ regardless of whether they are names, and French and Spanish do
+ not capitalize names that serve as adjectives. Another name for
+ this task is token classification.^[27]
+
+ Sentiment analysis (see also Multimodal sentiment analysis)
+ Extract subjective information usually from a set of documents,
+ often using online reviews to determine "polarity" about
+ specific objects. It is especially useful for identifying trends
+ of public opinion in social media, for marketing.
+
+ Terminology extraction
+ The goal of terminology extraction is to automatically extract
+ relevant terms from a given corpus.
+
+ Word-sense disambiguation (WSD)
+ Many words have more than one meaning; we have to select the
+ meaning which makes the most sense in context. For this problem,
+ we are typically given a list of words and associated word
+ senses, e.g. from a dictionary or an online resource such as
+ WordNet.
+
+ Entity linking
+ Many words—typically proper names—refer to named entities; here
+ we have to select the entity (a famous individual, a location, a
+ company, etc.) which is referred to in context.
+
+Relational semantics (semantics of individual sentences)[edit]
+
+ Relationship extraction
+ Given a chunk of text, identify the relationships among named
+ entities (e.g. who is married to whom).
+
+ Semantic parsing
+ Given a piece of text (typically a sentence), produce a formal
+ representation of its semantics, either as a graph (e.g., in AMR
+ parsing) or in accordance with a logical formalism (e.g., in DRT
+ parsing). This challenge typically includes aspects of several
+ more elementary NLP tasks from semantics (e.g., semantic role
+ labelling, word-sense disambiguation) and can be extended to
+ include full-fledged discourse analysis (e.g., discourse
+ analysis, coreference; see Natural language understanding
+ below).
+
+ Semantic role labelling (see also implicit semantic role labelling
+ below)
+ Given a single sentence, identify and disambiguate semantic
+ predicates (e.g., verbal frames), then identify and classify the
+ frame elements (semantic roles).
+
+Discourse (semantics beyond individual sentences)[edit]
+
+ Coreference resolution
+ Given a sentence or larger chunk of text, determine which words
+ ("mentions") refer to the same objects ("entities"). Anaphora
+ resolution is a specific example of this task, and is
+ specifically concerned with matching up pronouns with the nouns
+ or names to which they refer. The more general task of
+ coreference resolution also includes identifying so-called
+ "bridging relationships" involving referring expressions. For
+ example, in a sentence such as "He entered John's house through
+ the front door", "the front door" is a referring expression and
+ the bridging relationship to be identified is the fact that the
+ door being referred to is the front door of John's house (rather
+ than of some other structure that might also be referred to).
+
+ Discourse analysis
+ This rubric includes several related tasks. One task is
+ discourse parsing, i.e., identifying the discourse structure of
+ a connected text, i.e. the nature of the discourse relationships
+ between sentences (e.g. elaboration, explanation, contrast).
+ Another possible task is recognizing and classifying the speech
+ acts in a chunk of text (e.g. yes-no question, content question,
+ statement, assertion, etc.).
+
+ Implicit semantic role labelling
+ Given a single sentence, identify and disambiguate semantic
+ predicates (e.g., verbal frames) and their explicit semantic
+ roles in the current sentence (see Semantic role labelling
+ above). Then, identify semantic roles that are not explicitly
+ realized in the current sentence, classify them into arguments
+ that are explicitly realized elsewhere in the text and those
+ that are not specified, and resolve the former against the local
+ text. A closely related task is zero anaphora resolution, i.e.,
+ the extension of coreference resolution to pro-drop languages.
+
+ Recognizing textual entailment
+ Given two text fragments, determine if one being true entails
+ the other, entails the other's negation, or allows the other to
+ be either true or false.^[28]
+
+ Topic segmentation and recognition
+ Given a chunk of text, separate it into segments each of which
+ is devoted to a topic, and identify the topic of the segment.
+
+ Argument mining
+ The goal of argument mining is the automatic extraction and
+ identification of argumentative structures from natural language
+ text with the aid of computer programs.^[29] Such argumentative
+ structures include the premise, conclusions, the argument scheme
+ and the relationship between the main and subsidiary argument,
+ or the main and counter-argument within discourse.^[30]^[31]
+
+Higher-level NLP applications[edit]
+
+ Automatic summarization (text summarization)
+ Produce a readable summary of a chunk of text. Often used to
+ provide summaries of the text of a known type, such as research
+ papers, articles in the financial section of a newspaper.
+
+ Grammatical error correction
+ Grammatical error detection and correction involves a great band-width
+ of problems on all levels of linguistic analysis
+ (phonology/orthography, morphology, syntax, semantics, pragmatics).
+ Grammatical error correction is impactful since it affects hundreds of
+ millions of people that use or acquire English as a second language. It
+ has thus been subject to a number of shared tasks since
+ 2011.^[32]^[33]^[34] As far as orthography, morphology, syntax and
+ certain aspects of semantics are concerned, and due to the development
+ of powerful neural language models such as GPT-2, this can now (2019)
+ be considered a largely solved problem and is being marketed in various
+ commercial applications.
+
+ Machine translation (MT)
+
+ Automatically translate text from one human language to another. This
+ is one of the most difficult problems, and is a member of a class of
+ problems colloquially termed "AI-complete", i.e. requiring all of the
+ different types of knowledge that humans possess (grammar, semantics,
+ facts about the real world, etc.) to solve properly.
+
+ Natural-language understanding (NLU)
+ Convert chunks of text into more formal representations such as
+ first-order logic structures that are easier for computer programs to
+ manipulate. Natural language understanding involves the identification
+ of the intended semantic from the multiple possible semantics which can
+ be derived from a natural language expression which usually takes the
+ form of organized notations of natural language concepts. Introduction
+ and creation of language metamodel and ontology are efficient however
+ empirical solutions. An explicit formalization of natural language
+ semantics without confusions with implicit assumptions such as
+ closed-world assumption (CWA) vs. open-world assumption, or subjective
+ Yes/No vs. objective True/False is expected for the construction of a
+ basis of semantics formalization.^[35]
+
+ Natural-language generation (NLG):
+ Convert information from computer databases or semantic intents into
+ readable human language.
+
+ Book generation
+ Not an NLP task proper but an extension of natural language generation
+ and other NLP tasks is the creation of full-fledged books. The first
+ machine-generated book was created by a rule-based system in 1984
+ (Racter, The policeman's beard is half-constructed).^[36] The first
+ published work by a neural network was published in 2018, 1 the Road,
+ marketed as a novel, contains sixty million words. Both these systems
+ are basically elaborate but non-sensical (semantics-free) language
+ models. The first machine-generated science book was published in 2019
+ (Beta Writer, Lithium-Ion Batteries, Springer, Cham).^[37] Unlike
+ Racter and 1 the Road, this is grounded on factual knowledge and based
+ on text summarization.
+
+ Document AI
+ A Document AI platform sits on top of the NLP technology enabling users
+ with no prior experience of artificial intelligence, machine learning
+ or NLP to quickly train a computer to extract the specific data they
+ need from different document types. NLP-powered Document AI enables
+ non-technical teams to quickly access information hidden in documents,
+ for example, lawyers, business analysts and accountants.^[38]
+
+ Dialogue management
+ Computer systems intended to converse with a human.
+
+ Question answering
+ Given a human-language question, determine its answer. Typical
+ questions have a specific right answer (such as "What is the capital of
+ Canada?"), but sometimes open-ended questions are also considered (such
+ as "What is the meaning of life?").
+
+ Text-to-image generation
+ Given a description of an image, generate an image that matches the
+ description.^[39]
+
+ Text-to-scene generation
+ Given a description of a scene, generate a 3D model of the
+ scene.^[40]^[41]
+
+ Text-to-video
+ Given a description of a video, generate a video that matches the
+ description.^[42]^[43]
+
+General tendencies and (possible) future directions[edit]
+
+ Based on long-standing trends in the field, it is possible to
+ extrapolate future directions of NLP. As of 2020, three trends among
+ the topics of the long-standing series of CoNLL Shared Tasks can be
+ observed:^[44]
+ * Interest on increasingly abstract, "cognitive" aspects of natural
+ language (1999–2001: shallow parsing, 2002–03: named entity
+ recognition, 2006–09/2017–18: dependency syntax, 2004–05/2008–09
+ semantic role labelling, 2011–12 coreference, 2015–16: discourse
+ parsing, 2019: semantic parsing).
+ * Increasing interest in multilinguality, and, potentially,
+ multimodality (English since 1999; Spanish, Dutch since 2002;
+ German since 2003; Bulgarian, Danish, Japanese, Portuguese,
+ Slovenian, Swedish, Turkish since 2006; Basque, Catalan, Chinese,
+ Greek, Hungarian, Italian, Turkish since 2007; Czech since 2009;
+ Arabic since 2012; 2017: 40+ languages; 2018: 60+/100+ languages)
+ * Elimination of symbolic representations (rule-based over supervised
+ towards weakly supervised methods, representation learning and
+ end-to-end systems)
+
+ Cognition and NLP[edit]
+
+ Most higher-level NLP applications involve aspects that emulate
+ intelligent behaviour and apparent comprehension of natural language.
+ More broadly speaking, the technical operationalization of increasingly
+ advanced aspects of cognitive behaviour represents one of the
+ developmental trajectories of NLP (see trends among CoNLL shared tasks
+ above).
+
+ Cognition refers to "the mental action or process of acquiring
+ knowledge and understanding through thought, experience, and the
+ senses."^[45] Cognitive science is the interdisciplinary, scientific
+ study of the mind and its processes.^[46] Cognitive linguistics is an
+ interdisciplinary branch of linguistics, combining knowledge and
+ research from both psychology and linguistics.^[47] Especially during
+ the age of symbolic NLP, the area of computational linguistics
+ maintained strong ties with cognitive studies.
+
+ As an example, George Lakoff offers a methodology to build natural
+ language processing (NLP) algorithms through the perspective of
+ cognitive science, along with the findings of cognitive
+ linguistics,^[48] with two defining aspects:
+ 1. Apply the theory of conceptual metaphor, explained by Lakoff as
+ "the understanding of one idea, in terms of another" which provides
+ an idea of the intent of the author.^[49] For example, consider the
+ English word big. When used in a comparison ("That is a big tree"),
+ the author's intent is to imply that the tree is physically large
+ relative to other trees or the authors experience. When used
+ metaphorically ("Tomorrow is a big day"), the author's intent to
+ imply importance. The intent behind other usages, like in "She is a
+ big person", will remain somewhat ambiguous to a person and a
+ cognitive NLP algorithm alike without additional information.
+ 2. Assign relative measures of meaning to a word, phrase, sentence or
+ piece of text based on the information presented before and after
+ the piece of text being analyzed, e.g., by means of a probabilistic
+ context-free grammar (PCFG). The mathematical equation for such
+ algorithms is presented in US patent 9269353 :
+
+ [MATH: <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle
+ displaystyle="true" scriptlevel="0"> <mrow
+ class="MJX-TeXAtom-ORD"> <mi>R</mi> <mi>M</mi> <mi>M</mi>
+ <mo stretchy="false">(</mo> <mi>t</mi> <mi>o</mi>
+ <mi>k</mi> <mi>e</mi> <msub> <mi>n</mi> <mrow
+ class="MJX-TeXAtom-ORD"> <mi>N</mi> </mrow> </msub> <mo
+ stretchy="false">)</mo> </mrow> <mo>=</mo> <mrow
+ class="MJX-TeXAtom-ORD"> <mi>P</mi> <mi>M</mi> <mi>M</mi>
+ <mo stretchy="false">(</mo> <mi>t</mi> <mi>o</mi>
+ <mi>k</mi> <mi>e</mi> <msub> <mi>n</mi> <mrow
+ class="MJX-TeXAtom-ORD"> <mi>N</mi> </mrow> </msub> <mo
+ stretchy="false">)</mo> </mrow> <mo>×</mo> <mrow
+ class="MJX-TeXAtom-ORD"> <mfrac> <mn>1</mn> <mrow>
+ <mn>2</mn> <mi>d</mi> </mrow> </mfrac> </mrow> <mrow>
+ <mo>(</mo> <mrow> <munderover> <mo>∑</mo> <mrow
+ class="MJX-TeXAtom-ORD"> <mi>i</mi> <mo>=</mo> <mo>−</mo>
+ <mi>d</mi> </mrow> <mrow class="MJX-TeXAtom-ORD">
+ <mi>d</mi> </mrow> </munderover> <mrow
+ class="MJX-TeXAtom-ORD"> <mo stretchy="false">(</mo> <mo
+ stretchy="false">(</mo> <mi>P</mi> <mi>M</mi> <mi>M</mi>
+ <mo stretchy="false">(</mo> <mi>t</mi> <mi>o</mi>
+ <mi>k</mi> <mi>e</mi> <msub> <mi>n</mi> <mrow
+ class="MJX-TeXAtom-ORD"> <mi>N</mi> <mo>−</mo> <mn>1</mn>
+ </mrow> </msub> <mo stretchy="false">)</mo> </mrow>
+ <mo>×</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>P</mi>
+ <mi>F</mi> <mo stretchy="false">(</mo> <mi>t</mi>
+ <mi>o</mi> <mi>k</mi> <mi>e</mi> <msub> <mi>n</mi> <mrow
+ class="MJX-TeXAtom-ORD"> <mi>N</mi> </mrow> </msub>
+ <mo>,</mo> <mi>t</mi> <mi>o</mi> <mi>k</mi> <mi>e</mi>
+ <msub> <mi>n</mi> <mrow class="MJX-TeXAtom-ORD">
+ <mi>N</mi> <mo>−</mo> <mn>1</mn> </mrow> </msub> <mo
+ stretchy="false">)</mo> <msub> <mo stretchy="false">)</mo>
+ <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub>
+ </mrow> </mrow> <mo>)</mo> </mrow> </mstyle> </mrow>
+ <annotation encoding="application/x-tex">{\displaystyle
+ {RMM(token_{N})}={PMM(token_{N})}\times {\frac
+ {1}{2d}}\left(\sum _{i=-d}^{d}{((PMM(token_{N-1})}\times
+ {PF(token_{N},token_{N-1}))_{i}}\right)}</annotation>
+ </semantics> :MATH]
+ {\displaystyle {RMM(token_{N})}={PMM(token_{N})}\times
+ {\frac {1}{2d}}\left(\sum
+ _{i=-d}^{d}{((PMM(token_{N-1})}\times
+ {PF(token_{N},token_{N-1}))_{i}}\right)}
+
+ Where,
+
+ RMM, is the Relative Measure of Meaning
+ token, is any block of text, sentence, phrase or
+ word
+ N, is the number of tokens being analyzed
+ PMM, is the Probable Measure of Meaning based on a
+ corpora
+ d, is the location of the token along the sequence
+ of N-1 tokens
+ PF, is the Probability Function specific to a
+ language
+
+ Ties with cognitive linguistics are part of the historical heritage of
+ NLP, but they have been less frequently addressed since the statistical
+ turn during the 1990s. Nevertheless, approaches to develop cognitive
+ models towards technically operationalizable frameworks have been
+ pursued in the context of various frameworks, e.g., of cognitive
+ grammar,^[50] functional grammar,^[51] construction grammar,^[52]
+ computational psycholinguistics and cognitive neuroscience (e.g.,
+ ACT-R), however, with limited uptake in mainstream NLP (as measured by
+ presence on major conferences^[53] of the ACL). More recently, ideas of
+ cognitive NLP have been revived as an approach to achieve
+ explainability, e.g., under the notion of "cognitive AI".^[54]
+ Likewise, ideas of cognitive NLP are inherent to neural models
+ multimodal NLP (although rarely made explicit)^[55] and developments in
+ Artificial intelligence, specifically tools and technologies using
+ Large language model approaches.^[56]
+
+See also[edit]
+
+ * 1 the Road
+ * Automated essay scoring
+ * Biomedical text mining
+ * Compound term processing
+ * Computational linguistics
+ * Computer-assisted reviewing
+ * Controlled natural language
+ * Deep learning
+ * Deep linguistic processing
+ * Distributional semantics
+ * Foreign language reading aid
+ * Foreign language writing aid
+ * Information extraction
+ * Information retrieval
+ * Language and Communication Technologies
+ * Language model
+ * Language technology
+ * Latent semantic indexing
+ * Multi-agent system
+ * Native-language identification
+ * Natural-language programming
+ * Natural-language understanding
+ * Natural-language search
+ * Outline of natural language processing
+ * Query expansion
+ * Query understanding
+ * Reification (linguistics)
+ * Speech processing
+ * Spoken dialogue systems
+ * Text-proofing
+ * Text simplification
+ * Transformer (machine learning model)
+ * Truecasing
+ * Question answering
+ * Word2vec
+
+References[edit]
+
+ 1. ^ Hutchins, J. (2005). "The history of machine translation in a
+ nutshell" (PDF).^[self-published source]
+ 2. ^ "ALPAC: the (in)famous report", John Hutchins, MT News
+ International, no. 14, June 1996, pp. 9-12
+ 3. ^ Koskenniemi, Kimmo (1983), Two-level morphology: A general
+ computational model of word-form recognition and production (PDF),
+ Department of General Linguistics, University of Helsinki
+ 4. ^ Joshi, A. K., & Weinstein, S. (1981, August). Control of
+ Inference: Role of Some Aspects of Discourse Structure-Centering.
+ In IJCAI (pp. 385-387).
+ 5. ^ Guida, G.; Mauri, G. (July 1986). "Evaluation of natural language
+ processing systems: Issues and approaches". Proceedings of the
+ IEEE. 74 (7): 1026–1035. doi:10.1109/PROC.1986.13580.
+ ISSN 1558-2256. S2CID 30688575.
+ 6. ^ Chomskyan linguistics encourages the investigation of "corner
+ cases" that stress the limits of its theoretical models (comparable
+ to pathological phenomena in mathematics), typically created using
+ thought experiments, rather than the systematic investigation of
+ typical phenomena that occur in real-world data, as is the case in
+ corpus linguistics. The creation and use of such corpora of
+ real-world data is a fundamental part of machine-learning
+ algorithms for natural language processing. In addition,
+ theoretical underpinnings of Chomskyan linguistics such as the
+ so-called "poverty of the stimulus" argument entail that general
+ learning algorithms, as are typically used in machine learning,
+ cannot be successful in language processing. As a result, the
+ Chomskyan paradigm discouraged the application of such models to
+ language processing.
+ 7. ^ Bengio, Yoshua; Ducharme, Réjean; Vincent, Pascal; Janvin,
+ Christian (March 1, 2003). "A neural probabilistic language model".
+ The Journal of Machine Learning Research. 3: 1137–1155 – via ACM
+ Digital Library.
+ 8. ^ https://gwern.net/doc/ai/nn/rnn/2010-mikolov.pdf
+ 9. ^ Goldberg, Yoav (2016). "A Primer on Neural Network Models for
+ Natural Language Processing". Journal of Artificial Intelligence
+ Research. 57: 345–420. arXiv:1807.10854. doi:10.1613/jair.4992.
+ S2CID 8273530.
+ 10. ^ Goodfellow, Ian; Bengio, Yoshua; Courville, Aaron (2016). Deep
+ Learning. MIT Press.
+ 11. ^ Jozefowicz, Rafal; Vinyals, Oriol; Schuster, Mike; Shazeer, Noam;
+ Wu, Yonghui (2016). Exploring the Limits of Language Modeling.
+ arXiv:1602.02410. Bibcode:2016arXiv160202410J.
+ 12. ^ Choe, Do Kook; Charniak, Eugene. "Parsing as Language Modeling".
+ Emnlp 2016. Archived from the original on 2018-10-23. Retrieved
+ 2018-10-22.
+ 13. ^ Vinyals, Oriol; et al. (2014). "Grammar as a Foreign Language"
+ (PDF). Nips2015. arXiv:1412.7449. Bibcode:2014arXiv1412.7449V.
+ 14. ^ Turchin, Alexander; Florez Builes, Luisa F. (2021-03-19). "Using
+ Natural Language Processing to Measure and Improve Quality of
+ Diabetes Care: A Systematic Review". Journal of Diabetes Science
+ and Technology. 15 (3): 553–560. doi:10.1177/19322968211000831.
+ ISSN 1932-2968. PMC 8120048. PMID 33736486.
+ 15. ^ Lee, Jennifer; Yang, Samuel; Holland-Hall, Cynthia; Sezgin, Emre;
+ Gill, Manjot; Linwood, Simon; Huang, Yungui; Hoffman, Jeffrey
+ (2022-06-10). "Prevalence of Sensitive Terms in Clinical Notes
+ Using Natural Language Processing Techniques: Observational Study".
+ JMIR Medical Informatics. 10 (6): e38482. doi:10.2196/38482.
+ ISSN 2291-9694. PMC 9233261. PMID 35687381.
+ 16. ^ Winograd, Terry (1971). Procedures as a Representation for Data
+ in a Computer Program for Understanding Natural Language (Thesis).
+ 17. ^ Schank, Roger C.; Abelson, Robert P. (1977). Scripts, Plans,
+ Goals, and Understanding: An Inquiry Into Human Knowledge
+ Structures. Hillsdale: Erlbaum. ISBN 0-470-99033-3.
+ 18. ^ Mark Johnson. How the statistical revolution changes
+ (computational) linguistics. Proceedings of the EACL 2009 Workshop
+ on the Interaction between Linguistics and Computational
+ Linguistics.
+ 19. ^ Philip Resnik. Four revolutions. Language Log, February 5, 2011.
+ 20. ^ "Investigating complex-valued representation in NLP" (PDF).
+ 21. ^ Trabelsi, Chiheb; Bilaniuk, Olexa; Zhang, Ying; Serdyuk, Dmitriy;
+ Subramanian, Sandeep; Santos, João Felipe; Mehri, Soroush;
+ Rostamzadeh, Negar; Bengio, Yoshua; Pal, Christopher J.
+ (2018-02-25). "Deep Complex Networks". arXiv:1705.09792 [cs.NE].
+ 22. ^ Socher, Richard. "Deep Learning For NLP-ACL 2012 Tutorial".
+ www.socher.org. Retrieved 2020-08-17. This was an early Deep
+ Learning tutorial at the ACL 2012 and met with both interest and
+ (at the time) skepticism by most participants. Until then, neural
+ learning was basically rejected because of its lack of statistical
+ interpretability. Until 2015, deep learning had evolved into the
+ major framework of NLP. [Link is broken, try
+ http://web.stanford.edu/class/cs224n/]
+ 23. ^ Yi, Chucai; Tian, Yingli (2012), "Assistive Text Reading from
+ Complex Background for Blind Persons", Camera-Based Document
+ Analysis and Recognition, Springer Berlin Heidelberg, pp. 15–28,
+ CiteSeerX 10.1.1.668.869, doi:10.1007/978-3-642-29364-1_2,
+ ISBN 9783642293634
+ 24. ^ "What is Natural Language Processing? Intro to NLP in Machine
+ Learning". GyanSetu!. 2020-12-06. Retrieved 2021-01-09.
+ 25. ^ Kishorjit, N.; Vidya, Raj RK.; Nirmal, Y.; Sivaji, B. (2012).
+ "Manipuri Morpheme Identification" (PDF). Proceedings of the 3rd
+ Workshop on South and Southeast Asian Natural Language Processing
+ (SANLP). COLING 2012, Mumbai, December 2012: 95–108.{{cite
+ journal}}: CS1 maint: location (link)
+ 26. ^ Klein, Dan; Manning, Christopher D. (2002). "Natural language
+ grammar induction using a constituent-context model" (PDF).
+ Advances in Neural Information Processing Systems.
+ 27. ^ Kariampuzha, William; Alyea, Gioconda; Qu, Sue; Sanjak, Jaleal;
+ Mathé, Ewy; Sid, Eric; Chatelaine, Haley; Yadaw, Arjun; Xu, Yanji;
+ Zhu, Qian (2023). "Precision information extraction for rare
+ disease epidemiology at scale". Journal of Translational Medicine.
+ 21 (1): 157. doi:10.1186/s12967-023-04011-y. PMC 9972634.
+ PMID 36855134.
+ 28. ^ PASCAL Recognizing Textual Entailment Challenge (RTE-7)
+ https://tac.nist.gov//2011/RTE/
+ 29. ^ Lippi, Marco; Torroni, Paolo (2016-04-20). "Argumentation Mining:
+ State of the Art and Emerging Trends". ACM Transactions on Internet
+ Technology. 16 (2): 1–25. doi:10.1145/2850417. hdl:11585/523460.
+ ISSN 1533-5399. S2CID 9561587.
+ 30. ^ "Argument Mining - IJCAI2016 Tutorial". www.i3s.unice.fr.
+ Retrieved 2021-03-09.
+ 31. ^ "NLP Approaches to Computational Argumentation – ACL 2016,
+ Berlin". Retrieved 2021-03-09.
+ 32. ^ Administration. "Centre for Language Technology (CLT)". Macquarie
+ University. Retrieved 2021-01-11.
+ 33. ^ "Shared Task: Grammatical Error Correction". www.comp.nus.edu.sg.
+ Retrieved 2021-01-11.
+ 34. ^ "Shared Task: Grammatical Error Correction". www.comp.nus.edu.sg.
+ Retrieved 2021-01-11.
+ 35. ^ Duan, Yucong; Cruz, Christophe (2011). "Formalizing Semantic of
+ Natural Language through Conceptualization from Existence".
+ International Journal of Innovation, Management and Technology. 2
+ (1): 37–42. Archived from the original on 2011-10-09.
+ 36. ^ "U B U W E B :: Racter". www.ubu.com. Retrieved 2020-08-17.
+ 37. ^ Writer, Beta (2019). Lithium-Ion Batteries.
+ doi:10.1007/978-3-030-16800-1. ISBN 978-3-030-16799-8.
+ S2CID 155818532.
+ 38. ^ "Document Understanding AI on Google Cloud (Cloud Next '19) -
+ YouTube". www.youtube.com. Archived from the original on
+ 2021-10-30. Retrieved 2021-01-11.
+ 39. ^ Robertson, Adi (2022-04-06). "OpenAI's DALL-E AI image generator
+ can now edit pictures, too". The Verge. Retrieved 2022-06-07.
+ 40. ^ "The Stanford Natural Language Processing Group".
+ nlp.stanford.edu. Retrieved 2022-06-07.
+ 41. ^ Coyne, Bob; Sproat, Richard (2001-08-01). "WordsEye: an automatic
+ text-to-scene conversion system". Proceedings of the 28th Annual
+ Conference on Computer Graphics and Interactive Techniques.
+ SIGGRAPH '01. New York, NY, USA: Association for Computing
+ Machinery: 487–496. doi:10.1145/383259.383316.
+ ISBN 978-1-58113-374-5. S2CID 3842372.
+ 42. ^ "Google announces AI advances in text-to-video, language
+ translation, more". VentureBeat. 2022-11-02. Retrieved 2022-11-09.
+ 43. ^ Vincent, James (2022-09-29). "Meta's new text-to-video AI
+ generator is like DALL-E for video". The Verge. Retrieved
+ 2022-11-09.
+ 44. ^ "Previous shared tasks | CoNLL". www.conll.org. Retrieved
+ 2021-01-11.
+ 45. ^ "Cognition". Lexico. Oxford University Press and Dictionary.com.
+ Archived from the original on July 15, 2020. Retrieved 6 May 2020.
+ 46. ^ "Ask the Cognitive Scientist". American Federation of Teachers. 8
+ August 2014. "Cognitive science is an interdisciplinary field of
+ researchers from Linguistics, psychology, neuroscience, philosophy,
+ computer science, and anthropology that seek to understand the
+ mind."
+ 47. ^ Robinson, Peter (2008). Handbook of Cognitive Linguistics and
+ Second Language Acquisition. Routledge. pp. 3–8.
+ ISBN 978-0-805-85352-0.
+ 48. ^ Lakoff, George (1999). Philosophy in the Flesh: The Embodied Mind
+ and Its Challenge to Western Philosophy; Appendix: The Neural
+ Theory of Language Paradigm. New York Basic Books. pp. 569–583.
+ ISBN 978-0-465-05674-3.
+ 49. ^ Strauss, Claudia (1999). A Cognitive Theory of Cultural Meaning.
+ Cambridge University Press. pp. 156–164. ISBN 978-0-521-59541-4.
+ 50. ^ "Universal Conceptual Cognitive Annotation (UCCA)". Universal
+ Conceptual Cognitive Annotation (UCCA). Retrieved 2021-01-11.
+ 51. ^ Rodríguez, F. C., & Mairal-Usón, R. (2016). Building an RRG
+ computational grammar. Onomazein, (34), 86-117.
+ 52. ^ "Fluid Construction Grammar – A fully operational processing
+ system for construction grammars". Retrieved 2021-01-11.
+ 53. ^ "ACL Member Portal | The Association for Computational
+ Linguistics Member Portal". www.aclweb.org. Retrieved 2021-01-11.
+ 54. ^ "Chunks and Rules". www.w3.org. Retrieved 2021-01-11.
+ 55. ^ Socher, Richard; Karpathy, Andrej; Le, Quoc V.; Manning,
+ Christopher D.; Ng, Andrew Y. (2014). "Grounded Compositional
+ Semantics for Finding and Describing Images with Sentences".
+ Transactions of the Association for Computational Linguistics. 2:
+ 207–218. doi:10.1162/tacl_a_00177. S2CID 2317858.
+ 56. ^ Dasgupta, Ishita; Lampinen, Andrew K.; Chan, Stephanie C. Y.;
+ Creswell, Antonia; Kumaran, Dharshan; McClelland, James L.; Hill,
+ Felix (2022). "Language models show human-like content effects on
+ reasoning, Dasgupta, Lampinen et al". arXiv:2207.07051 [cs.CL].
+
+Further reading[edit]
+
+ *
+
+ Bates, M (1995). "Models of natural language understanding".
+ Proceedings of the National Academy of Sciences of the United States of
+ America. 92 (22): 9977–9982. Bibcode:1995PNAS...92.9977B.
+ doi:10.1073/pnas.92.22.9977. PMC 40721. PMID 7479812.
+
+ Steven Bird, Ewan Klein, and Edward Loper (2009). Natural Language
+ Processing with Python. O'Reilly Media. ISBN 978-0-596-51649-9.
+
+ Daniel Jurafsky and James H. Martin (2008). Speech and Language
+ Processing, 2nd edition. Pearson Prentice Hall. ISBN 978-0-13-187321-6.
+
+ Mohamed Zakaria Kurdi (2016). Natural Language Processing and
+ Computational Linguistics: speech, morphology, and syntax, Volume 1.
+ ISTE-Wiley. ISBN 978-1848218482.
+
+ Mohamed Zakaria Kurdi (2017). Natural Language Processing and
+ Computational Linguistics: semantics, discourse, and applications,
+ Volume 2. ISTE-Wiley. ISBN 978-1848219212.
+
+ Christopher D. Manning, Prabhakar Raghavan, and Hinrich Schütze
+ (2008). Introduction to Information Retrieval. Cambridge University
+ Press. ISBN 978-0-521-86571-5. Official html and pdf versions available
+ without charge.
+
+ Christopher D. Manning and Hinrich Schütze (1999). Foundations of
+ Statistical Natural Language Processing. The MIT Press.
+ ISBN 978-0-262-13360-9.
+
+ David M. W. Powers and Christopher C. R. Turk (1989). Machine
+ Learning of Natural Language. Springer-Verlag. ISBN 978-0-387-19557-5.
+
+External links[edit]
+
+ * Media related to Natural language processing at Wikimedia Commons
+
+ * v
+ * t
+ * e
+
+ Natural language processing
+
+ General terms
+
+ * AI-complete
+ * Bag-of-words
+ * n-gram
+ + Bigram
+ + Trigram
+ * Computational linguistics
+ * Natural-language understanding
+ * Stop words
+ * Text processing
+
+ Text analysis
+
+ * Collocation extraction
+ * Concept mining
+ * Coreference resolution
+ * Deep linguistic processing
+ * Distant reading
+ * Information extraction
+ * Named-entity recognition
+ * Ontology learning
+ * Parsing
+ * Part-of-speech tagging
+ * Semantic role labeling
+ * Semantic similarity
+ * Sentiment analysis
+ * Terminology extraction
+ * Text mining
+ * Textual entailment
+ * Truecasing
+ * Word-sense disambiguation
+ * Word-sense induction
+
+ Text segmentation
+ * Compound-term processing
+ * Lemmatisation
+ * Lexical analysis
+ * Text chunking
+ * Stemming
+ * Sentence segmentation
+ * Word segmentation
+
+ Automatic summarization
+
+ * Multi-document summarization
+ * Sentence extraction
+ * Text simplification
+
+ Machine translation
+
+ * Computer-assisted
+ * Example-based
+ * Rule-based
+ * Statistical
+ * Transfer-based
+ * Neural
+
+ Distributional semantics models
+
+ * BERT
+ * Document-term matrix
+ * Explicit semantic analysis
+ * fastText
+ * GloVe
+ * Language model (large)
+ * Latent semantic analysis
+ * Seq2seq
+ * Word embedding
+ * Word2vec
+
+ Language resources,
+ datasets and corpora
+
+ Types and
+ standards
+ * Corpus linguistics
+ * Lexical resource
+ * Linguistic Linked Open Data
+ * Machine-readable dictionary
+ * Parallel text
+ * PropBank
+ * Semantic network
+ * Simple Knowledge Organization System
+ * Speech corpus
+ * Text corpus
+ * Thesaurus (information retrieval)
+ * Treebank
+ * Universal Dependencies
+
+ Data
+ * BabelNet
+ * Bank of English
+ * DBpedia
+ * FrameNet
+ * Google Ngram Viewer
+ * UBY
+ * WordNet
+
+ Automatic identification
+ and data capture
+
+ * Speech recognition
+ * Speech segmentation
+ * Speech synthesis
+ * Natural language generation
+ * Optical character recognition
+
+ Topic model
+
+ * Document classification
+ * Latent Dirichlet allocation
+ * Pachinko allocation
+
+ Computer-assisted
+ reviewing
+
+ * Automated essay scoring
+ * Concordancer
+ * Grammar checker
+ * Predictive text
+ * Pronunciation assessment
+ * Spell checker
+ * Syntax guessing
+
+ Natural language
+ user interface
+
+ * Chatbot
+ * Interactive fiction
+ * Question answering
+ * Virtual assistant
+ * Voice user interface
+
+ Related
+
+ * Hallucination
+ * Natural Language Toolkit
+ * spaCy
+
+ Portal:
+ * icon Language
+
+ Authority control: National Edit this at Wikidata
+ * Israel
+ * United States
+ * Japan
+ * Czech Republic
+
+ Retrieved from
+ "https://en.wikipedia.org/w/index.php?title=Natural_language_processing
+ &oldid=1165047033"
+
+ Categories:
+ * Natural language processing
+ * Computational fields of study
+ * Computational linguistics
+ * Speech recognition
+
+ Hidden categories:
+ * All accuracy disputes
+ * Accuracy disputes from December 2013
+ * CS1 maint: location
+ * Articles with short description
+ * Short description is different from Wikidata
+ * Commons category link from Wikidata
+ * Articles with J9U identifiers
+ * Articles with LCCN identifiers
+ * Articles with NDL identifiers
+ * Articles with NKC identifiers
+
+ * This page was last edited on 12 July 2023, at 17:43 (UTC).
+ * Text is available under the Creative Commons Attribution-ShareAlike
+ License 4.0; additional terms may apply. By using this site, you
+ agree to the Terms of Use and Privacy Policy. Wikipedia® is a
+ registered trademark of the Wikimedia Foundation, Inc., a
+ non-profit organization.
+
+ * Privacy policy
+ * About Wikipedia
+ * Disclaimers
+ * Contact Wikipedia
+ * Code of Conduct
+ * Mobile view
+ * Developers
+ * Statistics
+ * Cookie statement
+
+ * Wikimedia Foundation
+ * Powered by MediaWiki
+
+ (BUTTON) Toggle limited content width
diff --git a/examples/scriptators/data/UIMA.txt b/examples/scriptators/data/UIMA.txt
new file mode 100644
index 0000000..d5d46f6
--- /dev/null
+++ b/examples/scriptators/data/UIMA.txt
@@ -0,0 +1,384 @@
+ #alternate Edit this page Wikipedia (en) Wikipedia Atom feed
+
+ Jump to content
+
+ [ ] Main menu
+ Main menu
+ (BUTTON) move to sidebar (BUTTON) hide
+ Navigation
+ * Main page
+ * Contents
+ * Current events
+ * Random article
+ * About Wikipedia
+ * Contact us
+ * Donate
+
+ Contribute
+ * Help
+ * Learn to edit
+ * Community portal
+ * Recent changes
+ * Upload file
+
+ Languages
+ Language links are at the top of the page across from the title.
+ Wikipedia The Free Encyclopedia
+ Search
+ ____________________
+ (BUTTON) Search
+
+ * Create account
+ * Log in
+
+ [ ] Personal tools
+ * Create account
+ * Log in
+
+ Pages for logged out editors learn more
+ * Contributions
+ * Talk
+
+ [ ]
+
+Contents
+
+ (BUTTON) move to sidebar (BUTTON) hide
+ * (Top)
+ * 1Structure
+ * 2Implementations and uses
+ * 3See also
+ * 4References
+ * 5External links
+
+ Toggle the table of contents
+
+ [ ] Toggle the table of contents
+
+UIMA
+
+ [ ] 4 languages
+ * Deutsch
+ * Español
+ * Italiano
+ * 中文
+
+ Edit links
+
+ * Article
+ * Talk
+
+ [ ] English
+
+ * Read
+ * Edit
+ * View history
+
+ [ ] Tools
+ Tools
+ (BUTTON) move to sidebar (BUTTON) hide
+ Actions
+ * Read
+ * Edit
+ * View history
+
+ General
+ * What links here
+ * Related changes
+ * Upload file
+ * Special pages
+ * Permanent link
+ * Page information
+ * Cite this page
+ * Wikidata item
+
+ Print/export
+ * Download as PDF
+ * Printable version
+
+ In other projects
+ * Wikimedia Commons
+
+ From Wikipedia, the free encyclopedia
+
+ UIMA (/juˈiːmə/ yoo-EE-mə),^[1] short for Unstructured Information
+ Management Architecture, is an OASIS standard^[2] for content
+ analytics, originally developed at IBM. It provides a component
+ software architecture for the development, discovery, composition, and
+ deployment of multi-modal analytics for the analysis of unstructured
+ information and integration with search technologies.
+
+Structure[edit]
+
+ The UIMA architecture can be thought of in four dimensions:
+ 1. It specifies component interfaces in an analytics pipeline.
+ 2. It describes a set of design patterns.
+ 3. It suggests two data representations: an in-memory representation
+ of annotations for high-performance analytics and an XML
+ representation of annotations for integration with remote web
+ services.
+ 4. It suggests development roles allowing tools to be used by users
+ with diverse skills.
+
+Implementations and uses[edit]
+
+ CAPTION: Apache UIMA
+
+ [120px-Apache_UIMA_logo.svg.png]
+ Developer(s) IBM, Apache Software Foundation (since October 2006)
+ Stable release
+ 3.1.1 / November 8, 2019; 3 years ago (2019-11-08)^[3]
+ Repository
+ * svn.apache.org/repos/asf/uima/ Edit this at Wikidata
+
+ Written in Java with C++ enablement
+ Operating system cross-platform
+ Type text mining, information extraction
+ License Apache License 2.0
+ Website uima.apache.org
+
+ Apache UIMA, a reference implementation of UIMA, is maintained by the
+ Apache Software Foundation.
+
+ UIMA is used in a number of software projects:
+ * IBM Research's Watson uses UIMA for analyzing unstructured
+ data.^[4]
+ * The Clinical Text Analysis and Knowledge Extraction System (Apache
+ cTAKES) is a UIMA-based system for information extraction from
+ medical records.
+ * DKPro Core is a collection of reusable UIMA components for
+ general-purpose natural language processing.
+
+See also[edit]
+
+ * Data Discovery and Query Builder
+ * Entity extraction
+ * General Architecture for Text Engineering (GATE)
+ * IBM Omnifind
+ * LanguageWare
+
+References[edit]
+
+ 1. ^ UIMA Frequently Asked Questions (FAQ's) The Apache Software
+ Foundation
+ 2. ^ UIMA Specification The Apache Software Foundation.
+ 3. ^ "Apache UIMA - News". uima.apache.org. Retrieved 11 December
+ 2019.
+ 4. ^ "Apache Innovation Bolsters IBM's "Smartest Machine on Earth" in
+ First-ever Man vs. Machine Competition on Jeopardy! Quiz Show : The
+ Apache Software Foundation Blog". blogs.apache.org. Retrieved 23
+ April 2018.
+
+External links[edit]
+
+ * Apache UIMA home page
+
+ * v
+ * t
+ * e
+
+ The Apache Software Foundation
+
+ Top-level
+ projects
+
+ * Accumulo
+ * ActiveMQ
+ * AGE
+ * Airflow
+ * Ambari
+ * Ant
+ * Aries
+ * Arrow
+ * Apache HTTP Server
+ * APR
+ * Avro
+ * Axis
+ * Axis2
+ * Beam
+ * Bloodhound
+ * Brooklyn
+ * Buildr
+ * Calcite
+ * Camel
+ * CarbonData
+ * Cassandra
+ * Cayenne
+ * Chemistry
+ * CloudStack
+ * Cocoon
+ * Cordova
+ * CouchDB
+ * cTAKES
+ * CXF
+ * Derby
+ * Directory
+ * Drill
+ * Druid
+ * Empire-db
+ * Felix
+ * Flex
+ * Flink
+ * Flume
+ * FreeMarker
+ * Geronimo
+ * Giraph
+ * Gump
+ * Hadoop
+ * HBase
+ * Helix
+ * Hive
+ * Impala
+ * Jackrabbit
+ * James
+ * Jena
+ * Jini
+ * JMeter
+ * Kafka
+ * Kudu
+ * Kylin
+ * Lucene
+ * Mahout
+ * Maven
+ * MINA
+ * mod_perl
+ * MyFaces
+ * NiFi
+ * NetBeans
+ * Nutch
+ * OFBiz
+ * Oozie
+ * OpenEJB
+ * OpenJPA
+ * OpenNLP
+ * OрenOffice
+ * ORC
+ * PDFBox
+ * Parquet
+ * Phoenix
+ * POI
+ * Pig
+ * Pinot
+ * Pivot
+ * Qpid
+ * Roller
+ * RocketMQ
+ * Samza
+ * ServiceMix
+ * Shiro
+ * SINGA
+ * Sling
+ * Solr
+ * Spark
+ * Storm
+ * SpamAssassin
+ * Struts 1
+ * Struts 2
+ * Subversion
+ * Superset
+ * SystemDS
+ * Tapestry
+ * Thrift
+ * Tika
+ * Tomcat
+ * Trafodion
+ * Traffic Server
+ * UIMA
+ * Velocity
+ * Wicket
+ * Xalan
+ * Xerces
+ * XMLBeans
+ * Yetus
+ * ZooKeeper
+
+ Commons
+
+ * BCEL
+ * BSF
+ * Daemon
+ * Jelly
+ * Logging
+
+ Incubator
+
+ * MXNet
+ * NuttX
+ * Taverna
+
+ Other projects
+
+ * Batik
+ * Chainsaw
+ * FOP
+ * Ivy
+ * Log4j
+
+ Attic
+
+ * Abdera
+ * Apex
+ * AxKit
+ * Beehive
+ * Bluesky
+ * iBATIS
+ * C++ Standard Library
+ * Cactus
+ * Click
+ * Continuum
+ * Deltacloud
+ * Etch
+ * Excalibur
+ * Forrest
+ * Hama
+ * Harmony
+ * HiveMind
+ * Jakarta
+ * Lenya
+ * Marmotta
+ * ODE
+ * Shale
+ * Shindig
+ * Slide
+ * Sqoop
+ * Stanbol
+ * Tuscany
+ * Wave
+ * Wink
+ * XML
+
+ Licenses
+
+ * Apache License
+
+ * Category
+
+ Retrieved from
+ "https://en.wikipedia.org/w/index.php?title=UIMA&oldid=1085866372"
+
+ Categories:
+ * Apache Software Foundation projects
+ * Software architecture
+ * Data mining and machine learning software
+
+ * This page was last edited on 2 May 2022, at 21:46 (UTC).
+ * Text is available under the Creative Commons Attribution-ShareAlike
+ License 4.0; additional terms may apply. By using this site, you
+ agree to the Terms of Use and Privacy Policy. Wikipedia® is a
+ registered trademark of the Wikimedia Foundation, Inc., a
+ non-profit organization.
+
+ * Privacy policy
+ * About Wikipedia
+ * Disclaimers
+ * Contact Wikipedia
+ * Code of Conduct
+ * Mobile view
+ * Developers
+ * Statistics
+ * Cookie statement
+
+ * Wikimedia Foundation
+ * Powered by MediaWiki
+
+ (BUTTON) Toggle limited content width
diff --git a/examples/scriptators/perltator/desc/Perltator.xml b/examples/scriptators/perltator/desc/Perltator.xml
new file mode 100644
index 0000000..4a5f6d8
--- /dev/null
+++ b/examples/scriptators/perltator/desc/Perltator.xml
@@ -0,0 +1,109 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+
+ <!--
+ ***************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ ***************************************************************
+ -->
+
+<taeDescription xmlns="http://uima.apache.org/resourceSpecifier">
+ <frameworkImplementation>org.apache.uima.cpp</frameworkImplementation>
+ <primitive>true</primitive>
+<annotatorImplementationName>libperltator</annotatorImplementationName>
+
+
+<analysisEngineMetaData>
+ <name>Perl Sample</name>
+ <description>Perl Annotator</description>
+ <version>1.0</version>
+ <vendor>Apache</vendor>
+
+
+<!--
+ Configuration Parameter Definitions
+-->
+ <configurationParameters>
+ <configurationParameter>
+ <name>DebugLevel</name>
+ <description>Sets the perl annotators debug level</description>
+ <type>Integer</type>
+ <multiValued>false</multiValued>
+ <mandatory>true</mandatory>
+ </configurationParameter>
+ <configurationParameter>
+ <name>SourceFile</name>
+ <description>Perl code to be loaded</description>
+ <type>String</type>
+ <multiValued>false</multiValued>
+ <mandatory>true</mandatory>
+ </configurationParameter>
+ </configurationParameters>
+
+<!--
+ Values for the configuration parameters
+-->
+ <configurationParameterSettings>
+ <nameValuePair>
+ <name>SourceFile</name>
+ <value>
+ <string>/usr/local/uimacpp/scripts/ae.pl</string>
+ </value>
+ </nameValuePair>
+ <nameValuePair>
+ <name>DebugLevel</name>
+ <value>
+ <integer>101</integer>
+ </value>
+ </nameValuePair>
+ </configurationParameterSettings>
+
+
+<!--
+ TypeSystem Definition
+-->
+
+<typeSystemDescription>
+ <types>
+ <typeDescription>
+ <name>org.apache.uima.Span</name>
+ <description></description>
+ <supertypeName>uima.tcas.Annotation</supertypeName>
+ <features>
+ </features>
+ </typeDescription>
+ </types>
+</typeSystemDescription>
+
+
+<!--
+Capabilities: Inputs, Outputs, and Preconditions
+-->
+<capabilities>
+ <capability>
+ <inputs/>
+ <outputs>
+ <type allAnnotatorFeatures="true">org.apache.uima.Span</type>
+ </outputs>
+ <languagesSupported>
+ <language>x-unspecified</language>
+ </languagesSupported>
+ </capability>
+</capabilities>
+
+</analysisEngineMetaData>
+</taeDescription>
diff --git a/examples/scriptators/perltator/scripts/ae.pl b/examples/scriptators/perltator/scripts/ae.pl
new file mode 100644
index 0000000..9f648c0
--- /dev/null
+++ b/examples/scriptators/perltator/scripts/ae.pl
@@ -0,0 +1,105 @@
+#!/usr/bin/perl
+
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing,
+ # software distributed under the License is distributed on an
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ # KIND, either express or implied. See the License for the
+ # specific language governing permissions and limitations
+ # under the License.
+
+use perltator;
+use strict;
+
+sub initialize {
+ $main::ac = shift;
+ $main::thisScript = $main::ac->extractValue("SourceFile");
+ $main::debug = $main::ac->extractIntegerValue("DebugLevel");
+ if ($main::debug > 0) {
+ print "$main::thisScript: debug = $main::debug \n";
+ }
+}
+
+sub typeSystemInit {
+ my $ts = shift;
+ if ($main::debug > 10) {
+ print "$main::thisScript: Type sytem init called"
+ }
+ my $spantype = "org.apache.uima.Span";
+ $main::spantype = $ts->getType($spantype);
+ if (!$main::spantype->isValid()) {
+ my $error = "$main::thisScript: $spantype is NOT found in type system!";
+ $main::ac->logError($error);
+ # set eval error to cause annotator to exit prematurely
+ $@ = $error;
+ }
+}
+
+#
+# the process method is passed two parameters, the CAS and
+# the ResultsSpecification
+sub process {
+ my ($tcas, $rs) = @_;
+ if ($main::debug > 10) {
+ print "$main::thisScript: This is a process function\n";
+ }
+ my $text = $tcas->getDocumentText();
+ my $index = $tcas->getIndexRepository();
+ my $annotCount = 0;
+ my @lines = split(/\n/, $text);
+ my $offset = 0;
+ my $inSpan = 1;
+ my $start = 0;
+ foreach my$line(@lines) {
+ if ($line=~m/^\s*$/){
+ if ($inSpan) {
+ if ($start != $offset) {
+ my $fs = $tcas->createAnnotation($main::spantype, $start, $offset);
+ $index->addFS($fs);
+ $annotCount++;
+ }
+ $inSpan = 0;
+ }
+ } elsif (! $inSpan) {
+ $start = $offset;
+ $inSpan = 1;
+ }
+ $offset += length($line);
+ }
+ if ($inSpan) {
+ my $fs = $tcas->createAnnotation($main::spantype, $start, length($text));
+ $index->addFS($fs);
+ $annotCount++;
+ }
+
+ if ($main::debug > 10) {
+ print "$main::thisScript: created $annotCount annotations\n";
+ }
+ if ($main::debug > 20) {
+ my $annots = 0;
+ my $iterator = $tcas->getAnnotationIndex($main::spantype)->iterator();
+ while ($iterator->isValid()) {
+ $annots += 1;
+ if ($main::debug > 30) {
+ my $anno = $iterator->get();
+ my $text = $anno->getCoveredText();
+ if (length($text)>40) {
+ $text = substr($text,0,20) . "...";
+ }
+ $text =~ s/\n+/ /g;
+ print "Annotation type=", $main::spantype->getName(),": \"$text\"\n";
+ }
+ $iterator->moveToNext();
+ }
+ print "$main::thisScript: found $annots annotations\n";
+ }
+}
diff --git a/examples/scriptators/perltator/scripts/perltator.pm b/examples/scriptators/perltator/scripts/perltator.pm
new file mode 100644
index 0000000..ad83142
--- /dev/null
+++ b/examples/scriptators/perltator/scripts/perltator.pm
@@ -0,0 +1,546 @@
+# This file was automatically generated by SWIG (https://www.swig.org).
+# Version 4.1.0
+#
+# Do not make changes to this file unless you know what you are doing - modify
+# the SWIG interface file instead.
+
+package perltator;
+use base qw(Exporter);
+use base qw(DynaLoader);
+package perltatorc;
+bootstrap perltator;
+package perltator;
+@EXPORT = qw();
+
+# ---------- BASE METHODS -------------
+
+package perltator;
+
+sub TIEHASH {
+ my ($classname,$obj) = @_;
+ return bless $obj, $classname;
+}
+
+sub CLEAR { }
+
+sub FIRSTKEY { }
+
+sub NEXTKEY { }
+
+sub FETCH {
+ my ($self,$field) = @_;
+ my $member_func = "swig_${field}_get";
+ $self->$member_func();
+}
+
+sub STORE {
+ my ($self,$field,$newval) = @_;
+ my $member_func = "swig_${field}_set";
+ $self->$member_func($newval);
+}
+
+sub this {
+ my $ptr = shift;
+ return tied(%$ptr);
+}
+
+
+# ------- FUNCTION WRAPPERS --------
+
+package perltator;
+
+
+############# Class : perltator::Type ##############
+
+package perltator::Type;
+use vars qw(@ISA %OWNER %ITERATORS %BLESSEDMEMBERS);
+@ISA = qw( perltator );
+%OWNER = ();
+%ITERATORS = ();
+*getName = *perltatorc::Type_getName;
+*isValid = *perltatorc::Type_isValid;
+*getFeatureByBaseName = *perltatorc::Type_getFeatureByBaseName;
+*getAppropriateFeatures = *perltatorc::Type_getAppropriateFeatures;
+sub new {
+ my $pkg = shift;
+ my $self = perltatorc::new_Type(@_);
+ bless $self, $pkg if defined($self);
+}
+
+sub DESTROY {
+ return unless $_[0]->isa('HASH');
+ my $self = tied(%{$_[0]});
+ return unless defined $self;
+ delete $ITERATORS{$self};
+ if (exists $OWNER{$self}) {
+ perltatorc::delete_Type($self);
+ delete $OWNER{$self};
+ }
+}
+
+sub DISOWN {
+ my $self = shift;
+ my $ptr = tied(%$self);
+ delete $OWNER{$ptr};
+}
+
+sub ACQUIRE {
+ my $self = shift;
+ my $ptr = tied(%$self);
+ $OWNER{$ptr} = 1;
+}
+
+
+############# Class : perltator::Feature ##############
+
+package perltator::Feature;
+use vars qw(@ISA %OWNER %ITERATORS %BLESSEDMEMBERS);
+@ISA = qw( perltator );
+%OWNER = ();
+%ITERATORS = ();
+*isValid = *perltatorc::Feature_isValid;
+*getName = *perltatorc::Feature_getName;
+*getRangeType = *perltatorc::Feature_getRangeType;
+*getIntroType = *perltatorc::Feature_getIntroType;
+sub new {
+ my $pkg = shift;
+ my $self = perltatorc::new_Feature(@_);
+ bless $self, $pkg if defined($self);
+}
+
+sub DESTROY {
+ return unless $_[0]->isa('HASH');
+ my $self = tied(%{$_[0]});
+ return unless defined $self;
+ delete $ITERATORS{$self};
+ if (exists $OWNER{$self}) {
+ perltatorc::delete_Feature($self);
+ delete $OWNER{$self};
+ }
+}
+
+sub DISOWN {
+ my $self = shift;
+ my $ptr = tied(%$self);
+ delete $OWNER{$ptr};
+}
+
+sub ACQUIRE {
+ my $self = shift;
+ my $ptr = tied(%$self);
+ $OWNER{$ptr} = 1;
+}
+
+
+############# Class : perltator::TypeSystem ##############
+
+package perltator::TypeSystem;
+use vars qw(@ISA %OWNER %ITERATORS %BLESSEDMEMBERS);
+@ISA = qw( perltator );
+%OWNER = ();
+*getType = *perltatorc::TypeSystem_getType;
+sub DISOWN {
+ my $self = shift;
+ my $ptr = tied(%$self);
+ delete $OWNER{$ptr};
+}
+
+sub ACQUIRE {
+ my $self = shift;
+ my $ptr = tied(%$self);
+ $OWNER{$ptr} = 1;
+}
+
+
+############# Class : perltator::CAS ##############
+
+package perltator::CAS;
+use vars qw(@ISA %OWNER %ITERATORS %BLESSEDMEMBERS);
+@ISA = qw( perltator );
+%OWNER = ();
+*iterator = *perltatorc::CAS_iterator;
+*getIndexRepository = *perltatorc::CAS_getIndexRepository;
+*createFS = *perltatorc::CAS_createFS;
+*getTypeSystem = *perltatorc::CAS_getTypeSystem;
+*getAnnotationIndex = *perltatorc::CAS_getAnnotationIndex;
+*setDocumentText = *perltatorc::CAS_setDocumentText;
+*getDocumentText = *perltatorc::CAS_getDocumentText;
+*setSofaDataString = *perltatorc::CAS_setSofaDataString;
+*setSofaDataArray = *perltatorc::CAS_setSofaDataArray;
+*setSofaDataURI = *perltatorc::CAS_setSofaDataURI;
+*getSofaDataURI = *perltatorc::CAS_getSofaDataURI;
+*createAnnotation = *perltatorc::CAS_createAnnotation;
+*getView = *perltatorc::CAS_getView;
+*createView = *perltatorc::CAS_createView;
+*getSofa = *perltatorc::CAS_getSofa;
+sub DISOWN {
+ my $self = shift;
+ my $ptr = tied(%$self);
+ delete $OWNER{$ptr};
+}
+
+sub ACQUIRE {
+ my $self = shift;
+ my $ptr = tied(%$self);
+ $OWNER{$ptr} = 1;
+}
+
+
+############# Class : perltator::FeatureStructure ##############
+
+package perltator::FeatureStructure;
+use vars qw(@ISA %OWNER %ITERATORS %BLESSEDMEMBERS);
+@ISA = qw( perltator );
+%OWNER = ();
+%ITERATORS = ();
+*getCAS = *perltatorc::FeatureStructure_getCAS;
+*isValid = *perltatorc::FeatureStructure_isValid;
+*getType = *perltatorc::FeatureStructure_getType;
+*clone = *perltatorc::FeatureStructure_clone;
+*isUntouchedFSValue = *perltatorc::FeatureStructure_isUntouchedFSValue;
+*getFSValue = *perltatorc::FeatureStructure_getFSValue;
+*setFSValue = *perltatorc::FeatureStructure_setFSValue;
+*getIntValue = *perltatorc::FeatureStructure_getIntValue;
+*setIntValue = *perltatorc::FeatureStructure_setIntValue;
+*getFloatValue = *perltatorc::FeatureStructure_getFloatValue;
+*setFloatValue = *perltatorc::FeatureStructure_setFloatValue;
+*getStringValue = *perltatorc::FeatureStructure_getStringValue;
+*setStringValue = *perltatorc::FeatureStructure_setStringValue;
+sub new {
+ my $pkg = shift;
+ my $self = perltatorc::new_FeatureStructure(@_);
+ bless $self, $pkg if defined($self);
+}
+
+sub DESTROY {
+ return unless $_[0]->isa('HASH');
+ my $self = tied(%{$_[0]});
+ return unless defined $self;
+ delete $ITERATORS{$self};
+ if (exists $OWNER{$self}) {
+ perltatorc::delete_FeatureStructure($self);
+ delete $OWNER{$self};
+ }
+}
+
+sub DISOWN {
+ my $self = shift;
+ my $ptr = tied(%$self);
+ delete $OWNER{$ptr};
+}
+
+sub ACQUIRE {
+ my $self = shift;
+ my $ptr = tied(%$self);
+ $OWNER{$ptr} = 1;
+}
+
+
+############# Class : perltator::AnnotationFS ##############
+
+package perltator::AnnotationFS;
+use vars qw(@ISA %OWNER %ITERATORS %BLESSEDMEMBERS);
+@ISA = qw( perltator::FeatureStructure perltator );
+%OWNER = ();
+%ITERATORS = ();
+*getBeginPosition = *perltatorc::AnnotationFS_getBeginPosition;
+*getEndPosition = *perltatorc::AnnotationFS_getEndPosition;
+*getLength = *perltatorc::AnnotationFS_getLength;
+*getCoveredText = *perltatorc::AnnotationFS_getCoveredText;
+sub new {
+ my $pkg = shift;
+ my $self = perltatorc::new_AnnotationFS(@_);
+ bless $self, $pkg if defined($self);
+}
+
+sub DESTROY {
+ return unless $_[0]->isa('HASH');
+ my $self = tied(%{$_[0]});
+ return unless defined $self;
+ delete $ITERATORS{$self};
+ if (exists $OWNER{$self}) {
+ perltatorc::delete_AnnotationFS($self);
+ delete $OWNER{$self};
+ }
+}
+
+sub DISOWN {
+ my $self = shift;
+ my $ptr = tied(%$self);
+ delete $OWNER{$ptr};
+}
+
+sub ACQUIRE {
+ my $self = shift;
+ my $ptr = tied(%$self);
+ $OWNER{$ptr} = 1;
+}
+
+
+############# Class : perltator::SofaFS ##############
+
+package perltator::SofaFS;
+use vars qw(@ISA %OWNER %ITERATORS %BLESSEDMEMBERS);
+@ISA = qw( perltator::FeatureStructure perltator );
+%OWNER = ();
+%ITERATORS = ();
+*getSofaURI = *perltatorc::SofaFS_getSofaURI;
+*getSofaMime = *perltatorc::SofaFS_getSofaMime;
+sub new {
+ my $pkg = shift;
+ my $self = perltatorc::new_SofaFS(@_);
+ bless $self, $pkg if defined($self);
+}
+
+sub DESTROY {
+ return unless $_[0]->isa('HASH');
+ my $self = tied(%{$_[0]});
+ return unless defined $self;
+ delete $ITERATORS{$self};
+ if (exists $OWNER{$self}) {
+ perltatorc::delete_SofaFS($self);
+ delete $OWNER{$self};
+ }
+}
+
+sub DISOWN {
+ my $self = shift;
+ my $ptr = tied(%$self);
+ delete $OWNER{$ptr};
+}
+
+sub ACQUIRE {
+ my $self = shift;
+ my $ptr = tied(%$self);
+ $OWNER{$ptr} = 1;
+}
+
+
+############# Class : perltator::FSIndexRepository ##############
+
+package perltator::FSIndexRepository;
+use vars qw(@ISA %OWNER %ITERATORS %BLESSEDMEMBERS);
+@ISA = qw( perltator );
+%OWNER = ();
+*getIndex = *perltatorc::FSIndexRepository_getIndex;
+*getIndexByType = *perltatorc::FSIndexRepository_getIndexByType;
+*addFS = *perltatorc::FSIndexRepository_addFS;
+sub DISOWN {
+ my $self = shift;
+ my $ptr = tied(%$self);
+ delete $OWNER{$ptr};
+}
+
+sub ACQUIRE {
+ my $self = shift;
+ my $ptr = tied(%$self);
+ $OWNER{$ptr} = 1;
+}
+
+
+############# Class : perltator::FSIterator ##############
+
+package perltator::FSIterator;
+use vars qw(@ISA %OWNER %ITERATORS %BLESSEDMEMBERS);
+@ISA = qw( perltator );
+%OWNER = ();
+%ITERATORS = ();
+*isValid = *perltatorc::FSIterator_isValid;
+*get = *perltatorc::FSIterator_get;
+*moveToNext = *perltatorc::FSIterator_moveToNext;
+*moveToPrevious = *perltatorc::FSIterator_moveToPrevious;
+*moveToFirst = *perltatorc::FSIterator_moveToFirst;
+*moveToLast = *perltatorc::FSIterator_moveToLast;
+*moveTo = *perltatorc::FSIterator_moveTo;
+*peekNext = *perltatorc::FSIterator_peekNext;
+*peekPrevious = *perltatorc::FSIterator_peekPrevious;
+sub new {
+ my $pkg = shift;
+ my $self = perltatorc::new_FSIterator(@_);
+ bless $self, $pkg if defined($self);
+}
+
+sub DESTROY {
+ return unless $_[0]->isa('HASH');
+ my $self = tied(%{$_[0]});
+ return unless defined $self;
+ delete $ITERATORS{$self};
+ if (exists $OWNER{$self}) {
+ perltatorc::delete_FSIterator($self);
+ delete $OWNER{$self};
+ }
+}
+
+sub DISOWN {
+ my $self = shift;
+ my $ptr = tied(%$self);
+ delete $OWNER{$ptr};
+}
+
+sub ACQUIRE {
+ my $self = shift;
+ my $ptr = tied(%$self);
+ $OWNER{$ptr} = 1;
+}
+
+
+############# Class : perltator::ANIterator ##############
+
+package perltator::ANIterator;
+use vars qw(@ISA %OWNER %ITERATORS %BLESSEDMEMBERS);
+@ISA = qw( perltator::FSIterator perltator );
+%OWNER = ();
+*get = *perltatorc::ANIterator_get;
+sub DISOWN {
+ my $self = shift;
+ my $ptr = tied(%$self);
+ delete $OWNER{$ptr};
+}
+
+sub ACQUIRE {
+ my $self = shift;
+ my $ptr = tied(%$self);
+ $OWNER{$ptr} = 1;
+}
+
+
+############# Class : perltator::ResultSpecification ##############
+
+package perltator::ResultSpecification;
+use vars qw(@ISA %OWNER %ITERATORS %BLESSEDMEMBERS);
+@ISA = qw( perltator );
+%OWNER = ();
+sub DISOWN {
+ my $self = shift;
+ my $ptr = tied(%$self);
+ delete $OWNER{$ptr};
+}
+
+sub ACQUIRE {
+ my $self = shift;
+ my $ptr = tied(%$self);
+ $OWNER{$ptr} = 1;
+}
+
+
+############# Class : perltator::AnnotatorContext ##############
+
+package perltator::AnnotatorContext;
+use vars qw(@ISA %OWNER %ITERATORS %BLESSEDMEMBERS);
+@ISA = qw( perltator );
+%OWNER = ();
+*extractValue = *perltatorc::AnnotatorContext_extractValue;
+*extractStringValue = *perltatorc::AnnotatorContext_extractStringValue;
+*extractIntegerValue = *perltatorc::AnnotatorContext_extractIntegerValue;
+*extractFloatValue = *perltatorc::AnnotatorContext_extractFloatValue;
+*logMessage = *perltatorc::AnnotatorContext_logMessage;
+*logWarning = *perltatorc::AnnotatorContext_logWarning;
+*logError = *perltatorc::AnnotatorContext_logError;
+sub DISOWN {
+ my $self = shift;
+ my $ptr = tied(%$self);
+ delete $OWNER{$ptr};
+}
+
+sub ACQUIRE {
+ my $self = shift;
+ my $ptr = tied(%$self);
+ $OWNER{$ptr} = 1;
+}
+
+
+############# Class : perltator::FSIndex ##############
+
+package perltator::FSIndex;
+use vars qw(@ISA %OWNER %ITERATORS %BLESSEDMEMBERS);
+@ISA = qw( perltator );
+%OWNER = ();
+%ITERATORS = ();
+*isValid = *perltatorc::FSIndex_isValid;
+*getSize = *perltatorc::FSIndex_getSize;
+*find = *perltatorc::FSIndex_find;
+*iterator = *perltatorc::FSIndex_iterator;
+sub new {
+ my $pkg = shift;
+ my $self = perltatorc::new_FSIndex(@_);
+ bless $self, $pkg if defined($self);
+}
+
+sub DESTROY {
+ return unless $_[0]->isa('HASH');
+ my $self = tied(%{$_[0]});
+ return unless defined $self;
+ delete $ITERATORS{$self};
+ if (exists $OWNER{$self}) {
+ perltatorc::delete_FSIndex($self);
+ delete $OWNER{$self};
+ }
+}
+
+sub DISOWN {
+ my $self = shift;
+ my $ptr = tied(%$self);
+ delete $OWNER{$ptr};
+}
+
+sub ACQUIRE {
+ my $self = shift;
+ my $ptr = tied(%$self);
+ $OWNER{$ptr} = 1;
+}
+
+
+############# Class : perltator::ANIndex ##############
+
+package perltator::ANIndex;
+use vars qw(@ISA %OWNER %ITERATORS %BLESSEDMEMBERS);
+@ISA = qw( perltator::FSIndex perltator );
+%OWNER = ();
+%ITERATORS = ();
+*iterator = *perltatorc::ANIndex_iterator;
+sub new {
+ my $pkg = shift;
+ my $self = perltatorc::new_ANIndex(@_);
+ bless $self, $pkg if defined($self);
+}
+
+sub DESTROY {
+ return unless $_[0]->isa('HASH');
+ my $self = tied(%{$_[0]});
+ return unless defined $self;
+ delete $ITERATORS{$self};
+ if (exists $OWNER{$self}) {
+ perltatorc::delete_ANIndex($self);
+ delete $OWNER{$self};
+ }
+}
+
+sub DISOWN {
+ my $self = shift;
+ my $ptr = tied(%$self);
+ delete $OWNER{$ptr};
+}
+
+sub ACQUIRE {
+ my $self = shift;
+ my $ptr = tied(%$self);
+ $OWNER{$ptr} = 1;
+}
+
+
+# ------- VARIABLE STUBS --------
+
+package perltator;
+
+
+# these are default methods that the user overrides, they
+# are here to prevent errors if they are not defined
+package main;
+sub initialize {};
+sub typeSystemInit {};
+sub destroy {};
+sub process {};
+sub reconfigure {};
+sub batchProcessComplete {};
+sub collectionProcessComplete {};
+1;
diff --git a/examples/scriptators/pythonnator/desc/Pythonnator.xml b/examples/scriptators/pythonnator/desc/Pythonnator.xml
new file mode 100644
index 0000000..d46a041
--- /dev/null
+++ b/examples/scriptators/pythonnator/desc/Pythonnator.xml
@@ -0,0 +1,109 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+
+ <!--
+ ***************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ ***************************************************************
+ -->
+
+<taeDescription xmlns="http://uima.apache.org/resourceSpecifier">
+ <frameworkImplementation>org.apache.uima.cpp</frameworkImplementation>
+ <primitive>true</primitive>
+<annotatorImplementationName>libpythonnator</annotatorImplementationName>
+
+
+<analysisEngineMetaData>
+ <name>Python Annotator</name>
+ <description>Run a python script</description>
+ <version>1.0</version>
+ <vendor>Apache</vendor>
+
+
+<!--
+ Configuration Parameter Definitions
+-->
+ <configurationParameters>
+ <configurationParameter>
+ <name>DebugLevel</name>
+ <description>Sets the python annotator debug level</description>
+ <type>Integer</type>
+ <multiValued>false</multiValued>
+ <mandatory>false</mandatory>
+ </configurationParameter>
+ <configurationParameter>
+ <name>SourceFile</name>
+ <description>Python module to be loaded from the PYTHONPATH</description>
+ <type>String</type>
+ <multiValued>false</multiValued>
+ <mandatory>true</mandatory>
+ </configurationParameter>
+ </configurationParameters>
+
+<!--
+ Values for the configuration parameters
+-->
+ <configurationParameterSettings>
+ <nameValuePair>
+ <name>SourceFile</name>
+ <value>
+ <string>ae</string>
+ </value>
+ </nameValuePair>
+ <nameValuePair>
+ <name>DebugLevel</name>
+ <value>
+ <integer>101</integer>
+ </value>
+ </nameValuePair>
+ </configurationParameterSettings>
+
+
+<!--
+ TypeSystem Definition
+-->
+
+<typeSystemDescription>
+ <types>
+ <typeDescription>
+ <name>org.apache.uima.Span</name>
+ <description></description>
+ <supertypeName>uima.tcas.Annotation</supertypeName>
+ <features>
+ </features>
+ </typeDescription>
+ </types>
+</typeSystemDescription>
+
+
+<!--
+Capabilities: Inputs, Outputs, and Preconditions
+-->
+<capabilities>
+ <capability>
+ <inputs/>
+ <outputs>
+ <type allAnnotatorFeatures="true">org.apache.uima.Span</type>
+ </outputs>
+ <languagesSupported>
+ <language>x-unspecified</language>
+ </languagesSupported>
+ </capability>
+</capabilities>
+
+</analysisEngineMetaData>
+</taeDescription>
diff --git a/examples/scriptators/pythonnator/scripts/ae.py b/examples/scriptators/pythonnator/scripts/ae.py
new file mode 100644
index 0000000..170a1a3
--- /dev/null
+++ b/examples/scriptators/pythonnator/scripts/ae.py
@@ -0,0 +1,95 @@
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing,
+ # software distributed under the License is distributed on an
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ # KIND, either express or implied. See the License for the
+ # specific language governing permissions and limitations
+ # under the License.
+
+import re
+import math
+
+# everything in the global namespace is eval'ed during initialization
+# by the Pythonnator UIMA annotator
+def initialize(annotContext):
+ global pattern
+ global source
+ global debug
+ global ac
+ ac = annotContext
+ source = ac.extractValue("SourceFile")
+ debug = ac.extractIntegerValue("DebugLevel")
+ if debug > 0:
+ print(source + ": initialized")
+
+
+def typeSystemInit(ts):
+ global source
+ global debug
+ global ac
+ if debug > 10:
+ print(source + ": Type sytem init called")
+ global spantype
+ spantype =ts.getType('org.apache.uima.Span')
+ if not spantype.isValid():
+ error = source + ": org.apache.uima.Span is NOT found in type system!"
+ ac.logError(error)
+ raise Exception(error)
+
+#
+# the process method is passed two parameters, the CAS and
+# the ResultsSpecification
+def process(tcas, rs):
+ global spantype
+ global source
+ global debug
+ global ac
+ if debug > 10:
+ print(source + ": This is a process function")
+ ac.logMessage("process called")
+ text = tcas.getDocumentText()
+ index = tcas.getIndexRepository()
+ lines = text.split('\n')
+ inSpan = True
+ offset = 0
+ annotCount = 0
+ start = 0
+ print(len(lines))
+ for line in lines:
+ #print(line.encode('utf-8').decode('ascii', errors='ignore'))
+ if line.strip() == "":
+ if inSpan:
+ if start != offset:
+ fs = tcas.createAnnotation(spantype, start, offset)
+ index.addFS(fs)
+ annotCount += 1
+ inSpan = False
+ # else: pass, # all white space
+ elif not inSpan:
+ inSpan = True
+ start = offset
+ # else; pass # span grows
+ offset += len(line)
+ if inSpan:
+ fs = tcas.createAnnotation(spantype, start, len(text))
+ index.addFS(fs)
+ annotCount += 1
+
+ if debug > 0:
+ print(source + ": created " + str(annotCount) + " annotations")
+ if debug > 20:
+ annots = 0
+ iterator = tcas.getAnnotationIndex(spantype).iterator()
+ while iterator.isValid():
+ annots += 1
+ iterator.moveToNext()
+ print(source + ": found " + str(annots) + " annotations")
diff --git a/examples/scriptators/pythonnator/scripts/pythonnator.py b/examples/scriptators/pythonnator/scripts/pythonnator.py
new file mode 100644
index 0000000..df133ad
--- /dev/null
+++ b/examples/scriptators/pythonnator/scripts/pythonnator.py
@@ -0,0 +1,399 @@
+# This file was automatically generated by SWIG (https://www.swig.org).
+# Version 4.1.0
+#
+# Do not make changes to this file unless you know what you are doing - modify
+# the SWIG interface file instead.
+
+from sys import version_info as _swig_python_version_info
+# Import the low-level C/C++ module
+if __package__ or "." in __name__:
+ from . import _pythonnator
+else:
+ import _pythonnator
+
+try:
+ import builtins as __builtin__
+except ImportError:
+ import __builtin__
+
+def _swig_repr(self):
+ try:
+ strthis = "proxy of " + self.this.__repr__()
+ except __builtin__.Exception:
+ strthis = ""
+ return "<%s.%s; %s >" % (self.__class__.__module__, self.__class__.__name__, strthis,)
+
+
+def _swig_setattr_nondynamic_instance_variable(set):
+ def set_instance_attr(self, name, value):
+ if name == "this":
+ set(self, name, value)
+ elif name == "thisown":
+ self.this.own(value)
+ elif hasattr(self, name) and isinstance(getattr(type(self), name), property):
+ set(self, name, value)
+ else:
+ raise AttributeError("You cannot add instance attributes to %s" % self)
+ return set_instance_attr
+
+
+def _swig_setattr_nondynamic_class_variable(set):
+ def set_class_attr(cls, name, value):
+ if hasattr(cls, name) and not isinstance(getattr(cls, name), property):
+ set(cls, name, value)
+ else:
+ raise AttributeError("You cannot add class attributes to %s" % cls)
+ return set_class_attr
+
+
+def _swig_add_metaclass(metaclass):
+ """Class decorator for adding a metaclass to a SWIG wrapped class - a slimmed down version of six.add_metaclass"""
+ def wrapper(cls):
+ return metaclass(cls.__name__, cls.__bases__, cls.__dict__.copy())
+ return wrapper
+
+
+class _SwigNonDynamicMeta(type):
+ """Meta class to enforce nondynamic attributes (no new attributes) for a class"""
+ __setattr__ = _swig_setattr_nondynamic_class_variable(type.__setattr__)
+
+
+class Type(object):
+ thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag")
+ __repr__ = _swig_repr
+
+ def getName(self):
+ return _pythonnator.Type_getName(self)
+
+ def isValid(self):
+ return _pythonnator.Type_isValid(self)
+
+ def getFeatureByBaseName(self, feature):
+ return _pythonnator.Type_getFeatureByBaseName(self, feature)
+
+ def getAppropriateFeatures(self):
+ return _pythonnator.Type_getAppropriateFeatures(self)
+
+ def __init__(self):
+ _pythonnator.Type_swiginit(self, _pythonnator.new_Type())
+ __swig_destroy__ = _pythonnator.delete_Type
+
+# Register Type in _pythonnator:
+_pythonnator.Type_swigregister(Type)
+class Feature(object):
+ thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag")
+ __repr__ = _swig_repr
+
+ def isValid(self):
+ return _pythonnator.Feature_isValid(self)
+
+ def getName(self):
+ return _pythonnator.Feature_getName(self)
+
+ def getRangeType(self):
+ return _pythonnator.Feature_getRangeType(self)
+
+ def getIntroType(self):
+ return _pythonnator.Feature_getIntroType(self)
+
+ def __init__(self):
+ _pythonnator.Feature_swiginit(self, _pythonnator.new_Feature())
+ __swig_destroy__ = _pythonnator.delete_Feature
+
+# Register Feature in _pythonnator:
+_pythonnator.Feature_swigregister(Feature)
+class TypeSystem(object):
+ thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag")
+
+ def __init__(self, *args, **kwargs):
+ raise AttributeError("No constructor defined")
+ __repr__ = _swig_repr
+
+ def getType(self, crName):
+ return _pythonnator.TypeSystem_getType(self, crName)
+
+# Register TypeSystem in _pythonnator:
+_pythonnator.TypeSystem_swigregister(TypeSystem)
+class CAS(object):
+ thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag")
+
+ def __init__(self, *args, **kwargs):
+ raise AttributeError("No constructor defined")
+ __repr__ = _swig_repr
+
+ def iterator(self):
+ return _pythonnator.CAS_iterator(self)
+
+ def getIndexRepository(self):
+ return _pythonnator.CAS_getIndexRepository(self)
+
+ def createFS(self, arg2):
+ return _pythonnator.CAS_createFS(self, arg2)
+
+ def getTypeSystem(self):
+ return _pythonnator.CAS_getTypeSystem(self)
+
+ def getAnnotationIndex(self, type):
+ return _pythonnator.CAS_getAnnotationIndex(self, type)
+
+ def setDocumentText(self, text):
+ return _pythonnator.CAS_setDocumentText(self, text)
+
+ def getDocumentText(self):
+ return _pythonnator.CAS_getDocumentText(self)
+
+ def setSofaDataString(self, text, mimetype):
+ return _pythonnator.CAS_setSofaDataString(self, text, mimetype)
+
+ def setSofaDataArray(self, array, mime):
+ return _pythonnator.CAS_setSofaDataArray(self, array, mime)
+
+ def setSofaDataURI(self, uri, mime):
+ return _pythonnator.CAS_setSofaDataURI(self, uri, mime)
+
+ def getSofaDataURI(self):
+ return _pythonnator.CAS_getSofaDataURI(self)
+
+ def createAnnotation(self, type, uiBeginPos, uiEndPos):
+ return _pythonnator.CAS_createAnnotation(self, type, uiBeginPos, uiEndPos)
+
+ def getView(self, viewName):
+ return _pythonnator.CAS_getView(self, viewName)
+
+ def createView(self, sofaName):
+ return _pythonnator.CAS_createView(self, sofaName)
+
+ def getSofa(self, *args):
+ return _pythonnator.CAS_getSofa(self, *args)
+
+# Register CAS in _pythonnator:
+_pythonnator.CAS_swigregister(CAS)
+class FeatureStructure(object):
+ thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag")
+ __repr__ = _swig_repr
+
+ def getCAS(self):
+ return _pythonnator.FeatureStructure_getCAS(self)
+
+ def isValid(self):
+ return _pythonnator.FeatureStructure_isValid(self)
+
+ def getType(self):
+ return _pythonnator.FeatureStructure_getType(self)
+
+ def clone(self, *args):
+ return _pythonnator.FeatureStructure_clone(self, *args)
+
+ def isUntouchedFSValue(self, crFeature):
+ return _pythonnator.FeatureStructure_isUntouchedFSValue(self, crFeature)
+
+ def getFSValue(self, crFeature):
+ return _pythonnator.FeatureStructure_getFSValue(self, crFeature)
+
+ def setFSValue(self, crFeature, anFS):
+ return _pythonnator.FeatureStructure_setFSValue(self, crFeature, anFS)
+
+ def getIntValue(self, crFeature):
+ return _pythonnator.FeatureStructure_getIntValue(self, crFeature)
+
+ def setIntValue(self, crFeature, i):
+ return _pythonnator.FeatureStructure_setIntValue(self, crFeature, i)
+
+ def getFloatValue(self, crFeature):
+ return _pythonnator.FeatureStructure_getFloatValue(self, crFeature)
+
+ def setFloatValue(self, crFeature, arg3):
+ return _pythonnator.FeatureStructure_setFloatValue(self, crFeature, arg3)
+
+ def getStringValue(self, crFeature):
+ return _pythonnator.FeatureStructure_getStringValue(self, crFeature)
+
+ def setStringValue(self, crFeature, cuStr):
+ return _pythonnator.FeatureStructure_setStringValue(self, crFeature, cuStr)
+
+ def __init__(self):
+ _pythonnator.FeatureStructure_swiginit(self, _pythonnator.new_FeatureStructure())
+ __swig_destroy__ = _pythonnator.delete_FeatureStructure
+
+# Register FeatureStructure in _pythonnator:
+_pythonnator.FeatureStructure_swigregister(FeatureStructure)
+class AnnotationFS(FeatureStructure):
+ thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag")
+ __repr__ = _swig_repr
+
+ def getBeginPosition(self):
+ return _pythonnator.AnnotationFS_getBeginPosition(self)
+
+ def getEndPosition(self):
+ return _pythonnator.AnnotationFS_getEndPosition(self)
+
+ def getLength(self):
+ return _pythonnator.AnnotationFS_getLength(self)
+
+ def getCoveredText(self):
+ return _pythonnator.AnnotationFS_getCoveredText(self)
+
+ def __init__(self):
+ _pythonnator.AnnotationFS_swiginit(self, _pythonnator.new_AnnotationFS())
+ __swig_destroy__ = _pythonnator.delete_AnnotationFS
+
+# Register AnnotationFS in _pythonnator:
+_pythonnator.AnnotationFS_swigregister(AnnotationFS)
+class SofaFS(FeatureStructure):
+ thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag")
+ __repr__ = _swig_repr
+
+ def getSofaURI(self):
+ return _pythonnator.SofaFS_getSofaURI(self)
+
+ def getSofaMime(self):
+ return _pythonnator.SofaFS_getSofaMime(self)
+
+ def __init__(self):
+ _pythonnator.SofaFS_swiginit(self, _pythonnator.new_SofaFS())
+ __swig_destroy__ = _pythonnator.delete_SofaFS
+
+# Register SofaFS in _pythonnator:
+_pythonnator.SofaFS_swigregister(SofaFS)
+class FSIndexRepository(object):
+ thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag")
+
+ def __init__(self, *args, **kwargs):
+ raise AttributeError("No constructor defined")
+ __repr__ = _swig_repr
+
+ def getIndex(self, crLabel):
+ return _pythonnator.FSIndexRepository_getIndex(self, crLabel)
+
+ def getIndexByType(self, crLabel, type):
+ return _pythonnator.FSIndexRepository_getIndexByType(self, crLabel, type)
+
+ def addFS(self, fs):
+ return _pythonnator.FSIndexRepository_addFS(self, fs)
+
+# Register FSIndexRepository in _pythonnator:
+_pythonnator.FSIndexRepository_swigregister(FSIndexRepository)
+class FSIterator(object):
+ thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag")
+ __repr__ = _swig_repr
+
+ def isValid(self):
+ return _pythonnator.FSIterator_isValid(self)
+
+ def get(self):
+ return _pythonnator.FSIterator_get(self)
+
+ def moveToNext(self):
+ return _pythonnator.FSIterator_moveToNext(self)
+
+ def moveToPrevious(self):
+ return _pythonnator.FSIterator_moveToPrevious(self)
+
+ def moveToFirst(self):
+ return _pythonnator.FSIterator_moveToFirst(self)
+
+ def moveToLast(self):
+ return _pythonnator.FSIterator_moveToLast(self)
+
+ def moveTo(self, fs):
+ return _pythonnator.FSIterator_moveTo(self, fs)
+
+ def peekNext(self):
+ return _pythonnator.FSIterator_peekNext(self)
+
+ def peekPrevious(self):
+ return _pythonnator.FSIterator_peekPrevious(self)
+
+ def __init__(self):
+ _pythonnator.FSIterator_swiginit(self, _pythonnator.new_FSIterator())
+ __swig_destroy__ = _pythonnator.delete_FSIterator
+
+# Register FSIterator in _pythonnator:
+_pythonnator.FSIterator_swigregister(FSIterator)
+class ANIterator(FSIterator):
+ thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag")
+
+ def __init__(self, *args, **kwargs):
+ raise AttributeError("No constructor defined")
+ __repr__ = _swig_repr
+
+ def get(self):
+ return _pythonnator.ANIterator_get(self)
+
+# Register ANIterator in _pythonnator:
+_pythonnator.ANIterator_swigregister(ANIterator)
+class ResultSpecification(object):
+ thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag")
+
+ def __init__(self, *args, **kwargs):
+ raise AttributeError("No constructor defined")
+ __repr__ = _swig_repr
+
+# Register ResultSpecification in _pythonnator:
+_pythonnator.ResultSpecification_swigregister(ResultSpecification)
+class AnnotatorContext(object):
+ thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag")
+
+ def __init__(self, *args, **kwargs):
+ raise AttributeError("No constructor defined")
+ __repr__ = _swig_repr
+
+ def extractValue(self, paramName):
+ return _pythonnator.AnnotatorContext_extractValue(self, paramName)
+
+ def extractStringValue(self, paramName):
+ return _pythonnator.AnnotatorContext_extractStringValue(self, paramName)
+
+ def extractIntegerValue(self, paramName):
+ return _pythonnator.AnnotatorContext_extractIntegerValue(self, paramName)
+
+ def extractFloatValue(self, paramName):
+ return _pythonnator.AnnotatorContext_extractFloatValue(self, paramName)
+
+ def logMessage(self, message):
+ return _pythonnator.AnnotatorContext_logMessage(self, message)
+
+ def logWarning(self, message):
+ return _pythonnator.AnnotatorContext_logWarning(self, message)
+
+ def logError(self, message):
+ return _pythonnator.AnnotatorContext_logError(self, message)
+
+# Register AnnotatorContext in _pythonnator:
+_pythonnator.AnnotatorContext_swigregister(AnnotatorContext)
+class FSIndex(object):
+ thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag")
+ __repr__ = _swig_repr
+
+ def isValid(self):
+ return _pythonnator.FSIndex_isValid(self)
+
+ def getSize(self):
+ return _pythonnator.FSIndex_getSize(self)
+
+ def find(self, anFS):
+ return _pythonnator.FSIndex_find(self, anFS)
+
+ def iterator(self):
+ return _pythonnator.FSIndex_iterator(self)
+
+ def __init__(self):
+ _pythonnator.FSIndex_swiginit(self, _pythonnator.new_FSIndex())
+ __swig_destroy__ = _pythonnator.delete_FSIndex
+
+# Register FSIndex in _pythonnator:
+_pythonnator.FSIndex_swigregister(FSIndex)
+class ANIndex(FSIndex):
+ thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag")
+ __repr__ = _swig_repr
+
+ def iterator(self):
+ return _pythonnator.ANIndex_iterator(self)
+
+ def __init__(self):
+ _pythonnator.ANIndex_swiginit(self, _pythonnator.new_ANIndex())
+ __swig_destroy__ = _pythonnator.delete_ANIndex
+
+# Register ANIndex in _pythonnator:
+_pythonnator.ANIndex_swigregister(ANIndex)
+
diff --git a/examples/src/ExampleApplication.cpp b/examples/src/ExampleApplication.cpp
index 328a722..48e0332 100644
--- a/examples/src/ExampleApplication.cpp
+++ b/examples/src/ExampleApplication.cpp
@@ -24,6 +24,7 @@
#include <sys/stat.h>
using namespace std;
+using namespace icu_72;
using namespace uima;
/**
* An example application that reads documents from files, sends them
diff --git a/src/test/src/SimpleTextSegmenter.cpp b/examples/src/SimpleTextSegmenter.cpp
similarity index 100%
copy from src/test/src/SimpleTextSegmenter.cpp
copy to examples/src/SimpleTextSegmenter.cpp
diff --git a/examples/src/SofaDataAnnotator.cpp b/examples/src/SofaDataAnnotator.cpp
index 3816b0d..ff64924 100644
--- a/examples/src/SofaDataAnnotator.cpp
+++ b/examples/src/SofaDataAnnotator.cpp
@@ -19,6 +19,7 @@
#include "uima/api.hpp"
using namespace std;
+using namespace icu_72;
using namespace uima;
diff --git a/examples/src/SofaExampleAnnotator.cpp b/examples/src/SofaExampleAnnotator.cpp
index 632c223..755e775 100644
--- a/examples/src/SofaExampleAnnotator.cpp
+++ b/examples/src/SofaExampleAnnotator.cpp
@@ -19,6 +19,7 @@
#include "uima/api.hpp"
using namespace std;
+using namespace icu_72;
using namespace uima;
const UChar * translate(UChar *);
diff --git a/examples/src/SofaExampleApplication.cpp b/examples/src/SofaExampleApplication.cpp
index 312635f..abd231e 100644
--- a/examples/src/SofaExampleApplication.cpp
+++ b/examples/src/SofaExampleApplication.cpp
@@ -35,6 +35,7 @@
#include "uima/envvars.h"
#include "unicode/ucnv.h"
using namespace std;
+using namespace icu_72;
using namespace uima;
/* ----------------------------------------------------------------------- */
diff --git a/src/test/src/SofaStreamHandlerFile.cpp b/examples/src/SofaStreamHandlerFile.cpp
similarity index 100%
copy from src/test/src/SofaStreamHandlerFile.cpp
copy to examples/src/SofaStreamHandlerFile.cpp
diff --git a/examples/src/XCasWriterCasConsumer.cpp b/examples/src/XCasWriterCasConsumer.cpp
index 736caeb..5b02564 100644
--- a/examples/src/XCasWriterCasConsumer.cpp
+++ b/examples/src/XCasWriterCasConsumer.cpp
@@ -21,6 +21,7 @@
#include "uima/dirwalk.hpp"
#include "uima/xmlwriter.hpp"
using namespace std;
+using namespace icu_72;
using namespace uima;
diff --git a/scriptators/perl/Makefile b/scriptators/perl/Makefile
index c394fb8..6963368 100644
--- a/scriptators/perl/Makefile
+++ b/scriptators/perl/Makefile
@@ -19,8 +19,10 @@ ifeq ($(UIMACPP_HOME),)
$(error UIMACPP_HOME not set)
endif
-SWIGVERSION:=$(shell swig -version | grep SWIG | sed "s/SWIG Version //" )
-#SWIGVERSION:=$(subst SWIG Version ,,$(filter SWIG Version 1.3.%,$(shell swig -version 2>&1)))
+SWIG=swig4.0
+
+SWIGVERSION:=$(shell $(SWIG) -version | grep SWIG | sed "s/SWIG Version //" )
+#SWIGVERSION:=$(subst SWIG Version ,,$(filter SWIG Version 1.3.%,$(shell $(SWIG) -version 2>&1)))
ifeq ($(SWIGVERSION),)
$(error unable to determine SWIGVERSION)
@@ -59,10 +61,10 @@ test:
echo done
perltator.pm uima_wrap.cxx: ../uima.i
- swig -o uima_wrap.cxx -outdir . -c++ -perl ../uima.i
+ $(SWIG) -o uima_wrap.cxx -outdir . -c++ -perl ../uima.i
uimaperlwrap.h:
- swig -outdir . -c++ -perl -external-runtime $@
+ $(SWIG) -outdir . -c++ -perl -external-runtime $@
Perltator.o: $(SWIGDEPS) ../ThreadAnnotator.h
diff --git a/scriptators/perl/Perltator.cpp b/scriptators/perl/Perltator.cpp
index b3e9176..ecea82b 100644
--- a/scriptators/perl/Perltator.cpp
+++ b/scriptators/perl/Perltator.cpp
@@ -28,6 +28,7 @@
#endif
using namespace uima;
+using namespace icu;
using namespace std;
#define MODULENAME "Perltator"
@@ -119,7 +120,7 @@ public:
// convert cas and rs to python variables (parameters)
- swig_module_info *module = SWIG_Perl_GetModule();
+ swig_module_info *module = SWIG_Perl_GetModule(NULL);
if (!module) {
cerr << MODULENAME ": could not get Perl swig module" << endl;
return UIMA_ERR_USER_ANNOTATOR_COULD_NOT_INIT;
diff --git a/scriptators/perl/Perltator.xml b/scriptators/perl/Perltator.xml
new file mode 100644
index 0000000..3cf2b83
--- /dev/null
+++ b/scriptators/perl/Perltator.xml
@@ -0,0 +1,109 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+
+ <!--
+ ***************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ ***************************************************************
+ -->
+
+<taeDescription xmlns="http://uima.apache.org/resourceSpecifier">
+ <frameworkImplementation>org.apache.uima.cpp</frameworkImplementation>
+ <primitive>true</primitive>
+<annotatorImplementationName>libperltator</annotatorImplementationName>
+
+
+<analysisEngineMetaData>
+ <name>Perl Sample</name>
+ <description>Perl Annotator</description>
+ <version>1.0</version>
+ <vendor>Apache</vendor>
+
+
+<!--
+ Configuration Parameter Definitions
+-->
+ <configurationParameters>
+ <configurationParameter>
+ <name>DebugLevel</name>
+ <description>Sets the perl annotators debug level</description>
+ <type>Integer</type>
+ <multiValued>false</multiValued>
+ <mandatory>true</mandatory>
+ </configurationParameter>
+ <configurationParameter>
+ <name>SourceFile</name>
+ <description>Perl code to be loaded</description>
+ <type>String</type>
+ <multiValued>false</multiValued>
+ <mandatory>true</mandatory>
+ </configurationParameter>
+ </configurationParameters>
+
+<!--
+ Values for the configuration parameters
+-->
+ <configurationParameterSettings>
+ <nameValuePair>
+ <name>SourceFile</name>
+ <value>
+ <string>/usr/local/uimacpp/scripts/ae.pl</string>
+ </value>
+ </nameValuePair>
+ <nameValuePair>
+ <name>DebugLevel</name>
+ <value>
+ <integer>100</integer>
+ </value>
+ </nameValuePair>
+ </configurationParameterSettings>
+
+
+<!--
+ TypeSystem Definition
+-->
+
+<typeSystemDescription>
+ <types>
+ <typeDescription>
+ <name>com.ibm.uima.examples.keyword</name>
+ <description></description>
+ <supertypeName>uima.tcas.Annotation</supertypeName>
+ <features>
+ </features>
+ </typeDescription>
+ </types>
+</typeSystemDescription>
+
+
+<!--
+Capabilities: Inputs, Outputs, and Preconditions
+-->
+<capabilities>
+ <capability>
+ <inputs/>
+ <outputs>
+ <type allAnnotatorFeatures="true">com.ibm.uima.examples.keyword</type>
+ </outputs>
+ <languagesSupported>
+ <language>x-unspecified</language>
+ </languagesSupported>
+ </capability>
+</capabilities>
+
+</analysisEngineMetaData>
+</taeDescription>
diff --git a/scriptators/perl/ae.pl b/scriptators/perl/ae.pl
new file mode 100644
index 0000000..6618c00
--- /dev/null
+++ b/scriptators/perl/ae.pl
@@ -0,0 +1,87 @@
+#!/usr/bin/perl
+
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing,
+ # software distributed under the License is distributed on an
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ # KIND, either express or implied. See the License for the
+ # specific language governing permissions and limitations
+ # under the License.
+
+use perltator;
+use strict;
+
+sub initialize {
+ $main::ac = shift;
+ $main::matchString = 'Dave|David|human\s+rights';
+ $main::thisScript = $main::ac->extractValue("SourceFile");
+ $main::debug = $main::ac->extractIntegerValue("DebugLevel");
+ if ($main::debug > 0) {
+ print "$main::thisScript: Using match string = $main::matchString \n";
+ }
+}
+
+sub typeSystemInit {
+ my $ts = shift;
+ if ($main::debug > 10) {
+ print "$main::thisScript: Type sytem init called"
+ }
+ my $keytype = "com.ibm.uima.examples.keyword";
+ $main::keywordtype = $ts->getType($keytype);
+ if (!$main::keywordtype->isValid()) {
+ my $error = "$main::thisScript: $keytype is NOT found in type system!";
+ $main::ac->logError($error);
+ # set eval error to cause annotator to exit prematurely
+ $@ = $error;
+ }
+}
+
+#
+# the process method is passed two parameters, the CAS and
+# the ResultsSpecification
+sub process {
+ my ($tcas, $rs) = @_;
+ if ($main::debug > 10) {
+ print "$main::thisScript: This is a process function\n";
+ }
+ my $text = $tcas->getDocumentText();
+ my $index = $tcas->getIndexRepository();
+ my $annotCount = 0;
+ while ($text =~ m/($main::matchString)/igo) {
+ my $length = length($1);
+ my $end= pos($text);
+ my $fs = $tcas->createAnnotation($main::keywordtype, $end-$length, $end);
+ $index->addFS($fs);
+ $annotCount++;
+ }
+ if ($main::debug > 10) {
+ print "$main::thisScript: created $annotCount annotations\n";
+ }
+ if ($main::debug > 20) {
+ my $annots = 0;
+ my $iterator = $tcas->getAnnotationIndex($main::keywordtype)->iterator();
+ while ($iterator->isValid()) {
+ $annots += 1;
+ if ($main::debug > 30) {
+ my $anno = $iterator->get();
+ my $text = $anno->getCoveredText();
+ if (length($text)>40) {
+ $text = substr($text,0,20) . "...";
+ }
+ $text =~ s/\n+/ /g;
+ print "Annotation type=", $main::keywordtype->getName(),": \"$text\"\n";
+ }
+ $iterator->moveToNext();
+ }
+ print "$main::thisScript: found $annots annotations\n";
+ }
+}
diff --git a/scriptators/python/Makefile b/scriptators/python/Makefile
index 65b8eeb..19c9514 100644
--- a/scriptators/python/Makefile
+++ b/scriptators/python/Makefile
@@ -19,8 +19,10 @@ ifeq ($(UIMACPP_HOME),)
$(error UIMACPP_HOME not set)
endif
-SWIGVERSION:=$(shell swig -version | grep SWIG | sed "s/SWIG Version //" )
-#SWIGVERSION:=$(subst SWIG Version ,,$(filter SWIG Version $(SWIGMAJORMINOR).% ,$(shell swig -version 2>&1)))
+SWIG=swig4.0
+
+SWIGVERSION:=$(shell $(SWIG) -version | grep SWIG | sed "s/SWIG Version //" )
+#SWIGVERSION:=$(subst SWIG Version ,,$(filter SWIG Version $(SWIGMAJORMINOR).% ,$(shell $(SWIG) -version 2>&1)))
ifeq ($(SWIGVERSION),)
$(error unable to determine SWIGVERSION looking for $(SWIGMAJOR).$(MAJOR)) Rerun passing in SWIGMAJOR and SWIGMINOR)
@@ -29,7 +31,7 @@ endif
SWIGDEPS=uimapywrap.h
SWIGFLAGS=-DSWIG$(subst .,_,$(SWIGVERSION))
-PYTHONVERSION:=$(subst ., ,$(subst Python ,,$(shell python -V 2>&1)))
+PYTHONVERSION:=$(subst ., ,$(subst Python ,,$(shell python3 -V 2>&1)))
ifeq ($(PYTHONVERSION),)
$(error PYTHONVERSION could not be determined)
endif
@@ -70,10 +72,10 @@ include $(UIMACPP_HOME)/lib/base.mak
###################################
pythonnator.py uima_wrap.cxx: ../uima.i
- swig -o uima_wrap.cxx -outdir . -c++ -python ../uima.i
+ $(SWIG) -o uima_wrap.cxx -outdir . -c++ -python ../uima.i
uimapywrap.h:
- swig -outdir . -c++ -python -external-runtime $@
+ $(SWIG) -outdir . -c++ -python -external-runtime $@
pythonnator.o: $(SWIGDEPS)
diff --git a/scriptators/python/Pythonnator.xml b/scriptators/python/Pythonnator.xml
new file mode 100644
index 0000000..8cb8217
--- /dev/null
+++ b/scriptators/python/Pythonnator.xml
@@ -0,0 +1,109 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+
+ <!--
+ ***************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ ***************************************************************
+ -->
+
+<taeDescription xmlns="http://uima.apache.org/resourceSpecifier">
+ <frameworkImplementation>org.apache.uima.cpp</frameworkImplementation>
+ <primitive>true</primitive>
+<annotatorImplementationName>libpythonnator</annotatorImplementationName>
+
+
+<analysisEngineMetaData>
+ <name>Python Annotator</name>
+ <description>Run a python script</description>
+ <version>1.0</version>
+ <vendor>Apache</vendor>
+
+
+<!--
+ Configuration Parameter Definitions
+-->
+ <configurationParameters>
+ <configurationParameter>
+ <name>DebugLevel</name>
+ <description>Sets the python annotator debug level</description>
+ <type>Integer</type>
+ <multiValued>false</multiValued>
+ <mandatory>false</mandatory>
+ </configurationParameter>
+ <configurationParameter>
+ <name>SourceFile</name>
+ <description>Python module to be loaded from the PYTHONPATH</description>
+ <type>String</type>
+ <multiValued>false</multiValued>
+ <mandatory>true</mandatory>
+ </configurationParameter>
+ </configurationParameters>
+
+<!--
+ Values for the configuration parameters
+-->
+ <configurationParameterSettings>
+ <nameValuePair>
+ <name>SourceFile</name>
+ <value>
+ <string>ae</string>
+ </value>
+ </nameValuePair>
+ <nameValuePair>
+ <name>DebugLevel</name>
+ <value>
+ <integer>0</integer>
+ </value>
+ </nameValuePair>
+ </configurationParameterSettings>
+
+
+<!--
+ TypeSystem Definition
+-->
+
+<typeSystemDescription>
+ <types>
+ <typeDescription>
+ <name>com.ibm.uima.examples.keyword</name>
+ <description></description>
+ <supertypeName>uima.tcas.Annotation</supertypeName>
+ <features>
+ </features>
+ </typeDescription>
+ </types>
+</typeSystemDescription>
+
+
+<!--
+Capabilities: Inputs, Outputs, and Preconditions
+-->
+<capabilities>
+ <capability>
+ <inputs/>
+ <outputs>
+ <type allAnnotatorFeatures="true">com.ibm.uima.examples.keyword</type>
+ </outputs>
+ <languagesSupported>
+ <language>x-unspecified</language>
+ </languagesSupported>
+ </capability>
+</capabilities>
+
+</analysisEngineMetaData>
+</taeDescription>
diff --git a/scriptators/python/ae.py b/scriptators/python/ae.py
new file mode 100644
index 0000000..ae235ff
--- /dev/null
+++ b/scriptators/python/ae.py
@@ -0,0 +1,77 @@
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing,
+ # software distributed under the License is distributed on an
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ # KIND, either express or implied. See the License for the
+ # specific language governing permissions and limitations
+ # under the License.
+
+import re
+import math
+
+pattern = re.compile('Dave|David|(human\s+rights)', re.IGNORECASE)
+
+# everything in the global namespace is eval'ed during initialization
+# by the Pythonnator UIMA annotator
+def initialize(annotContext):
+ global pattern
+ global source
+ global debug
+ global ac
+ ac = annotContext
+ source = ac.extractValue("SourceFile")
+ debug = ac.extractIntegerValue("DebugLevel")
+ if debug > 0:
+ print(source + ": initialized")
+
+
+def typeSystemInit(ts):
+ global source
+ global debug
+ global ac
+ if debug > 10:
+ print(source + ": Type sytem init called")
+ global keywordtype
+ keywordtype =ts.getType('com.ibm.uima.examples.keyword')
+ if not keywordtype.isValid():
+ error = source + ": com.ibm.uima.examples.keyword is NOT found in type system!"
+ ac.logError(error)
+ raise Exception(error)
+
+#
+# the process method is passed two parameters, the CAS and
+# the ResultsSpecification
+def process(tcas, rs):
+ global keywordtype
+ global source
+ global debug
+ global ac
+ if debug > 10:
+ print(source + ": This is a process function")
+ ac.logMessage("process called")
+ text = tcas.getDocumentText()
+ index = tcas.getIndexRepository()
+ iterator = pattern.finditer(text)
+ annotCount = 0
+ for match in iterator:
+ fs = tcas.createAnnotation(keywordtype, match.start(), match.end())
+ index.addFS(fs)
+ annotCount += 1
+ if debug > 0:
+ print(source + ": created " + str(annotCount) + " annotations")
+ if debug > 20:
+ annots = 0
+ iterator = tcas.getAnnotationIndex(keywordtype).iterator()
+ while iterator.isValid():
+ annots += 1
+ iterator.moveToNext()
+ print(source + ": found " + str(annots) + " annotations")
diff --git a/scriptators/uima.i b/scriptators/uima.i
index 6d65ca3..cb485a3 100644
--- a/scriptators/uima.i
+++ b/scriptators/uima.i
@@ -165,9 +165,11 @@ static bool ConvertUnicodeStringRef(const UnicodeStringRef &ref,
argvi++;
}
-%insert(perl) {
+
+%insert(perl) %{
# these are default methods that the user overrides, they
# are here to prevent errors if they are not defined
+
package main;
sub initialize {};
sub typeSystemInit {};
@@ -176,7 +178,7 @@ sub process {};
sub reconfigure {};
sub batchProcessComplete {};
sub collectionProcessComplete {};
-}
+%}
#endif
#ifdef SWIGTCL
diff --git a/src/base.mak b/src/base.mak
index dfac98c..9edc594 100644
--- a/src/base.mak
+++ b/src/base.mak
@@ -32,8 +32,6 @@
#
# Variables which must have been previously set:
#
-# UIMACPP_HOME for the base of the UIMA C++ tree
-#
# TARGET_FILE=foo to create foo.$(DLL_SUFFIX) (shared lib)
# or foo (application)
# DEBUG=1 to build for debug
@@ -46,13 +44,11 @@
# USER_LINKFLAGS to specify additional linker parameters
# USER_XTARGET to specify extra user target to build
#
-# Mac OS X users must modify this file, changing the CC, includes and
-# libraries. Follow the README for more instructions.
#
#############################################################################
ifeq ($(UIMACPP_HOME),)
- $(error UIMACPP_HOME not set)
+ UIMACPP_HOME=/usr/local/uimacpp
endif
ifeq ($(TARGET_FILE),)
@@ -90,7 +86,7 @@ endif
endif
# include directory for compile
-INCLUDES=-I$(UIMACPP_HOME)/include -I$(UIMACPP_HOME)/include/apr-1
+INCLUDES=-I$(UIMACPP_HOME)/include -I `apr-config --includedir`
# compiler flags:
CFLAGS=-Wall -x c++ $(BUILD_CFLAGS) $(INCLUDES) $(USER_CFLAGS) \
diff --git a/src/test/fvt.sh b/src/test/fvt.sh
index 4d2ba9e..23cba20 100755
--- a/src/test/fvt.sh
+++ b/src/test/fvt.sh
@@ -17,7 +17,7 @@
if [ ! $UIMACPP_HOME ]
then
- export UIMACPP_HOME=../..
+ export UIMACPP_HOME=/usr/local/uimacpp
fi
Testlib=.libs:../framework/.libs
diff --git a/src/test/src/SimpleTextSegmenter.cpp b/src/test/src/SimpleTextSegmenter.cpp
deleted file mode 100644
index 34a738d..0000000
--- a/src/test/src/SimpleTextSegmenter.cpp
+++ /dev/null
@@ -1,193 +0,0 @@
-/** SimpleTextSegmenter.cpp
-
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
-
-
-Example CAS Multiplier
-*/
-#include <stdio.h>
-#include "uima/api.hpp"
-using namespace uima;
-using namespace std;
-
-class SimpleTextSegmenter : public Annotator {
-private:
-
- UnicodeStringRef docTextUS;
-
- size_t threshhold;
- size_t docLen;
- size_t start;
- size_t delimLen;
- size_t remainingLen;
-
- icu::UnicodeString delimUS;
- const UChar * delimP;
- const UChar * docTextBeginP;
- const UChar * remainingTextP;
- bool hasMore;
-
- AnnotatorContext * pAnc;
-
- int segmentNum;
-
- /* We have a separate function getConfigValues()
- for initialize() and reconfigure()
- */
- TyErrorId getConfigValues() {
- return (TyErrorId)UIMA_ERR_NONE;
- }
-
-
-public:
-
- SimpleTextSegmenter(void) {
- // cout << "SimpleTextSegmenter: Constructor" << endl;
- }
-
- ~SimpleTextSegmenter(void) {
- // cout << "SimpleTextSegmenter: Destructor" << endl;
- }
-
- /** */
- TyErrorId initialize(AnnotatorContext & rclAnnotatorContext) {
- cout << "SimpleTextSegmenter: initialize()" << endl;
-
- pAnc = &rclAnnotatorContext;
-
- /* default delimiter */
- delimUS = "\n";
-
- /* read in configuration parameter setting */
- icu::UnicodeString param("SegmentDelimiter");
- if (rclAnnotatorContext.isParameterDefined(param) ) {
- rclAnnotatorContext.extractValue(param, delimUS);
- }
-
- delimP = delimUS.getBuffer();
- delimLen = delimUS.length();
- if (delimLen < 1 ) {
- pAnc->getLogger().logError("initialize() Invalid delimiter specified. Must be at least one character in length");
- UIMA_EXC_THROW_NEW(Exception,
- UIMA_ERR_USER_ANNOTATOR_COULD_NOT_INIT,
- UIMA_MSG_ID_EXCON_CONFIG_VALUE_EXTRACT,
- ErrorMessage(UIMA_MSG_ID_LITERAL_STRING, "Invalid value for SegmentDelimiter"),
- ErrorInfo::unrecoverable);
- }
-
- cout << "initialize() using segment delimiter " << delimUS << " with length " << delimLen << endl;
-
- pAnc->getLogger().logMessage("initialize() Using Segment Delimiter '" + delimUS + "'");
- segmentNum=0;
- return (TyErrorId)UIMA_ERR_NONE;
- }
-
- // Segment the input text.
- TyErrorId process(CAS & cas, ResultSpecification const & crResultSpecification) {
- cout << "SimpleTextSegmenter: process()" << endl;
-
- cout << endl << "Test custom index..............." << endl;
- FSIndex featureIndex = cas.getIndexRepository().getIndex("TestIndex");
- cout << "featureIndex.isValid() = " << featureIndex.isValid() << endl;
- cout << "Test custom index ok..............." << endl << endl;
-
- // Get the text document
- docTextUS = cas.getDocumentText();
- /* Get number of Unicode chars (UTF-16 code units) */
- docLen = docTextUS.length();
- start = 0;
- remainingTextP = docTextUS.getBuffer();
- remainingLen=docLen;
- return (TyErrorId)UIMA_ERR_NONE;
- }
-
-
- bool hasNext() {
- cout << "SimpleTextSegmenter:hasNext() " << remainingLen << endl;
-
- if (remainingLen < 1 || remainingTextP==NULL) {
- hasMore=false;
- } else {
- hasMore = true;
- }
- return hasMore;
- }
-
- /** */
- CAS & next() {
-
- cout << "SimpleTextSegmenter: next()" << endl;
-
- if (!hasMore) {
- UIMA_EXC_THROW_NEW(Exception,
- UIMA_ERR_USER_ANNOTATOR_COULD_NOT_PROCESS,
- UIMA_MSG_ID_EXCON_PROCESSING_CAS,
- ErrorMessage(UIMA_MSG_ID_LITERAL_STRING, "There is not next() available."),
- ErrorInfo::unrecoverable);
- }
-
- UChar * segEndP = u_strFindFirst(remainingTextP, remainingLen, delimP, delimLen);
- size_t segLen=remainingLen;
-
- if (segEndP != NULL) {
- segLen = segEndP - remainingTextP;
- }
-
- //get a CAS from pool.
- CAS & cas = pAnc->getEmptyCAS();
-
- //create a sofa for the segment
- if (segEndP) {
- UnicodeStringRef segStr(remainingTextP, segLen+delimLen);
- remainingTextP = segEndP + delimLen;
- remainingLen = remainingLen-(segLen+delimLen);
- cas.setDocumentText(segStr.getBuffer(), segStr.length());
- } else {
- if (remainingLen > 0) { //when delim not found, create Sofa with remaining
- UnicodeStringRef segStr(remainingTextP, remainingLen);
-
- cas.setDocumentText(segStr.getBuffer(), segStr.length());
- }
- remainingLen=0;
- remainingTextP = NULL;
- }
-
- return cas;
- }
-
- int getCasInstancesRequired() {
- return 1;
- }
-
- /** */
- TyErrorId typeSystemInit(TypeSystem const & crTypeSystem) {
- cout << " typeSystemInit()" << endl;
-
- return (TyErrorId)UIMA_ERR_NONE;
- }
-
- /** */
- TyErrorId destroy() {
- cout << "SimpleTextSegmenter: destroy()" << endl;
- return (TyErrorId)UIMA_ERR_NONE;
- }
-};
-
-// This macro exports an entry point that is used to create the annotator.
-
-MAKE_AE(SimpleTextSegmenter);
diff --git a/src/test/src/SimpleTextSegmenter.cpp b/src/test/src/SimpleTextSegmenter.cpp
new file mode 120000
index 0000000..b4fd677
--- /dev/null
+++ b/src/test/src/SimpleTextSegmenter.cpp
@@ -0,0 +1 @@
+../../../examples/src/SimpleTextSegmenter.cpp
\ No newline at end of file
diff --git a/src/test/src/SofaStreamHandlerFile.cpp b/src/test/src/SofaStreamHandlerFile.cpp
deleted file mode 100644
index 6b3b788..0000000
--- a/src/test/src/SofaStreamHandlerFile.cpp
+++ /dev/null
@@ -1,256 +0,0 @@
-/*
-
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
-
------------------------------------------------------------------------------
-
- \brief An example implementation of the Sofa Stream Handler APIs
- defined in uima/sofastreamhandler.hpp
- to support the file URI scheme
-
------------------------------------------------------------------------------
-
- Description: This implements a SofaStreamHandler for a file URI scheme.
- The file to be read is specified as an argument to
- openStream(char * uriString). uriString is expected to be
- contain the URI scheme and path to file as follows:
- file:myfile
- eg: file://C:\myfile.txt on Windows
- file:///home/myid/myfile.txt on UNIX
-
- To build the SofaStreamHandlerFile DLL:
- On linux:
- make -f SofaStreamHandlerFile.mak
- On Windows:
- devenv SofaStreamHandlerFile.vcproj /build Release
-
- The handler dll must be registered with the UIMA framework
- by setting an environement variable as follows:
- On Linux:
- export UIMACPP_STREAMHANDLERS=file:SofaStreamHandlerFile
- On Windows:
- set UIMACPP_STREAMHANDLERS=file:SofaStreamHandlerFile
-
--------------------------------------------------------------------------- */
-
-
-
-#include <uima/api.hpp>
-#include <uima/sofastreamhandler.hpp>
-#include <sys/stat.h>
-#if defined(__OS_UNIX__)
-#define LSEEK lseek
-#include <unistd.h>
-#elif defined(__OS_WIN32__)
-#define LSEEK _lseeki64
-#include <io.h>
-#endif
-#include <stdio.h>
-
-
-using namespace uima;
-using namespace std;
-
-
-class SofaStreamHandlerFile : public SofaStreamHandler {
-
-
-private:
- FILE * stream;
- char * buffer;
- INT64 filesize;
- size_t buflen;
- INT64 cur_pos ;
-
-public:
-//--------------------------------------------------------------
-// openStream
-// @param - uriString in UTF-8
-//--------------------------------------------------------------
- void openStream(const char * uriString) {
- return openStream(uriString,BUFSIZ);
- ;
- }
-
-//--------------------------------------------------------------
-// openStreamWithMinInternalBufSize
-// open the file for reading and allocate an internal buffer of at least the
-// specified size.
-// @param - uriString in UTF-8
-// minimumBufferSize
-//--------------------------------------------------------------
- void openStream (const char * uriString, size_t minimumBufferSize) {
- cout << "SofaStreamHandlerFile: openStream() " << uriString << endl;
- if (strncmp(uriString, "file://", 7) == 0) {
- buffer = new char[minimumBufferSize];
- buflen = minimumBufferSize;
- cur_pos=0;
- const char * filename = uriString+7;
- stream = fopen(filename,"rb");
- if (stream == NULL) {
- UIMA_EXC_THROW_NEW(Exception,
- UIMA_ERR_USER_ANNOTATOR_COULD_NOT_INIT,
- UIMA_MSG_ID_EXCON_CONFIG_VALUE_EXTRACT,
- ErrorMessage(UIMA_MSG_ID_LITERAL_STRING, "Invalid value for SegmentDelimiter"),
- ErrorInfo::unrecoverable);
-
- } else {
- int rc = setvbuf (stream, buffer, _IOFBF, buflen);
- if (rc==0) {
- struct stat fstat;
- stat(filename, &fstat);
- filesize = fstat.st_size;
- }
- }
- } else {
- UIMA_EXC_THROW_NEW(Exception,
- UIMA_ERR_USER_ANNOTATOR_COULD_NOT_INIT,
- UIMA_MSG_ID_EXCON_CONFIG_VALUE_EXTRACT,
- ErrorMessage(UIMA_MSG_ID_LITERAL_STRING, "Invalid value for SegmentDelimiter"),
- ErrorInfo::unrecoverable);
-
- }
- return ;
- }
-
-//--------------------------------------------------------------
-// getTotalStreamSize
-// @return total size of the data
-//--------------------------------------------------------------
- INT64 getTotalStreamSize() {
- return filesize;
- }
-
-//--------------------------------------------------------------
-// getBufferSize
-// @return buffer length
-//--------------------------------------------------------------
- size_t getBufferSize() {
- return buflen;
- }
-
-//--------------------------------------------------------------
-// howManyAvailable
-// @return number of bytes available for read
-// -1 indicates EOF
-//--------------------------------------------------------------
- INT64 howManyAvailable() {
- if (filesize-cur_pos < 0) return -1;
- else return (filesize-cur_pos);
- }
-
-//--------------------------------------------------------------
-// getNext - deliver the specified number of bytes in network
-// byte order.
-// reads the specified number of bytes from the
-// current position. Advances the current position
-// by the number of bytes read. This implementation
-// assumes that the data in the file is in network
-// byte order.
-// This call blocks till read request is satisfied or
-// EOF is reached.
-//
-// @param - numBytes, the number of bytes to read
-// - pBuffer, buffer into which the bytes are to be copied.
-//
-// @returns number of bytes actually copied into buffer.
-// This may be less than the number of
-// bytes requested.
-// -1 indicates EOF.
-//--------------------------------------------------------------
- INT64 getNext(size_t numBytes,
- void * pBuffer) {
- cout << "SofaStreamHandlerFile: getNext() " << numBytes << endl;
- if (feof(stream)) {
- return -1;
- }
-
- size_t numread = fread(pBuffer,1,numBytes,stream);
- cur_pos += numread;
-
- return numread;
- }
-
-//--------------------------------------------------------------
-// seek
-// sets the position within the current stream.
-// @param - offset - number of bytes from origin
-// - origin is one of the following (taken from lseek spec):
-// If SEEK_SET, the position is set to offset bytes.
-// If SEEK_CUR, the position is set to its
-// current location plus offset bytes.
-// If SEEK_END, the position is set to the total length
-// plus offset bytes.
-// These constants are defined in stdio.h
-//
-// @return 0 indicates success
-//--------------------------------------------------------------
- int seek (INT64 offset, int origin) {
-
- if (offset == 0) {
- cur_pos = fseek(stream, offset, origin );
- return 0;
- }
-
- INT64 remaining = offset;
- long off = 0;
-
- while (remaining > 0) {
- if (remaining > LONG_MAX) {
- off = LONG_MAX;
- } else {
- off = remaining;
- }
- remaining = remaining-off;
- cur_pos = fseek(stream, off, origin );
- }
-
- if (cur_pos == -1L)
- return -3;
- else return 0 ;
-
- }
-
-//--------------------------------------------------------------
-// closeStream
-// close the stream. Delete the internal buffer.
-//-------------------------------------------------------------
- void closeStream() {
- if (stream != NULL) {
- fclose(stream);
- stream=NULL;
- }
- if (buffer != NULL) {
- delete[] buffer;
- buffer=NULL;
- }
- return;
- }
-
-//--------------------------------------------------------------
-// getDataPointer
-// This returns a pointer to the data in memory.
-// @return NULL.
-//--------------------------------------------------------------
-
- void * getDataPointer() {
- return NULL;
- }
-};
-
-MAKE_HANDLER(SofaStreamHandlerFile);
diff --git a/src/test/src/SofaStreamHandlerFile.cpp b/src/test/src/SofaStreamHandlerFile.cpp
new file mode 120000
index 0000000..6cdc148
--- /dev/null
+++ b/src/test/src/SofaStreamHandlerFile.cpp
@@ -0,0 +1 @@
+../../../examples/src/SofaStreamHandlerFile.cpp
\ No newline at end of file